Add CompilePreferences standard library

This commit adds the `CompilePreferences` standard library; a way to store a
TOML-serializable dictionary into top-level `Project.toml` files, then
force recompilation of child projects when the preferences are modified.

This commid adds the `CompilePreferences` standard library, which does
the actual writing to `Project.toml` files, as well as modifies the
loading code to check whether the preferences have changed.

Author: staticfloat

NEWS.md

@@ -111,6 +111,8 @@ Standard library changes
 * The `Pkg.Artifacts` module has been imported as a separate standard library.  It is still available as
   `Pkg.Artifacts`, however starting from Julia v1.6+, packages may import simply `Artifacts` without importing
   all of `Pkg` alongside. ([#37320])
+* A new standard library, `CompilePreferences`, has been added to allow packages to store settings within the top-
+  level `Project.toml`, and force recompilation when the preferences are changed. ([#xxxxx])
 
 #### LinearAlgebra
 

base/loading.jl

@@ -321,6 +321,31 @@ function manifest_deps_get(env::String, where::PkgId, name::String, cache::TOMLC
     return nothing
 end
 
+function uuid_in_environment(project_file::String, uuid::UUID, cache::TOMLCache)
+    # First, check to see if we're looking for the environment itself
+    proj_uuid = get(parsed_toml(cache, project_file), "uuid", nothing)
+    if proj_uuid !== nothing && UUID(proj_uuid) == uuid
+        return true
+    end
+
+    # Check to see if there's a Manifest.toml associated with this project
+    manifest_file = project_file_manifest_path(project_file, cache)
+    if manifest_file === nothing
+        return false
+    end
+    manifest = parsed_toml(cache, manifest_file)
+    for (dep_name, entries) in manifest
+        for entry in entries
+            entry_uuid = get(entry, "uuid", nothing)::Union{String, Nothing}
+            if uuid !== nothing && UUID(entry_uuid) == uuid
+                return true
+            end
+        end
+    end
+    # If all else fails, return `false`
+    return false
+end
+
 function manifest_uuid_path(env::String, pkg::PkgId, cache::TOMLCache)::Union{Nothing,String}
     project_file = env_project_file(env)
     if project_file isa String
@@ -943,7 +968,7 @@ function _require(pkg::PkgId, cache::TOMLCache)
             if (0 == ccall(:jl_generating_output, Cint, ())) || (JLOptions().incremental != 0)
                 # spawn off a new incremental pre-compile task for recursive `require` calls
                 # or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable)
-                cachefile = compilecache(pkg, path)
+                cachefile = compilecache(pkg, path, cache)
                 if isa(cachefile, Exception)
                     if precompilableerror(cachefile)
                         verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug
@@ -1183,7 +1208,7 @@ end
 @assert precompile(create_expr_cache, (String, String, typeof(_concrete_dependencies), Nothing))
 @assert precompile(create_expr_cache, (String, String, typeof(_concrete_dependencies), UUID))
 
-function compilecache_path(pkg::PkgId)::String
+function compilecache_path(pkg::PkgId, cache::TOMLCache)::String
     entrypath, entryfile = cache_file_entry(pkg)
     cachepath = joinpath(DEPOT_PATH[1], entrypath)
     isdir(cachepath) || mkpath(cachepath)
@@ -1193,6 +1218,7 @@ function compilecache_path(pkg::PkgId)::String
         crc = _crc32c(something(Base.active_project(), ""))
         crc = _crc32c(unsafe_string(JLOptions().image_file), crc)
         crc = _crc32c(unsafe_string(JLOptions().julia_bin), crc)
+        crc = _crc32c(get_preferences_hash(pkg.uuid, cache), crc)
         project_precompile_slug = slug(crc, 5)
         abspath(cachepath, string(entryfile, "_", project_precompile_slug, ".ji"))
     end
@@ -1209,14 +1235,14 @@ for important notes.
 function compilecache(pkg::PkgId, cache::TOMLCache = TOMLCache())
     path = locate_package(pkg, cache)
     path === nothing && throw(ArgumentError("$pkg not found during precompilation"))
-    return compilecache(pkg, path)
+    return compilecache(pkg, path, cache)
 end
 
 const MAX_NUM_PRECOMPILE_FILES = 10
 
-function compilecache(pkg::PkgId, path::String)
+function compilecache(pkg::PkgId, path::String, cache::TOMLCache = TOMLCache())
     # decide where to put the resulting cache file
-    cachefile = compilecache_path(pkg)
+    cachefile = compilecache_path(pkg, cache)
     cachepath = dirname(cachefile)
     # prune the directory with cache files
     if pkg.uuid !== nothing
@@ -1320,6 +1346,8 @@ function parse_cache_header(f::IO)
         end
         totbytes -= 4 + 4 + n2 + 8
     end
+    prefs_hash = read(f, UInt64)
+    totbytes -= 8
     @assert totbytes == 12 "header of cache file appears to be corrupt"
     srctextpos = read(f, Int64)
     # read the list of modules that are required to be present during loading
@@ -1332,7 +1360,7 @@ function parse_cache_header(f::IO)
         build_id = read(f, UInt64) # build id
         push!(required_modules, PkgId(uuid, sym) => build_id)
     end
-    return modules, (includes, requires), required_modules, srctextpos
+    return modules, (includes, requires), required_modules, srctextpos, prefs_hash
 end
 
 function parse_cache_header(cachefile::String; srcfiles_only::Bool=false)
@@ -1341,21 +1369,21 @@ function parse_cache_header(cachefile::String; srcfiles_only::Bool=false)
         !isvalid_cache_header(io) && throw(ArgumentError("Invalid header in cache file $cachefile."))
         ret = parse_cache_header(io)
         srcfiles_only || return ret
-        modules, (includes, requires), required_modules, srctextpos = ret
+        modules, (includes, requires), required_modules, srctextpos, prefs_hash = ret
         srcfiles = srctext_files(io, srctextpos)
         delidx = Int[]
         for (i, chi) in enumerate(includes)
             chi.filename ∈ srcfiles || push!(delidx, i)
         end
         deleteat!(includes, delidx)
-        return modules, (includes, requires), required_modules, srctextpos
+        return modules, (includes, requires), required_modules, srctextpos, prefs_hash
     finally
         close(io)
     end
 end
 
 function cache_dependencies(f::IO)
-    defs, (includes, requires), modules = parse_cache_header(f)
+    defs, (includes, requires), modules, srctextpos, prefs_hash = parse_cache_header(f)
     return modules, map(chi -> (chi.filename, chi.mtime), includes)  # return just filename and mtime
 end
 
@@ -1370,7 +1398,7 @@ function cache_dependencies(cachefile::String)
 end
 
 function read_dependency_src(io::IO, filename::AbstractString)
-    modules, (includes, requires), required_modules, srctextpos = parse_cache_header(io)
+    modules, (includes, requires), required_modules, srctextpos, prefs_hash = parse_cache_header(io)
     srctextpos == 0 && error("no source-text stored in cache file")
     seek(io, srctextpos)
     return _read_dependency_src(io, filename)
@@ -1415,6 +1443,37 @@ function srctext_files(f::IO, srctextpos::Int64)
     return files
 end
 
+# Find the Project.toml that we should load/store to for Preferences
+function get_preferences_project_path(uuid::UUID, cache::TOMLCache = TOMLCache())
+    for env in load_path()
+        project_file = env_project_file(env)
+        if !isa(project_file, String)
+            continue
+        end
+        if uuid_in_environment(project_file, uuid, cache)
+            return project_file
+        end
+    end
+    return nothing
+end
+
+function get_preferences(uuid::UUID, cache::TOMLCache = TOMLCache();
+                         prefs_key::String = "compile-preferences")
+    project_path = get_preferences_project_path(uuid, cache)
+    if project_path !== nothing
+        preferences = get(parsed_toml(cache, project_path), prefs_key, Dict{String,Any}())
+        if haskey(preferences, string(uuid))
+            return preferences[string(uuid)]
+        end
+    end
+    # Fall back to default value of "no preferences".
+    return Dict{String,Any}()
+end
+get_preferences_hash(uuid::UUID, cache::TOMLCache = TOMLCache()) = UInt64(hash(get_preferences(uuid, cache)))
+get_preferences_hash(m::Module, cache::TOMLCache = TOMLCache()) = get_preferences_hash(PkgId(m).uuid, cache)
+get_preferences_hash(::Nothing, cache::TOMLCache = TOMLCache()) = UInt64(hash(Dict{String,Any}()))
+
+
 # returns true if it "cachefile.ji" is stale relative to "modpath.jl"
 # otherwise returns the list of dependencies to also check
 stale_cachefile(modpath::String, cachefile::String) = stale_cachefile(modpath, cachefile, TOMLCache())
@@ -1425,7 +1484,7 @@ function stale_cachefile(modpath::String, cachefile::String, cache::TOMLCache)
             @debug "Rejecting cache file $cachefile due to it containing an invalid cache header"
             return true # invalid cache file
         end
-        (modules, (includes, requires), required_modules) = parse_cache_header(io)
+        modules, (includes, requires), required_modules, srctextpos, prefs_hash = parse_cache_header(io)
         id = isempty(modules) ? nothing : first(modules).first
         modules = Dict{PkgId, UInt64}(modules)
 
@@ -1501,6 +1560,12 @@ function stale_cachefile(modpath::String, cachefile::String, cache::TOMLCache)
         end
 
         if isa(id, PkgId)
+            curr_prefs_hash = get_preferences_hash(id.uuid, cache)
+            if prefs_hash != curr_prefs_hash
+                @debug "Rejecting cache file $cachefile because preferences hash does not match 0x$(string(prefs_hash, base=16)) != 0x$(string(curr_prefs_hash, base=16))"
+                return true
+            end
+
             get!(PkgOrigin, pkgorigins, id).cachepath = cachefile
         end
 

base/sysimg.jl

@@ -47,6 +47,7 @@ let
             :Distributed,
             :SharedArrays,
             :TOML,
+            :CompilePreferences,
             :Artifacts,
             :Pkg,
             :Test,

base/util.jl

@@ -385,6 +385,8 @@ _crc32c(io::IO, crc::UInt32=0x00000000) = _crc32c(io, typemax(Int64), crc)
 _crc32c(io::IOStream, crc::UInt32=0x00000000) = _crc32c(io, filesize(io)-position(io), crc)
 _crc32c(uuid::UUID, crc::UInt32=0x00000000) =
     ccall(:jl_crc32c, UInt32, (UInt32, Ref{UInt128}, Csize_t), crc, uuid.value, 16)
+_crc32c(x::Integer, crc::UInt32=0x00000000) =
+    ccall(:jl_crc32c, UInt32, (UInt32, Vector{UInt8}, Csize_t), crc, reinterpret(UInt8, [x]), sizeof(x))
 
 """
     @kwdef typedef

src/dump.c

@@ -1123,6 +1123,31 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t **udepsp, jl_array_t *
             write_int32(s, 0);
         }
         write_int32(s, 0); // terminator, for ease of reading
+
+        // Calculate CompilePreferences hash for current package.
+        jl_value_t *prefs_hash = NULL;
+        if (jl_base_module) {
+            // Toplevel module is the module we're currently compiling, use it to get our preferences hash
+            jl_value_t * toplevel = (jl_value_t*)jl_get_global(jl_base_module, jl_symbol("__toplevel__"));
+            jl_value_t * prefs_hash_func = jl_get_global(jl_base_module, jl_symbol("get_preferences_hash"));
+
+            if (toplevel && prefs_hash_func) {
+                // call get_preferences_hash(__toplevel__)
+                jl_value_t *prefs_hash_args[2] = {prefs_hash_func, (jl_value_t*)toplevel};
+                size_t last_age = jl_get_ptls_states()->world_age;
+                jl_get_ptls_states()->world_age = jl_world_counter;
+                prefs_hash = (jl_value_t*)jl_apply(prefs_hash_args, 2);
+                jl_get_ptls_states()->world_age = last_age;
+            }
+        }
+
+        // If we successfully got the preferences, write it out, otherwise write `0` for this `.ji` file.
+        if (prefs_hash != NULL) {
+            write_uint64(s, jl_unbox_uint64(prefs_hash));
+        } else {
+            write_uint64(s, 0);
+        }
+
         // write a dummy file position to indicate the beginning of the source-text
         pos = ios_pos(s);
         ios_seek(s, initial_pos);

stdlib/CompilePreferences/Project.toml

@@ -0,0 +1,12 @@
+name = "CompilePreferences"
+uuid = "21216c6a-2e73-6563-6e65-726566657250"
+
+[deps]
+TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+
+[extras]
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["Test", "Pkg"]

stdlib/CompilePreferences/docs/src/index.md

@@ -0,0 +1,47 @@
+# CompilePreferences
+
+!!! compat "Julia 1.6"
+    Julia's `CompilePreferences` API requires at least Julia 1.6.
+
+`CompilePreferences` support embedding a simple `Dict` of metadata for a package on a per-project basis.  These preferences allow for packages to set simple, persistent pieces of data, and trigger recompilation of the package when the preferences change, to allow for customization of package behavior at compile-time.
+
+## API Overview
+
+`CompilePreferences` are used primarily through the `@load_preferences`, `@save_preferences` and `@modify_preferences` macros.  These macros will auto-detect the UUID of the calling package, throwing an error if the calling module does not belong to a package.  The function forms can be used to load, save or modify preferences belonging to another package.
+
+Example usage:
+
+```julia
+using CompilePreferences
+
+function get_preferred_backend()
+    prefs = @load_preferences()
+    return get(prefs, "backend", "native")
+end
+
+function set_backend(new_backend)
+    @modify_preferences!() do prefs
+        prefs["backend"] = new_backend
+    end
+end
+```
+
+Preferences are stored within the first `Project.toml` that represents an environment that contains the given UUID, even as a transitive dependency.
+If no project that contains the given UUID is found, the preference is recorded in the `Project.toml` file of the currently-active project.
+The initial state for preferences is an empty dictionary, package authors that wish to have a default value set for their preferences should use the `get(prefs, key, default)` pattern as shown in the code example above.
+
+# API Reference
+
+!!! compat "Julia 1.6"
+    Julia's `CompilePreferences` API requires at least Julia 1.6.
+
+```@docs
+CompilePreferences.load_preferences
+CompilePreferences.@load_preferences
+CompilePreferences.save_preferences!
+CompilePreferences.@save_preferences!
+CompilePreferences.modify_preferences!
+CompilePreferences.@modify_preferences!
+CompilePreferences.clear_preferences!
+CompilePreferences.@clear_preferences!
+```