Add Preferences standard library

This commit adds the `Preferences` standard library; a way to store a
TOML-serializable dictionary into top-level `Project.toml` files, then
force recompilation of child projects when the preferences are modified.

This pull request adds the `Preferences` standard library, which does
the actual writing to `Project.toml` files, as well as modifies the
loading code to check whether the preferences have changed.

Author: staticfloat

NEWS.md

@@ -105,6 +105,8 @@ Standard library changes
 * The `Pkg.Artifacts` module has been imported as a separate standard library.  It is still available as
   `Pkg.Artifacts`, however starting from Julia v1.6+, packages may import simply `Artifacts` without importing
   all of `Pkg` alongside. ([#37320])
+* A new standard library, `Preferences`, has been added to allow packages to store settings within the top-
+  level `Project.toml`, and force recompilation when the preferences are changed. ([#xxxxx])
 
 #### LinearAlgebra
 * New method `LinearAlgebra.issuccess(::CholeskyPivoted)` for checking whether pivoted Cholesky factorization was successful ([#36002]).

base/loading.jl

@@ -1316,6 +1316,8 @@ function parse_cache_header(f::IO)
         end
         totbytes -= 4 + 4 + n2 + 8
     end
+    prefs_hash = read(f, UInt64)
+    totbytes -= 8
     @assert totbytes == 12 "header of cache file appears to be corrupt"
     srctextpos = read(f, Int64)
     # read the list of modules that are required to be present during loading
@@ -1328,7 +1330,7 @@ function parse_cache_header(f::IO)
         build_id = read(f, UInt64) # build id
         push!(required_modules, PkgId(uuid, sym) => build_id)
     end
-    return modules, (includes, requires), required_modules, srctextpos
+    return modules, (includes, requires), required_modules, srctextpos, prefs_hash
 end
 
 function parse_cache_header(cachefile::String; srcfiles_only::Bool=false)
@@ -1337,21 +1339,21 @@ function parse_cache_header(cachefile::String; srcfiles_only::Bool=false)
         !isvalid_cache_header(io) && throw(ArgumentError("Invalid header in cache file $cachefile."))
         ret = parse_cache_header(io)
         srcfiles_only || return ret
-        modules, (includes, requires), required_modules, srctextpos = ret
+        modules, (includes, requires), required_modules, srctextpos, prefs_hash = ret
         srcfiles = srctext_files(io, srctextpos)
         delidx = Int[]
         for (i, chi) in enumerate(includes)
             chi.filename ∈ srcfiles || push!(delidx, i)
         end
         deleteat!(includes, delidx)
-        return modules, (includes, requires), required_modules, srctextpos
+        return modules, (includes, requires), required_modules, srctextpos, prefs_hash
     finally
         close(io)
     end
 end
 
 function cache_dependencies(f::IO)
-    defs, (includes, requires), modules = parse_cache_header(f)
+    defs, (includes, requires), modules, srctextpos, prefs_hash = parse_cache_header(f)
     return modules, map(chi -> (chi.filename, chi.mtime), includes)  # return just filename and mtime
 end
 
@@ -1366,7 +1368,7 @@ function cache_dependencies(cachefile::String)
 end
 
 function read_dependency_src(io::IO, filename::AbstractString)
-    modules, (includes, requires), required_modules, srctextpos = parse_cache_header(io)
+    modules, (includes, requires), required_modules, srctextpos, prefs_hash = parse_cache_header(io)
     srctextpos == 0 && error("no source-text stored in cache file")
     seek(io, srctextpos)
     return _read_dependency_src(io, filename)
@@ -1411,6 +1413,20 @@ function srctext_files(f::IO, srctextpos::Int64)
     return files
 end
 
+function get_preferences_hash(uuid::UUID, cache::TOMLCache = TOMLCache())
+    # check that project preferences match by first loading the Project.toml
+    active_project_file = Base.active_project()
+    if isfile(active_project_file)
+        preferences = get(parsed_toml(cache, active_project_file), "preferences", Dict{String,Any}())
+        if haskey(preferences, string(uuid))
+            return UInt64(hash(preferences[string(uuid)]))
+        end
+    end
+    return UInt64(hash(Dict{String,Any}()))
+end
+get_preferences_hash(::Nothing, cache::TOMLCache = TOMLCache()) = UInt64(hash(Dict{String,Any}()))
+get_preferences_hash(m::Module, cache::TOMLCache = TOMLCache()) = get_preferences_hash(PkgId(m).uuid, cache)
+
 # returns true if it "cachefile.ji" is stale relative to "modpath.jl"
 # otherwise returns the list of dependencies to also check
 stale_cachefile(modpath::String, cachefile::String) = stale_cachefile(modpath, cachefile, TOMLCache())
@@ -1421,7 +1437,7 @@ function stale_cachefile(modpath::String, cachefile::String, cache::TOMLCache)
             @debug "Rejecting cache file $cachefile due to it containing an invalid cache header"
             return true # invalid cache file
         end
-        (modules, (includes, requires), required_modules) = parse_cache_header(io)
+        modules, (includes, requires), required_modules, srctextpos, prefs_hash = parse_cache_header(io)
         id = isempty(modules) ? nothing : first(modules).first
         modules = Dict{PkgId, UInt64}(modules)
 
@@ -1496,6 +1512,12 @@ function stale_cachefile(modpath::String, cachefile::String, cache::TOMLCache)
         end
 
         if isa(id, PkgId)
+            curr_prefs_hash = get_preferences_hash(id.uuid, cache)
+            if prefs_hash != curr_prefs_hash
+                @debug "Rejecting cache file $cachefile because preferences hash does not match 0x$(string(prefs_hash, base=16)) != 0x$(string(curr_prefs_hash, base=16))"
+                return true
+            end
+
             pkgorigins[id] = PkgOrigin(cachefile)
         end
 

base/sysimg.jl

@@ -47,6 +47,7 @@ let
             :Distributed,
             :SharedArrays,
             :TOML,
+            :Preferences,
             :Artifacts,
             :Pkg,
             :Test,

src/dump.c

@@ -1123,6 +1123,31 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t **udepsp, jl_array_t *
             write_int32(s, 0);
         }
         write_int32(s, 0); // terminator, for ease of reading
+
+        // Calculate Preferences hash for current package.
+        jl_value_t *prefs_hash = NULL;
+        if (jl_base_module) {
+            // Toplevel module is the module we're currently compiling, use it to get our preferences hash
+            jl_value_t * toplevel = (jl_value_t*)jl_get_global(jl_base_module, jl_symbol("__toplevel__"));
+            jl_value_t * prefs_hash_func = jl_get_global(jl_base_module, jl_symbol("get_preferences_hash"));
+
+            if (toplevel && prefs_hash_func) {
+                // call get_preferences_hash(__toplevel__)
+                jl_value_t *prefs_hash_args[2] = {prefs_hash_func, (jl_value_t*)toplevel};
+                size_t last_age = jl_get_ptls_states()->world_age;
+                jl_get_ptls_states()->world_age = jl_world_counter;
+                prefs_hash = (jl_value_t*)jl_apply(prefs_hash_args, 2);
+                jl_get_ptls_states()->world_age = last_age;
+            }
+        }
+
+        // If we successfully got the preferences, write it out, otherwise write `0` for this `.ji` file.
+        if (prefs_hash != NULL) {
+            write_uint64(s, jl_unbox_uint64(prefs_hash));
+        } else {
+            write_uint64(s, 0);
+        }
+
         // write a dummy file position to indicate the beginning of the source-text
         pos = ios_pos(s);
         ios_seek(s, initial_pos);

stdlib/Makefile

@@ -16,7 +16,7 @@ $(build_datarootdir)/julia/stdlib/$(VERSDIR):
 
 STDLIBS = Artifacts Base64 CRC32c Dates DelimitedFiles Distributed FileWatching \
           Future InteractiveUtils Libdl LibGit2 LinearAlgebra Logging \
-          Markdown Mmap Printf Profile Random REPL Serialization SHA \
+          Markdown Mmap Preferences Printf Profile Random REPL Serialization SHA \
           SharedArrays Sockets SparseArrays SuiteSparse Test TOML Unicode UUIDs
 STDLIBS_EXT = Pkg Statistics
 PKG_GIT_URL := git://github.com/JuliaLang/Pkg.jl.git

stdlib/Preferences/Project.toml

@@ -0,0 +1,12 @@
+name = "Preferences"
+uuid = "21216c6a-2e73-6563-6e65-726566657250"
+
+[deps]
+TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+
+[extras]
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["Test", "Pkg"]

stdlib/Preferences/docs/src/index.md

@@ -0,0 +1,46 @@
+# Preferences
+
+!!! compat "Julia 1.6"
+    Julia's `Preferences` API requires at least Julia 1.6.
+
+Preferences support embedding a simple `Dict` of metadata for a package on a per-project basis.  These preferences allow for packages to set simple, persistent pieces of data that the user has selected, that can persist across multiple versions of a package.
+
+## API Overview
+
+`Preferences` are used primarily through the `@load_preferences`, `@save_preferences` and `@modify_preferences` macros.  These macros will auto-detect the UUID of the calling package, throwing an error if the calling module does not belong to a package.  The function forms can be used to load, save or modify preferences belonging to another package.
+
+Example usage:
+
+```julia
+using Preferences
+
+function get_preferred_backend()
+    prefs = @load_preferences()
+    return get(prefs, "backend", "native")
+end
+
+function set_backend(new_backend)
+    @modify_preferences!() do prefs
+        prefs["backend"] = new_backend
+    end
+end
+```
+
+By default, preferences are stored within the `Project.toml` file of the currently-active project, and as such all new projects will start from a blank state, with all preferences being un-set.
+Package authors that wish to have a default value set for their preferences should use the `get(prefs, key, default)` pattern as shown in the code example above.
+
+# API Reference
+
+!!! compat "Julia 1.6"
+    Julia's `Preferences` API requires at least Julia 1.6.
+
+```@docs
+Preferences.load_preferences
+Preferences.@load_preferences
+Preferences.save_preferences!
+Preferences.@save_preferences!
+Preferences.modify_preferences!
+Preferences.@modify_preferences!
+Preferences.clear_preferences!
+Preferences.@clear_preferences!
+```

stdlib/Preferences/src/Preferences.jl

@@ -0,0 +1,171 @@
+module Preferences
+using TOML
+using Base: UUID
+
+export load_preferences, @load_preferences,
+       save_preferences!, @save_preferences!,
+       modify_preferences!, @modify_preferences!,
+       clear_preferences!, @clear_preferences!
+
+# Helper function to get the UUID of a module, throwing an error if it can't.
+function get_uuid(m::Module)
+    uuid = Base.PkgId(m).uuid
+    if uuid === nothing
+        throw(ArgumentError("Module does not correspond to a loaded package!"))
+    end
+    return uuid
+end
+
+
+"""
+    load_preferences(uuid_or_module)
+
+Load the preferences for the given package, returning them as a `Dict`.  Most users
+should use the `@load_preferences()` macro which auto-determines the calling `Module`.
+"""
+function load_preferences(uuid::UUID)
+    prefs = Dict{String,Any}()
+
+    # Finally, load from the currently-active project:
+    proj_path = Base.active_project()
+    if isfile(proj_path)
+        project = TOML.parsefile(proj_path)
+        if haskey(project, "preferences") && isa(project["preferences"], Dict)
+            prefs = get(project["preferences"], string(uuid), Dict())
+        end
+    end
+    return prefs
+end
+load_preferences(m::Module) = load_preferences(get_uuid(m))
+
+
+"""
+    save_preferences!(uuid_or_module, prefs::Dict)
+
+Save the preferences for the given package.  Most users should use the
+`@save_preferences!()` macro which auto-determines the calling `Module`.  See also the
+`modify_preferences!()` function (and the associated `@modifiy_preferences!()` macro) for
+easy load/modify/save workflows.
+"""
+function save_preferences!(uuid::UUID, prefs::Dict)
+    # Save to Project.toml
+    proj_path = Base.active_project()
+    mkpath(dirname(proj_path))
+    project = Dict{String,Any}()
+    if isfile(proj_path)
+        project = TOML.parsefile(proj_path)
+    end
+    if !haskey(project, "preferences")
+        project["preferences"] = Dict{String,Any}()
+    end
+    if !isa(project["preferences"], Dict)
+        error("$(proj_path) has conflicting `preferences` entry type: Not a Dict!")
+    end
+    project["preferences"][string(uuid)] = prefs
+    open(proj_path, "w") do io
+        TOML.print(io, project, sorted=true)
+    end
+    return nothing
+end
+function save_preferences!(m::Module, prefs::Dict)
+    return save_preferences!(get_uuid(m), prefs)
+end
+
+
+"""
+    modify_preferences!(f::Function, uuid::UUID)
+    modify_preferences!(f::Function, m::Module)
+
+Supports `do`-block modification of preferences.  Loads the preferences, passes them to a
+user function, then writes the modified `Dict` back to the preferences file.  Example:
+
+```julia
+modify_preferences!(@__MODULE__) do prefs
+    prefs["key"] = "value"
+end
+```
+
+This function returns the full preferences object.  Most users should use the
+`@modify_preferences!()` macro which auto-determines the calling `Module`.
+Note that this method does not support modifying depot-wide preferences; modifications
+always are saved to the active project.
+"""
+function modify_preferences!(f::Function, uuid::UUID)
+    prefs = load_preferences(uuid)
+    f(prefs)
+    save_preferences!(uuid, prefs)
+    return prefs
+end
+modify_preferences!(f::Function, m::Module) = modify_preferences!(f, get_uuid(m))
+
+
+"""
+    clear_preferences!(uuid::UUID)
+    clear_preferences!(m::Module)
+
+Convenience method to remove all preferences for the given package.  Most users should
+use the `@clear_preferences!()` macro, which auto-determines the calling `Module`.
+"""
+function clear_preferences!(uuid::UUID)
+    # Clear the project preferences key, if it exists
+    proj_path = Base.active_project()
+    if isfile(proj_path)
+        project = TOML.parsefile(proj_path)
+        if haskey(project, "preferences") && isa(project["preferences"], Dict)
+            delete!(project["preferences"], string(uuid))
+            open(proj_path, "w") do io
+                TOML.print(io, project, sorted=true)
+            end
+        end
+    end
+end
+
+
+"""
+    @load_preferences()
+
+Convenience macro to call `load_preferences()` for the current package.
+"""
+macro load_preferences()
+    return quote
+        load_preferences($(esc(get_uuid(__module__))))
+    end
+end
+
+
+"""
+    @save_preferences!(prefs)
+
+Convenience macro to call `save_preferences!()` for the current package.
+"""
+macro save_preferences!(prefs)
+    return quote
+        save_preferences!($(esc(get_uuid(__module__))), $(esc(prefs)))
+    end
+end
+
+
+"""
+    @modify_preferences!(func)
+
+Convenience macro to call `modify_preferences!()` for the current package.
+"""
+macro modify_preferences!(func)
+    return quote
+        modify_preferences!($(esc(func)), $(esc(get_uuid(__module__))))
+    end
+end
+
+
+"""
+    @clear_preferences!()
+
+Convenience macro to call `clear_preferences!()` for the current package.
+"""
+macro clear_preferences!()
+    return quote
+        preferences!($(esc(get_uuid(__module__))))
+    end
+end
+
+end # module Preferences

stdlib/Preferences/test/UsesPreferences/Project.toml

@@ -0,0 +1,12 @@
+name = "UsesPreferences"
+uuid = "056c4eb5-4491-6b91-3d28-8fffe3ee2af9"
+version = "0.1.0"
+
+[deps]
+Preferences = "21216c6a-2e73-6563-6e65-726566657250"
+
+[extras]
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["Test"]