[loading]: Rework preferences loading

Implements the `Preferences` loading framework as outlined in [0]. The
most drastic change is that the list of compile-time preferences is no
longer sequestered within its own dictionary, but is instead
autodetected at compile-time and communicated back to the compiler.
This list of compile-time preferences is now embedded as an array of
strings that the loader must load, then index into the preferences
dictionary with that list to check the preferences hash.

In a somewhat bizarre turn of events, because we want the `.ji` filename
to incorporate the preferences hash, and because we can't know how to
generate the hash until after we've precompiled, I had to move the `.ji`
filename generation step to _after_ we precompile the `.ji` file.

[0]: #37791 (comment)

Author: staticfloat

base/loading.jl

@@ -273,6 +273,8 @@ end
 
 const project_names = ("JuliaProject.toml", "Project.toml")
 const manifest_names = ("JuliaManifest.toml", "Manifest.toml")
+const preferences_names = ("JuliaPreferences.toml", "Preferences.toml")
+const local_preferences_names = ("LocalJuliaPreferences.toml", "LocalPreferences.toml")
 
 # classify the LOAD_PATH entry to be one of:
 #  - `false`: nonexistant / nothing to see here
@@ -322,31 +324,6 @@ function manifest_deps_get(env::String, where::PkgId, name::String, cache::TOMLC
     return nothing
 end
 
-function uuid_in_environment(project_file::String, uuid::UUID, cache::TOMLCache)
-    # First, check to see if we're looking for the environment itself
-    proj_uuid = get(parsed_toml(cache, project_file), "uuid", nothing)
-    if proj_uuid !== nothing && UUID(proj_uuid) == uuid
-        return true
-    end
-
-    # Check to see if there's a Manifest.toml associated with this project
-    manifest_file = project_file_manifest_path(project_file, cache)
-    if manifest_file === nothing
-        return false
-    end
-    manifest = parsed_toml(cache, manifest_file)
-    for (dep_name, entries) in manifest
-        for entry in entries
-            entry_uuid = get(entry, "uuid", nothing)::Union{String, Nothing}
-            if uuid !== nothing && UUID(entry_uuid) == uuid
-                return true
-            end
-        end
-    end
-    # If all else fails, return `false`
-    return false
-end
-
 function manifest_uuid_path(env::String, pkg::PkgId, cache::TOMLCache)::Union{Nothing,String}
     project_file = env_project_file(env)
     if project_file isa String
@@ -1220,7 +1197,12 @@ end
 @assert precompile(create_expr_cache, (PkgId, String, String, typeof(_concrete_dependencies), Bool))
 @assert precompile(create_expr_cache, (PkgId, String, String, typeof(_concrete_dependencies), Bool))
 
-function compilecache_path(pkg::PkgId, cache::TOMLCache)::String
+function compilecache_dir(pkg::PkgId)
+    entrypath, entryfile = cache_file_entry(pkg)
+    return joinpath(DEPOT_PATH[1], entrypath)
+end
+
+function compilecache_path(pkg::PkgId, prefs_hash::UInt64, cache::TOMLCache)::String
     entrypath, entryfile = cache_file_entry(pkg)
     cachepath = joinpath(DEPOT_PATH[1], entrypath)
     isdir(cachepath) || mkpath(cachepath)
@@ -1230,7 +1212,7 @@ function compilecache_path(pkg::PkgId, cache::TOMLCache)::String
         crc = _crc32c(something(Base.active_project(), ""))
         crc = _crc32c(unsafe_string(JLOptions().image_file), crc)
         crc = _crc32c(unsafe_string(JLOptions().julia_bin), crc)
-        crc = _crc32c(get_preferences_hash(pkg.uuid, cache), crc)
+        crc = _crc32c(prefs_hash, crc)
         project_precompile_slug = slug(crc, 5)
         abspath(cachepath, string(entryfile, "_", project_precompile_slug, ".ji"))
     end
@@ -1254,17 +1236,8 @@ const MAX_NUM_PRECOMPILE_FILES = 10
 
 function compilecache(pkg::PkgId, path::String, cache::TOMLCache = TOMLCache(), show_errors::Bool = true)
     # decide where to put the resulting cache file
-    cachefile = compilecache_path(pkg, cache)
-    cachepath = dirname(cachefile)
-    # prune the directory with cache files
-    if pkg.uuid !== nothing
-        entrypath, entryfile = cache_file_entry(pkg)
-        cachefiles = filter!(x -> startswith(x, entryfile * "_"), readdir(cachepath))
-        if length(cachefiles) >= MAX_NUM_PRECOMPILE_FILES
-            idx = findmin(mtime.(joinpath.(cachepath, cachefiles)))[2]
-            rm(joinpath(cachepath, cachefiles[idx]))
-        end
-    end
+    cachepath = compilecache_dir(pkg)
+
     # build up the list of modules that we want the precompile process to preserve
     concrete_deps = copy(_concrete_dependencies)
     for (key, mod) in loaded_modules
@@ -1278,6 +1251,7 @@ function compilecache(pkg::PkgId, path::String, cache::TOMLCache = TOMLCache(),
 
     # create a temporary file in `cachepath` directory, write the cache in it,
     # write the checksum, _and then_ atomically move the file to `cachefile`.
+    mkpath(cachepath)
     tmppath, tmpio = mktemp(cachepath)
     local p
     try
@@ -1291,6 +1265,21 @@ function compilecache(pkg::PkgId, path::String, cache::TOMLCache = TOMLCache(),
             # inherit permission from the source file
             chmod(tmppath, filemode(path) & 0o777)
 
+            # Read preferences hash back from .ji file (we can't precompute because
+            # we don't actually know what the list of compile-time preferences are without compiling)
+            prefs_hash = preferences_hash(tmppath)
+            cachefile = compilecache_path(pkg, prefs_hash, cache)
+
+            # prune the directory with cache files
+            if pkg.uuid !== nothing
+                entrypath, entryfile = cache_file_entry(pkg)
+                cachefiles = filter!(x -> startswith(x, entryfile * "_"), readdir(cachepath))
+                if length(cachefiles) >= MAX_NUM_PRECOMPILE_FILES
+                    idx = findmin(mtime.(joinpath.(cachepath, cachefiles)))[2]
+                    rm(joinpath(cachepath, cachefiles[idx]))
+                end
+            end
+
             # this is atomic according to POSIX:
             rename(tmppath, cachefile; force=true)
             return cachefile
@@ -1334,7 +1323,10 @@ function parse_cache_header(f::IO)
     requires = Pair{PkgId, PkgId}[]
     while true
         n2 = read(f, Int32)
-        n2 == 0 && break
+        if n2 == 0
+            totbytes -= 4
+            break
+        end
         depname = String(read(f, n2))
         mtime = read(f, Float64)
         n1 = read(f, Int32)
@@ -1358,10 +1350,20 @@ function parse_cache_header(f::IO)
         end
         totbytes -= 4 + 4 + n2 + 8
     end
+    prefs = String[]
+    while true
+        n2 = read(f, Int32)
+        if n2 == 0
+            totbytes -= 4
+            break
+        end
+        push!(prefs, String(read(f, n2)))
+        totbytes -= 4 + n2
+    end
     prefs_hash = read(f, UInt64)
-    totbytes -= 8
-    @assert totbytes == 12 "header of cache file appears to be corrupt"
     srctextpos = read(f, Int64)
+    totbytes -= 8 + 8
+    @assert totbytes == 0 "header of cache file appears to be corrupt"
     # read the list of modules that are required to be present during loading
     required_modules = Vector{Pair{PkgId, UInt64}}()
     while true
@@ -1372,7 +1374,7 @@ function parse_cache_header(f::IO)
         build_id = read(f, UInt64) # build id
         push!(required_modules, PkgId(uuid, sym) => build_id)
     end
-    return modules, (includes, requires), required_modules, srctextpos, prefs_hash
+    return modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash
 end
 
 function parse_cache_header(cachefile::String; srcfiles_only::Bool=false)
@@ -1381,21 +1383,37 @@ function parse_cache_header(cachefile::String; srcfiles_only::Bool=false)
         !isvalid_cache_header(io) && throw(ArgumentError("Invalid header in cache file $cachefile."))
         ret = parse_cache_header(io)
         srcfiles_only || return ret
-        modules, (includes, requires), required_modules, srctextpos, prefs_hash = ret
+        modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash = ret
         srcfiles = srctext_files(io, srctextpos)
         delidx = Int[]
         for (i, chi) in enumerate(includes)
             chi.filename ∈ srcfiles || push!(delidx, i)
         end
         deleteat!(includes, delidx)
-        return modules, (includes, requires), required_modules, srctextpos, prefs_hash
+        return modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash
+    finally
+        close(io)
+    end
+end
+
+
+
+preferences_hash(f::IO) = parse_cache_header(f)[end]
+function preferences_hash(cachefile::String)
+    io = open(cachefile, "r")
+    try
+        if !isvalid_cache_header(io)
+            throw(ArgumentError("Invalid header in cache file $cachefile."))
+        end
+        return preferences_hash(io)
     finally
         close(io)
     end
 end
 
+
 function cache_dependencies(f::IO)
-    defs, (includes, requires), modules, srctextpos, prefs_hash = parse_cache_header(f)
+    defs, (includes, requires), modules, srctextpos, prefs, prefs_hash = parse_cache_header(f)
     return modules, map(chi -> (chi.filename, chi.mtime), includes)  # return just filename and mtime
 end
 
@@ -1410,7 +1428,7 @@ function cache_dependencies(cachefile::String)
 end
 
 function read_dependency_src(io::IO, filename::AbstractString)
-    modules, (includes, requires), required_modules, srctextpos, prefs_hash = parse_cache_header(io)
+    modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash = parse_cache_header(io)
     srctextpos == 0 && error("no source-text stored in cache file")
     seek(io, srctextpos)
     return _read_dependency_src(io, filename)
@@ -1455,36 +1473,132 @@ function srctext_files(f::IO, srctextpos::Int64)
     return files
 end
 
-# Find the Project.toml that we should load/store to for Preferences
-function get_preferences_project_path(uuid::UUID, cache::TOMLCache = TOMLCache())
+function get_uuid_name(project_toml::String, uuid::UUID, cache::TOMLCache = TOMLCache())
+    d = parsed_toml(cache, project_toml)
+
+    # Test to see if this UUID is mentioned in this `Project.toml`; either as
+    # the top-level UUID (e.g. that of the project itself) or as a dependency.
+    if haskey(d, "uuid") && haskey(d, "name") && UUID(d["uuid"]) == uuid
+        return d["name"]
+    elseif haskey(d, "deps")
+        # Search for the UUID as one of the deps
+        for (k, v) in d["deps"]
+            if v == uuid
+                return k
+            end
+        end
+    end
+
+    return nothing
+end
+
+function collect_preferences!(dicts::Vector{Dict}, project_toml::String,
+                              pkg_name::String, cache::TOMLCache = TOMLCache())
+    # Look first for `Preferences.toml`, then for `LocalPreferences.toml`
+    for names_collection in (preferences_names, local_preferences_names)
+        for name in names_collection
+            toml_path = joinpath(project_dir, name)
+            if isfile(toml_path)
+                prefs = TOML.parsefile(toml_path)
+                push!(dicts, get(prefs, pkg_name, Dict()))
+
+                # If we find `JuliaPreferences.toml`, don't look for `Preferences.toml`,
+                # but do look for `Local(Julia)Preferences.toml`
+                break
+            end
+        end
+    end
+
+    return dicts
+end
+
+function get_clear_keys(d::Dict)
+    if haskey(d, "__clear_keys__") && isa(d["__clear_keys__"], Vector)
+        return d["__clear_keys__"]
+    end
+    return String[]
+end
+
+"""
+    recursive_merge(base::Dict, overrides::Dict...)
+
+Helper function to merge preference dicts recursively, honoring overrides in nested
+dictionaries properly.
+"""
+function recursive_merge(base::Dict, overrides::Dict...)
+    new_base = Base._typeddict(base, overrides...)
+
+    for override in overrides
+        # Clear keys are keys that should be deleted from any previous setting.
+        clear_keys = get_clear_keys(base)
+        for k in clear_keys
+            delete!(new_base, k)
+        end
+
+        for (k, v) in override
+            # Note that if `base` has a mapping that is _not_ a `Dict`, and `override`
+            if haskey(new_base, k) && isa(new_base[k], Dict) && isa(override[k], Dict)
+                new_base[k] = recursive_merge(new_base[k], override[k])
+            else
+                new_base[k] = override[k]
+            end
+        end
+    end
+    return new_base
+end
+
+function get_preferences(uuid::UUID, cache::TOMLCache = TOMLCache())
+    merged_prefs = Dict{String,Any}()
     for env in load_path()
-        project_file = env_project_file(env)
-        if !isa(project_file, String)
+        project_toml = env_project_file(env)
+        if !isa(project_toml, String)
             continue
         end
-        if uuid_in_environment(project_file, uuid, cache)
-            return project_file
+
+        pkg_name = get_uuid_name(project_toml, uuid, cache)
+        if pkg_name !== nothing
+            project_dir = dirname(project_toml)
+            recursive_merge(merged_prefs, collect_preferences!(merged_prefs, project_dir, pkg_name, cache)...)
         end
     end
-    return nothing
+    return merged_prefs
 end
 
-function get_preferences(uuid::UUID, cache::TOMLCache = TOMLCache();
-                         prefs_key::String = "compile-preferences")
-    project_path = get_preferences_project_path(uuid, cache)
-    if project_path !== nothing
-        preferences = get(parsed_toml(cache, project_path), prefs_key, Dict{String,Any}())
-        if haskey(preferences, string(uuid))
-            return preferences[string(uuid)]
+function get_preferences_hash(uuid::UUID, prefs_list::Vector{String}, cache::TOMLCache = TOMLCache())
+    # Start from the "null" hash
+    h = get_preferences_hash(nothing, prefs_list, cache)
+
+    # Load the preferences
+    prefs = get_preferences(uuid, cache)
+
+    # Walk through each name that's called out as a compile-time preference
+    for name in prefs_list
+        if haskey(prefs, name)
+            h = hash(prefs[name], h)
         end
     end
-    # Fall back to default value of "no preferences".
-    return Dict{String,Any}()
+    return h
 end
-get_preferences_hash(uuid::UUID, cache::TOMLCache = TOMLCache()) = UInt64(hash(get_preferences(uuid, cache)))
-get_preferences_hash(m::Module, cache::TOMLCache = TOMLCache()) = get_preferences_hash(PkgId(m).uuid, cache)
-get_preferences_hash(::Nothing, cache::TOMLCache = TOMLCache()) = UInt64(hash(Dict{String,Any}()))
+get_preferences_hash(m::Module, prefs_list::Vector{String}, cache::TOMLCache = TOMLCache()) = get_preferences_hash(PkgId(m).uuid, prefs_list, cache)
+get_preferences_hash(::Nothing, prefs_list::Vector{String}, cache::TOMLCache = TOMLCache()) = UInt64(0x6e65726566657250)
 
+# This is how we keep track of who is using what preferences at compile-time
+const COMPILETIME_PREFERENCES = Dict{UUID,Set{String}}()
+
+# In `Preferences.jl`, if someone calls `load_preference(@__MODULE__, key)` while we're precompiling,
+# we mark that usage as a usage at compile-time and call this method, so that at the end of `.ji` generation,
+# we can 
+function record_compiletime_preference(uuid::UUID, key::String)
+    if !haskey(COMPILETIME_PREFERENCES, uuid)
+        COMPILETIME_PREFERENCES[uuid] = Set((key,))
+    else
+        push!(COMPILETIME_PREFERENCES[uuid], key)
+    end
+    return nothing
+end
+get_compiletime_preferences(uuid::UUID) = get(COMPILETIME_PREFERENCES, uuid, String[])
+get_compiletime_preferences(m::Module) = get_compiletime_preferences(PkgId(m).uuid)
+get_compiletime_preferences(::Nothing) = String[]
 
 # returns true if it "cachefile.ji" is stale relative to "modpath.jl"
 # otherwise returns the list of dependencies to also check
@@ -1496,7 +1610,7 @@ function stale_cachefile(modpath::String, cachefile::String, cache::TOMLCache)
             @debug "Rejecting cache file $cachefile due to it containing an invalid cache header"
             return true # invalid cache file
         end
-        modules, (includes, requires), required_modules, srctextpos, prefs_hash = parse_cache_header(io)
+        modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash = parse_cache_header(io)
         id = isempty(modules) ? nothing : first(modules).first
         modules = Dict{PkgId, UInt64}(modules)
 
@@ -1572,7 +1686,7 @@ function stale_cachefile(modpath::String, cachefile::String, cache::TOMLCache)
         end
 
         if isa(id, PkgId)
-            curr_prefs_hash = get_preferences_hash(id.uuid, cache)
+            curr_prefs_hash = get_preferences_hash(id.uuid, prefs, cache)
             if prefs_hash != curr_prefs_hash
                 @debug "Rejecting cache file $cachefile because preferences hash does not match 0x$(string(prefs_hash, base=16)) != 0x$(string(curr_prefs_hash, base=16))"
                 return true