ask the package server for zstd compressed data

Together with JuliaPackaging/PkgServer.jl#220 this allows Pkg to download registries, packages and artifacts as zst archives which based on benchmarks decompress significantly faster and also tend to be smaller.

Having the registry stored as a zst archive is not a backwards compatibility issue because even on older Pkgs it can decompress these with 7z

Author: KristofferC

Project.toml

@@ -23,6 +23,7 @@ SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+Zstd_jll = "3161d3a3-bdf6-5164-811a-617609db77b4"
 p7zip_jll = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
 
 [weakdeps]
@@ -41,11 +42,12 @@ Libdl = "1.11"
 Logging = "1.11"
 Markdown = "1.11"
 Printf = "1.11"
-Random = "1.11"
 REPL = "1.11"
+Random = "1.11"
 SHA = "0.7, 1"
 TOML = "1"
 Tar = "1.10"
 UUIDs = "1.11"
+Zstd_jll = "1.5.7"
 julia = "1.12"
 p7zip_jll = "17.5"

docs/src/protocol.md

@@ -136,6 +136,18 @@ The client can make GET or HEAD requests to the following resources:
 
 Only the `/registries` changes - all other resources can be cached forever and the server will indicate this with the appropriate HTTP headers.
 
+### Compression Negotiation
+
+The Pkg protocol supports multiple compression formats.
+
+- **Zstd compression** (current): Modern clients send `Accept-Encoding: zstd, gzip` to request Zstandard-compressed resources with gzip as a fallback.
+- **Gzip compression** (legacy): Older clients that only support gzip send `Accept-Encoding: gzip` or omit the header entirely.
+
+Clients verify the actual compression format by reading file magic bytes after download to determine the correct file extension for caching:
+
+- **Zstd format**: Magic bytes `0x28 0xB5 0x2F 0xFD` (4 bytes)
+- **Gzip format**: Magic bytes `0x1F 0x8B` (2 bytes)
+
 ### Reference Implementation
 
 A reference implementation of the Pkg Server protocol is available at [PkgServer.jl](https://github.com/JuliaPackaging/PkgServer.jl).

src/PlatformEngines.jl

@@ -4,15 +4,17 @@
 
 module PlatformEngines
 
-using SHA, Downloads, Tar
+using SHA, Downloads, Tar, Dates, Printf
 import ...Pkg: Pkg, TOML, pkg_server, depots1, can_fancyprint, stderr_f, atomic_toml_write
 using ..MiniProgressBars
-using Base.BinaryPlatforms, p7zip_jll
+using Base.BinaryPlatforms, p7zip_jll, Zstd_jll
 
-export verify, unpack, package, download_verify_unpack
+export verify, unpack, package, download_verify_unpack, get_extract_cmd, detect_archive_format
 
 const EXE7Z_LOCK = ReentrantLock()
 const EXE7Z = Ref{String}()
+const EXEZSTD_LOCK = ReentrantLock()
+const EXEZSTD = Ref{String}()
 
 function exe7z()
     # If the JLL is available, use the wrapper function defined in there
@@ -28,6 +30,20 @@ function exe7z()
     end
 end
 
+function exezstd()
+    # If the JLL is available, use the wrapper function defined in there
+    if Zstd_jll.is_available()
+        return Zstd_jll.zstd()
+    end
+
+    return lock(EXEZSTD_LOCK) do
+        if !isassigned(EXEZSTD)
+            EXEZSTD[] = findzstd()
+        end
+        return Cmd([EXEZSTD[]])
+    end
+end
+
 function find7z()
     name = "7z"
     Sys.iswindows() && (name = "$name.exe")
@@ -40,6 +56,18 @@ function find7z()
     error("7z binary not found")
 end
 
+function findzstd()
+    name = "zstd"
+    Sys.iswindows() && (name = "$name.exe")
+    for dir in (joinpath("..", "libexec"), ".")
+        path = normpath(Sys.BINDIR::String, dir, name)
+        isfile(path) && return path
+    end
+    path = Sys.which(name)
+    path !== nothing && return path
+    error("zstd binary not found")
+end
+
 is_secure_url(url::AbstractString) =
     occursin(r"^(https://|\w+://(127\.0\.0\.1|localhost)(:\d+)?($|/))"i, url)
 
@@ -232,6 +260,13 @@ function get_metadata_headers(url::AbstractString)
     end
     push!(headers, "Julia-CI-Variables" => join(ci_info, ';'))
     push!(headers, "Julia-Interactive" => string(isinteractive()))
+
+    # Add Accept-Encoding header only for compressed archive resources
+    # (registries, packages, artifacts - not for metadata endpoints like /registries or /meta)
+    if occursin(r"/(registry|package|artifact)/", url)
+        push!(headers, "Accept-Encoding" => "zstd, gzip")
+    end
+
     for (key, val) in ENV
         m = match(r"^JULIA_PKG_SERVER_([A-Z0-9_]+)$"i, key)
         m === nothing && continue
@@ -403,22 +438,76 @@ function copy_symlinks()
         lowercase(var) in ("false", "f", "no", "n", "0") ? false : nothing
 end
 
+"""
+    detect_archive_format(tarball_path::AbstractString)
+
+Detect compression format by reading file magic bytes.
+Returns "zstd" or "gzip".
+
+Note: This is primarily used for determining the correct file extension after download.
+For decompression, we always use zstd as it can efficiently handle both formats.
+"""
+function detect_archive_format(tarball_path::AbstractString)
+    file_size = filesize(tarball_path)
+
+    if file_size == 0
+        error("cannot detect compression format: $tarball_path is empty")
+    end
+
+    magic = open(tarball_path, "r") do io
+        read(io, min(4, file_size))
+    end
+
+    # Zstd magic number: 0x28 0xB5 0x2F 0xFD
+    if length(magic) >= 4 && magic[1:4] == [0x28, 0xB5, 0x2F, 0xFD]
+        return "zstd"
+    end
+    # Gzip magic number: 0x1F 0x8B
+    if length(magic) >= 2 && magic[1:2] == [0x1F, 0x8B]
+        return "gzip"
+    end
+
+    # Show hex dump of magic bytes for debugging
+    hex_dump = length(magic) > 0 ? join([@sprintf("0x%02X", b) for b in magic], " ") : "none"
+    error("unknown compression format for $tarball_path (magic bytes: $hex_dump, expected zstd [0x28 0xB5 0x2F 0xFD] or gzip [0x1F 0x8B])")
+end
+
+"""
+    get_extract_cmd(tarball_path::AbstractString)
+
+Get the decompression command for a tarball.
+Uses zstd for all decompression as it handles both zstd and gzip formats efficiently.
+"""
+function get_extract_cmd(tarball_path::AbstractString)
+    # zstd can decompress both zstd and gzip formats, and is ~3x faster than 7z for gzip
+    return `$(exezstd()) -d -c $tarball_path`
+end
+
 function unpack(
         tarball_path::AbstractString,
         dest::AbstractString;
         verbose::Bool = false,
     )
-    return Tar.extract(`$(exe7z()) x $tarball_path -so`, dest, copy_symlinks = copy_symlinks())
+    return Tar.extract(get_extract_cmd(tarball_path), dest, copy_symlinks = copy_symlinks())
 end
 
 """
     package(src_dir::AbstractString, tarball_path::AbstractString)
 
 Compress `src_dir` into a tarball located at `tarball_path`.
+Supports both gzip and zstd compression based on file extension.
 """
 function package(src_dir::AbstractString, tarball_path::AbstractString; io = stderr_f())
     rm(tarball_path, force = true)
-    cmd = `$(exe7z()) a -si -tgzip -mx9 $tarball_path`
+    # Choose compression based on file extension (case-insensitive)
+    tarball_lower = lowercase(tarball_path)
+    if endswith(tarball_lower, ".zst") || endswith(tarball_lower, ".tar.zst")
+        # Use zstd compression (level 19 for good compression)
+        cmd = `$(exezstd()) -19 -c -T -o $tarball_path`
+    else
+        # Use gzip compression (default)
+        cmd = `$(exe7z()) a -si -tgzip -mx9 $tarball_path`
+    end
     return open(pipeline(cmd, stdout = devnull, stderr = io), write = true) do io
         Tar.create(src_dir, io)
     end
@@ -496,8 +585,8 @@ function download_verify_unpack(
         end
 
         # If extension of url contains a recognized extension, use it, otherwise use ".gz"
-        ext = url_ext(url)
-        if !(ext in ["tar", "gz", "tgz", "bz2", "xz"])
+        ext = lowercase(url_ext(url))
+        if !(ext in ["tar", "gz", "tgz", "bz2", "xz", "zst"])
             ext = "gz"
         end
 
@@ -538,7 +627,7 @@ function download_verify_unpack(
             @info("Unpacking $(tarball_path) into $(dest)...")
         end
         isnothing(progress) || progress(10000, 10000; status = "unpacking")
-        open(`$(exe7z()) x $tarball_path -so`) do io
+        open(get_extract_cmd(tarball_path)) do io
             Tar.extract(io, dest, copy_symlinks = copy_symlinks())
         end
     finally
@@ -690,7 +779,7 @@ function verify_archive_tree_hash(tar_gz::AbstractString, expected_hash::Base.SH
     # tarball, tree hash verification requires that the file can i) be
     # decompressed and ii) is a proper archive.
     calc_hash = try
-        Base.SHA1(open(Tar.tree_hash, `$(exe7z()) x $tar_gz -so`))
+        Base.SHA1(open(Tar.tree_hash, get_extract_cmd(tar_gz)))
     catch err
         @warn "unable to decompress and read archive" exception = err
         return false

src/Registry/Registry.jl

@@ -41,7 +41,7 @@ module Registry
 import ..Pkg
 using ..Pkg: depots, depots1, printpkgstyle, stderr_f, isdir_nothrow, pathrepr, pkg_server,
     GitTools, atomic_toml_write, create_cachedir_tag
-using ..Pkg.PlatformEngines: download_verify_unpack, download, download_verify, exe7z, verify_archive_tree_hash
+using ..Pkg.PlatformEngines: download_verify_unpack, download, download_verify, verify_archive_tree_hash, get_extract_cmd, detect_archive_format
 using UUIDs, LibGit2, TOML, Dates
 import FileWatching
 
@@ -240,6 +240,18 @@ function check_registry_state(reg)
     return nothing
 end
 
+# Detect compression format by reading file magic bytes
+# Returns file extension (.tar.zst or .tar.gz)
+# This is defensive: we request zstd, but the server might not support it yet
+function detect_compression_format(filepath::AbstractString)::String
+    format = detect_archive_format(filepath)
+    if format == "zstd"
+        return ".tar.zst"
+    else
+        return ".tar.gz"
+    end
+end
+
 function download_registries(io::IO, regs::Vector{RegistrySpec}, depots::Union{String, Vector{String}} = depots())
     # Use the first depot as the target
     target_depot = depots1(depots)
@@ -282,8 +294,10 @@ function download_registries(io::IO, regs::Vector{RegistrySpec}, depots::Union{S
                     reg_unc = uncompress_registry(tmp)
                     reg.name = TOML.parse(reg_unc["Registry.toml"])["name"]::String
                 end
-                mv(tmp, joinpath(regdir, reg.name * ".tar.gz"); force = true)
-                reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(_hash), "path" => reg.name * ".tar.gz")
+                # Detect what we actually got from the server (defensive against servers that don't support zstd yet)
+                ext = detect_compression_format(tmp)
+                mv(tmp, joinpath(regdir, reg.name * ext); force = true)
+                reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(_hash), "path" => reg.name * ext)
                 atomic_toml_write(joinpath(regdir, reg.name * ".toml"), reg_info)
                 registry_update_log[string(reg.uuid)] = now()
                 printpkgstyle(io, :Added, "`$(reg.name)` registry to $(Base.contractuser(regdir))")
@@ -546,8 +560,10 @@ function update(regs::Vector{RegistrySpec}; io::IO = stderr_f(), force::Bool = t
                                             Base.rm(reg.path; recursive = true, force = true)
                                         end
                                         registry_path = dirname(reg.path)
-                                        mv(tmp, joinpath(registry_path, reg.name * ".tar.gz"); force = true)
-                                        reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(hash), "path" => reg.name * ".tar.gz")
+                                        # Detect what we actually got from the server (defensive against servers that don't support zstd yet)
+                                        ext = detect_compression_format(tmp)
+                                        mv(tmp, joinpath(registry_path, reg.name * ext); force = true)
+                                        reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(hash), "path" => reg.name * ext)
                                         atomic_toml_write(joinpath(registry_path, reg.name * ".toml"), reg_info)
                                         registry_update_log[string(reg.uuid)] = now()
                                         @label done_tarball_read

src/Registry/registry_instance.jl

@@ -279,7 +279,7 @@ function uncompress_registry(tar_gz::AbstractString)
     data = Dict{String, String}()
     buf = Vector{UInt8}(undef, Tar.DEFAULT_BUFFER_SIZE)
     io = IOBuffer()
-    open(`$(exe7z()) x $tar_gz -so`) do tar
+    open(get_extract_cmd(tar_gz)) do tar
         Tar.read_tarball(x -> true, tar; buf = buf) do hdr, _
             if hdr.type == :file
                 Tar.read_data(tar, io; size = hdr.size, buf = buf)

src/precompile.jl

@@ -82,15 +82,13 @@ function _run_precompilation_script_setup()
             repo = "$(escape_string(tmp))/TestPkg.jl"
             """,
         )
-        Tar.create("registries/Registry", "registries/Registry.tar")
-        cmd = `$(Pkg.PlatformEngines.exe7z()) a "registries/Registry.tar.gz" -tgzip "registries/Registry.tar"`
-        run(pipeline(cmd, stdout = stdout_f(), stderr = stderr_f()))
+        Pkg.PlatformEngines.package("registries/Registry", "registries/Registry.tar.zst")
         write(
             "registries/Registry.toml",
             """
             git-tree-sha1 = "11b5fad51c4f98cfe0c145ceab0b8fb63fed6f81"
             uuid = "37c07fec-e54c-4851-934c-2e3885e4053e"
-            path = "Registry.tar.gz"
+            path = "Registry.tar.zst"
             """,
         )
         Base.rm("registries/Registry"; recursive = true)