From 60d7623f1320470bf2fdb92fd2dca1e9a27b98ce Mon Sep 17 00:00:00 2001 From: Ivan Petkov Date: Mon, 16 Dec 2024 04:46:04 +0000 Subject: [PATCH] vendorGitDeps: perform basic URL decoding of attributes (#764) --- CHANGELOG.md | 3 +++ checks/default.nix | 3 ++- checks/simple-git/Cargo.lock | 38 +++++++++++++++++++++++++++++--- checks/simple-git/Cargo.toml | 5 +++++ checks/simple-git/src/main.rs | 1 + lib/default.nix | 5 ++++- lib/internalPercentDecode.nix | 41 +++++++++++++++++++++++++++++++++++ lib/vendorGitDeps.nix | 9 ++++++-- 8 files changed, 98 insertions(+), 7 deletions(-) create mode 100644 lib/internalPercentDecode.nix diff --git a/CHANGELOG.md b/CHANGELOG.md index e9ff398a..024dcfdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. * `crateNameFromCargoToml` will ignore store contexts when parsing a Cargo.toml file (avoiding errors like `the string ... is not allowed to refer to a store path`). +* `vendorGitDeps` will perform a basic URL-decoding of git dependency entries in + the `Cargo.lock` file since lockfiles now encode special characters starting + at version 4 ### Meta * Dropped support for publishing releases to https://flakestry.dev/ diff --git a/checks/default.nix b/checks/default.nix index d8adf2fb..4b74417d 100644 --- a/checks/default.nix +++ b/checks/default.nix @@ -535,9 +535,10 @@ in simpleGitWithHashes = myLib.buildPackage { src = myLib.cleanCargoSource ./simple-git; outputHashes = { - "git+https://github.com/BurntSushi/byteorder.git#18f32ca3a41c9823138e782752bc439e99ef7ec8" = "sha256-B98zLO/PICVR6QL9dq9Z8LMKnko3kdiYQgj5L0F+gOk="; + "git+https://github.com/BurntSushi/byteorder.git#5a82625fae462e8ba64cec8146b24a372b4d75c6" = "sha256-wDzXLhmBNcqrIrH/k7gcu+k/52XqidbpfrlAEFnY47c="; "git+https://github.com/dtolnay/rustversion.git?rev=2abd4d0e00db08bb91145cb88e5dcbad2f45bbcb#2abd4d0e00db08bb91145cb88e5dcbad2f45bbcb" = "sha256-deS6eoNuWPZ1V3XO9UzR07vLHZjT9arAYL0xEJCoU6E="; "git+https://github.com/dtolnay/unicode-ident.git?rev=a8736e7e62be959d87970d2d137a098ba533d78b#a8736e7e62be959d87970d2d137a098ba533d78b" = "sha256-++OSSdXBaHKeJnC8LOq/ouL+UAJMasDVsBzFClLnjaU="; + "git+https://github.com/ipetkov/crane-test-repo?branch=something/or/other#be0d3c039d260457a55c26e229ad9aa30242c2cf" = "sha256-X6Unf7eirBm6Lat99nROpPd9EUQUL0ru++zDkubj57I="; "git+https://github.com/rust-lang/libc.git?branch=main#a0f5b4b21391252fe38b2df9310dc65e37b07d9f" = "sha256-UwNxrPk6jrmtXeYef+RYYNfpNSlHQllANs/U4bmxlok="; "git+https://github.com/seanmonstar/num_cpus.git?tag=v1.13.1#5f1b03332000b4c4274b5bd35fac516049ff1c6b" = "sha256-mNMxS/WXjNokO9mFXQSwyuIpIp/n94EQ9Ni0Bl40es8="; }; diff --git a/checks/simple-git/Cargo.lock b/checks/simple-git/Cargo.lock index f268884f..545c7add 100644 --- a/checks/simple-git/Cargo.lock +++ b/checks/simple-git/Cargo.lock @@ -1,11 +1,25 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "byteorder" version = "1.5.0" -source = "git+https://github.com/BurntSushi/byteorder.git#18f32ca3a41c9823138e782752bc439e99ef7ec8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "git+https://github.com/BurntSushi/byteorder.git#5a82625fae462e8ba64cec8146b24a372b4d75c6" + +[[package]] +name = "hello" +version = "0.2.0" +source = "git+https://github.com/ipetkov/crane-test-repo?branch=something%2For%2Fother#be0d3c039d260457a55c26e229ad9aa30242c2cf" +dependencies = [ + "byteorder 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "hermit-abi" @@ -36,6 +50,15 @@ dependencies = [ "libc 0.2.153", ] +[[package]] +name = "print" +version = "0.2.0" +source = "git+https://github.com/ipetkov/crane-test-repo?branch=something%2For%2Fother#be0d3c039d260457a55c26e229ad9aa30242c2cf" +dependencies = [ + "hello", + "world", +] + [[package]] name = "rustversion" version = "1.0.6" @@ -45,12 +68,21 @@ source = "git+https://github.com/dtolnay/rustversion.git?rev=2abd4d0e00db08bb911 name = "simple-git" version = "0.1.0" dependencies = [ - "byteorder", + "byteorder 1.5.0 (git+https://github.com/BurntSushi/byteorder.git)", "libc 0.2.151", "num_cpus", + "print", "rustversion", ] +[[package]] +name = "world" +version = "0.2.0" +source = "git+https://github.com/ipetkov/crane-test-repo?branch=something%2For%2Fother#be0d3c039d260457a55c26e229ad9aa30242c2cf" +dependencies = [ + "byteorder 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[patch.unused]] name = "unicode-ident" version = "1.0.12" diff --git a/checks/simple-git/Cargo.toml b/checks/simple-git/Cargo.toml index 77511ffa..590b982d 100644 --- a/checks/simple-git/Cargo.toml +++ b/checks/simple-git/Cargo.toml @@ -7,6 +7,11 @@ edition = "2021" version = "*" git = "https://github.com/BurntSushi/byteorder.git" +[dependencies.crane-test-repo] +package = "print" +git = "https://github.com/ipetkov/crane-test-repo" +branch = "something/or/other" + [dependencies.libc] version = "*" git = "https://github.com/rust-lang/libc.git" diff --git a/checks/simple-git/src/main.rs b/checks/simple-git/src/main.rs index 46825dc3..12919d41 100644 --- a/checks/simple-git/src/main.rs +++ b/checks/simple-git/src/main.rs @@ -7,4 +7,5 @@ fn main() { println!("{:?}", std::any::TypeId::of::()); println!("{:?}", std::any::TypeId::of::()); println!("{}: {}", CHANNEL, num_cpus::get()); + crane_test_repo::print(); } diff --git a/lib/default.nix b/lib/default.nix index e8bd71bd..26d148ee 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -45,6 +45,7 @@ let internalCrateNameForCleanSource = callPackage ./internalCrateNameForCleanSource.nix { inherit internalCrateNameFromCargoToml; }; + internalPercentDecode = callPackage ./internalPercentDecode.nix { }; in { appendCrateRegistries = input: self.overrideScope (_final: prev: { @@ -134,7 +135,9 @@ let vendorCargoDeps = callPackage ./vendorCargoDeps.nix { }; vendorMultipleCargoDeps = callPackage ./vendorMultipleCargoDeps.nix { }; vendorCargoRegistries = callPackage ./vendorCargoRegistries.nix { }; - vendorGitDeps = callPackage ./vendorGitDeps.nix { }; + vendorGitDeps = callPackage ./vendorGitDeps.nix { + inherit internalPercentDecode; + }; writeTOML = callPackage ./writeTOML.nix { }; }; diff --git a/lib/internalPercentDecode.nix b/lib/internalPercentDecode.nix new file mode 100644 index 00000000..05e3679f --- /dev/null +++ b/lib/internalPercentDecode.nix @@ -0,0 +1,41 @@ +{}: + +let + codes = { + "%09" = "\t"; + "%0A" = "\n"; + "%0D" = "\r"; + "%20" = " "; + "%21" = "!"; + "%22" = "\""; + "%23" = "#"; + "%24" = "$"; + "%25" = "%"; + "%26" = "&"; + "%27" = "'"; + "%28" = "("; + "%29" = ")"; + "%2A" = "*"; + "%2B" = "+"; + "%2C" = ","; + "%2F" = "/"; + "%3A" = ":"; + "%3B" = ";"; + "%3C" = "<"; + "%3D" = "="; + "%3E" = ">"; + "%3F" = "?"; + "%40" = "@"; + "%5B" = "["; + "%5C" = "\\"; + "%5D" = "]"; + "%5E" = "^"; + "%60" = "`"; + "%7B" = "{"; + "%7C" = "|"; + "%7D" = "}"; + }; +in +builtins.replaceStrings + (builtins.attrNames codes) + (builtins.attrValues codes) diff --git a/lib/vendorGitDeps.nix b/lib/vendorGitDeps.nix index c3f37751..3f26196d 100644 --- a/lib/vendorGitDeps.nix +++ b/lib/vendorGitDeps.nix @@ -1,4 +1,5 @@ { downloadCargoPackageFromGit +, internalPercentDecode , lib , pkgsBuildBuild }: @@ -77,9 +78,13 @@ let }; # Local crates will show up in the lock file with no checksum/source - lockedPackagesFromGit = filter + # NB: cargo started url encoding source urls starting with version 4 + # but we need to undo that as package fetching and cargo configs expect + # the unencoded URLs. + lockedPackagesFromGit = map (lib.mapAttrs (_: internalPercentDecode)) (filter (p: hasPrefix "git" (p.source or "")) - lockPackages; + lockPackages + ); lockedGitGroups = groupBy (p: p.id) (map (p: (parseGitUrl p) // { package = p; }) lockedPackagesFromGit