From e7d37acfd1ebac970bf558dcbca51690b5cc487e Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Sun, 9 Feb 2025 08:38:42 -0800 Subject: [PATCH] Fixed crate_universe when using Rust >= 1.85.0 --- .../crates/BUILD.crates-index-3.5.0.bazel | 1 + .../BUILD.rustc-stable-hash-0.1.1.bazel | 83 +++++++++++++++++++ crate_universe/3rdparty/crates/defs.bzl | 20 +++-- crate_universe/Cargo.lock | 10 ++- crate_universe/Cargo.toml | 5 +- crate_universe/src/metadata.rs | 10 +++ crate_universe/src/metadata/cargo_bin.rs | 12 +++ crate_universe/src/splicing.rs | 34 +++++--- .../src/splicing/crate_index_lookup.rs | 12 +-- examples/crate_universe/MODULE.bazel | 2 +- rust/private/common.bzl | 2 +- 11 files changed, 165 insertions(+), 26 deletions(-) create mode 100644 crate_universe/3rdparty/crates/BUILD.rustc-stable-hash-0.1.1.bazel diff --git a/crate_universe/3rdparty/crates/BUILD.crates-index-3.5.0.bazel b/crate_universe/3rdparty/crates/BUILD.crates-index-3.5.0.bazel index db97961d06..69f34d373c 100644 --- a/crate_universe/3rdparty/crates/BUILD.crates-index-3.5.0.bazel +++ b/crate_universe/3rdparty/crates/BUILD.crates-index-3.5.0.bazel @@ -92,6 +92,7 @@ rust_library( "@cui__home-0.5.5//:home", "@cui__memchr-2.6.4//:memchr", "@cui__rustc-hash-2.0.0//:rustc_hash", + "@cui__rustc-stable-hash-0.1.1//:rustc_stable_hash", "@cui__semver-1.0.24//:semver", "@cui__serde-1.0.217//:serde", "@cui__serde_json-1.0.135//:serde_json", diff --git a/crate_universe/3rdparty/crates/BUILD.rustc-stable-hash-0.1.1.bazel b/crate_universe/3rdparty/crates/BUILD.rustc-stable-hash-0.1.1.bazel new file mode 100644 index 0000000000..1a247fc5a7 --- /dev/null +++ b/crate_universe/3rdparty/crates/BUILD.rustc-stable-hash-0.1.1.bazel @@ -0,0 +1,83 @@ +############################################################################### +# @generated +# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To +# regenerate this file, run the following: +# +# bazel run @@//crate_universe/3rdparty:crates_vendor +############################################################################### + +load("@rules_rust//rust:defs.bzl", "rust_library") + +package(default_visibility = ["//visibility:public"]) + +rust_library( + name = "rustc_stable_hash", + srcs = glob( + include = ["**/*.rs"], + allow_empty = True, + ), + compile_data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + crate_root = "src/lib.rs", + edition = "2021", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-bazel", + "crate-name=rustc-stable-hash", + "manual", + "noclippy", + "norustfmt", + ], + target_compatible_with = select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": [], + "@rules_rust//rust/platform:aarch64-apple-ios": [], + "@rules_rust//rust/platform:aarch64-apple-ios-sim": [], + "@rules_rust//rust/platform:aarch64-linux-android": [], + "@rules_rust//rust/platform:aarch64-pc-windows-msvc": [], + "@rules_rust//rust/platform:aarch64-unknown-fuchsia": [], + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [], + "@rules_rust//rust/platform:aarch64-unknown-uefi": [], + "@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:armv7-linux-androideabi": [], + "@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:i686-apple-darwin": [], + "@rules_rust//rust/platform:i686-linux-android": [], + "@rules_rust//rust/platform:i686-pc-windows-msvc": [], + "@rules_rust//rust/platform:i686-unknown-freebsd": [], + "@rules_rust//rust/platform:i686-unknown-linux-gnu": [], + "@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [], + "@rules_rust//rust/platform:s390x-unknown-linux-gnu": [], + "@rules_rust//rust/platform:thumbv7em-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabi": [], + "@rules_rust//rust/platform:wasm32-unknown-unknown": [], + "@rules_rust//rust/platform:wasm32-wasip1": [], + "@rules_rust//rust/platform:x86_64-apple-darwin": [], + "@rules_rust//rust/platform:x86_64-apple-ios": [], + "@rules_rust//rust/platform:x86_64-linux-android": [], + "@rules_rust//rust/platform:x86_64-pc-windows-msvc": [], + "@rules_rust//rust/platform:x86_64-unknown-freebsd": [], + "@rules_rust//rust/platform:x86_64-unknown-fuchsia": [], + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-none": [], + "@rules_rust//rust/platform:x86_64-unknown-uefi": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + version = "0.1.1", +) diff --git a/crate_universe/3rdparty/crates/defs.bzl b/crate_universe/3rdparty/crates/defs.bzl index b3e5e4bb7b..c8f51c04de 100644 --- a/crate_universe/3rdparty/crates/defs.bzl +++ b/crate_universe/3rdparty/crates/defs.bzl @@ -16,6 +16,7 @@ """ load("@bazel_skylib//lib:selects.bzl", "selects") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") @@ -864,12 +865,11 @@ def crate_repositories(): ) maybe( - http_archive, + new_git_repository, name = "cui__crates-index-3.5.0", - sha256 = "13058139052295533e5f7b0ed22ecf3eb7d7a5c2cd5657d6a7c8b4d8d8e093e6", - type = "tar.gz", - urls = ["https://static.crates.io/crates/crates-index/3.5.0/download"], - strip_prefix = "crates-index-3.5.0", + commit = "a77d2d0981aaef368e10406912febbbdad4b83d7", + init_submodules = True, + remote = "https://github.com/UebelAndre/rust-crates-index.git", build_file = Label("//crate_universe/3rdparty/crates:BUILD.crates-index-3.5.0.bazel"), ) @@ -2183,6 +2183,16 @@ def crate_repositories(): build_file = Label("//crate_universe/3rdparty/crates:BUILD.rustc-hash-2.0.0.bazel"), ) + maybe( + http_archive, + name = "cui__rustc-stable-hash-0.1.1", + sha256 = "2febf9acc5ee5e99d1ad0afcdbccc02d87aa3f857a1f01f825b80eacf8edfcd1", + type = "tar.gz", + urls = ["https://static.crates.io/crates/rustc-stable-hash/0.1.1/download"], + strip_prefix = "rustc-stable-hash-0.1.1", + build_file = Label("//crate_universe/3rdparty/crates:BUILD.rustc-stable-hash-0.1.1.bazel"), + ) + maybe( http_archive, name = "cui__rustix-0.38.41", diff --git a/crate_universe/Cargo.lock b/crate_universe/Cargo.lock index 3d91ff7dc5..3a15b2af07 100644 --- a/crate_universe/Cargo.lock +++ b/crate_universe/Cargo.lock @@ -326,14 +326,14 @@ dependencies = [ [[package]] name = "crates-index" version = "3.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13058139052295533e5f7b0ed22ecf3eb7d7a5c2cd5657d6a7c8b4d8d8e093e6" +source = "git+https://github.com/UebelAndre/rust-crates-index.git?rev=a77d2d0981aaef368e10406912febbbdad4b83d7#a77d2d0981aaef368e10406912febbbdad4b83d7" dependencies = [ "gix", "hex", "home", "memchr", "rustc-hash", + "rustc-stable-hash", "semver", "serde", "serde_derive", @@ -1850,6 +1850,12 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" +[[package]] +name = "rustc-stable-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2febf9acc5ee5e99d1ad0afcdbccc02d87aa3f857a1f01f825b80eacf8edfcd1" + [[package]] name = "rustix" version = "0.38.41" diff --git a/crate_universe/Cargo.toml b/crate_universe/Cargo.toml index f41e78836a..64c85aa9b3 100644 --- a/crate_universe/Cargo.toml +++ b/crate_universe/Cargo.toml @@ -69,7 +69,10 @@ cargo-lock = "10.0.1" cargo-platform = "0.1.9" cfg-expr = "0.17.2" clap = { version = "4.5.26", features = ["derive", "env"] } -crates-index = { version = "3.5.0", default-features = false, features = [ +# crates-index = { version = "3.5.0", default-features = false, features = [ +# "git", +# ] } +crates-index = { git = "https://github.com/UebelAndre/rust-crates-index.git", rev = "a77d2d0981aaef368e10406912febbbdad4b83d7", default-features = false, features = [ "git", ] } hex = "0.4.3" diff --git a/crate_universe/src/metadata.rs b/crate_universe/src/metadata.rs index 729c88228c..dce766f667 100644 --- a/crate_universe/src/metadata.rs +++ b/crate_universe/src/metadata.rs @@ -220,6 +220,7 @@ impl LockGenerator { // Ensure the Cargo cache is up to date to simulate the behavior // of having just generated a new one + tracing::debug!("Fetching crates for {}", manifest_path); let output = self .cargo_bin .command()? @@ -230,6 +231,7 @@ impl LockGenerator { .arg("fetch") .arg("--manifest-path") .arg(manifest_path.as_std_path()) + .arg("--verbose") .output() .context(format!( "Error running cargo to fetch crates '{}'", @@ -244,6 +246,14 @@ impl LockGenerator { output.status )) } + tracing::trace!( + "Cargo fetch stderr:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + tracing::trace!( + "Cargo fetch stdout:\n{}", + String::from_utf8_lossy(&output.stdout) + ); } else { debug!("Generating new lockfile"); // Simply invoke `cargo generate-lockfile` diff --git a/crate_universe/src/metadata/cargo_bin.rs b/crate_universe/src/metadata/cargo_bin.rs index 7d00dc72e2..c0b0cb17c5 100644 --- a/crate_universe/src/metadata/cargo_bin.rs +++ b/crate_universe/src/metadata/cargo_bin.rs @@ -117,6 +117,18 @@ impl Cargo { bail!("Couldn't parse cargo version"); } + /// Determine if Cargo is on a version which uses new hashing behavior + /// introduced in Rust 1.86.0. For details see + pub(crate) fn uses_stable_registry_hash(&self) -> Result { + let full_version = self.full_version()?; + let version_str = full_version.split(' ').nth(1); + if let Some(version_str) = version_str { + let version = Version::parse(version_str).context("Failed to parse cargo version")?; + return Ok(version.major >= 1 && version.minor >= 85); + } + bail!("Couldn't parse cargo version"); + } + fn env(&self) -> Result> { let mut map = BTreeMap::new(); diff --git a/crate_universe/src/splicing.rs b/crate_universe/src/splicing.rs index 92f7d6e05a..4417386bea 100644 --- a/crate_universe/src/splicing.rs +++ b/crate_universe/src/splicing.rs @@ -307,6 +307,12 @@ impl WorkspaceMetadata { } }; + let crate_index_hash_kind = if cargo.uses_stable_registry_hash()? { + crates_index::HashKind::Stable + } else { + crates_index::HashKind::Legacy + }; + // Load each index for easy access let crate_indexes = index_urls .into_iter() @@ -320,20 +326,27 @@ impl WorkspaceMetadata { let index = if cargo.use_sparse_registries_for_crates_io()? && index_url == utils::CRATES_IO_INDEX_URL { - CrateIndexLookup::Http(crates_index::SparseIndex::from_url( + CrateIndexLookup::Http(crates_index::SparseIndex::from_url_with_hash_kind( "sparse+https://index.crates.io/", + &crate_index_hash_kind, )?) } else if index_url.starts_with("sparse+") { - CrateIndexLookup::Http(crates_index::SparseIndex::from_url(index_url)?) + CrateIndexLookup::Http(crates_index::SparseIndex::from_url_with_hash_kind( + index_url, + &crate_index_hash_kind, + )?) } else { match source_kind { SourceKind::Registry => { let index = { // Load the index for the current url - let index = crates_index::GitIndex::from_url(index_url) - .with_context(|| { - format!("Failed to load index for url: {index_url}") - })?; + let index = crates_index::GitIndex::from_url_with_hash_kind( + index_url, + &crate_index_hash_kind, + ) + .with_context(|| { + format!("Failed to load index for url: {index_url}") + })?; // Ensure each index has a valid index config index.index_config().with_context(|| { @@ -344,11 +357,12 @@ impl WorkspaceMetadata { }; CrateIndexLookup::Git(index) } - SourceKind::SparseRegistry => { - CrateIndexLookup::Http(crates_index::SparseIndex::from_url( + SourceKind::SparseRegistry => CrateIndexLookup::Http( + crates_index::SparseIndex::from_url_with_hash_kind( format!("sparse+{}", index_url).as_str(), - )?) - } + &crate_index_hash_kind, + )?, + ), unknown => { return Err(anyhow!( "'{:?}' crate index type is not supported (caused by '{}')", diff --git a/crate_universe/src/splicing/crate_index_lookup.rs b/crate_universe/src/splicing/crate_index_lookup.rs index 05bd34f79f..2c26e0558d 100644 --- a/crate_universe/src/splicing/crate_index_lookup.rs +++ b/crate_universe/src/splicing/crate_index_lookup.rs @@ -16,11 +16,11 @@ impl CrateIndexLookup { let crate_ = match self { // The crates we care about should all be in the cache already, // because `cargo metadata` ran which should have fetched them. - Self::Http(index) => Some( - index - .crate_from_cache(pkg.name.as_str()) - .with_context(|| format!("Failed to get crate from cache for {pkg:?}"))?, - ), + Self::Http(index) => { + Some(index.crate_from_cache(pkg.name.as_str()).with_context(|| { + format!("Failed to get crate from cache: {:?}\n{:?}", index, pkg) + })?) + } Self::Git(index) => index.crate_(pkg.name.as_str()), }; let source_info = crate_.and_then(|crate_idx| { @@ -97,7 +97,7 @@ mod test { ); } { - let _e = EnvVarResetter::set("CARGO_HOME", + let _e = EnvVarResetter::set("CARGO_HOME", runfiles::rlocation!(runfiles, "rules_rust/crate_universe/test_data/crate_indexes/rewritten_lazy_static/cargo_home").unwrap()); let index = CrateIndexLookup::Http( diff --git a/examples/crate_universe/MODULE.bazel b/examples/crate_universe/MODULE.bazel index 76adfda506..269ca10430 100644 --- a/examples/crate_universe/MODULE.bazel +++ b/examples/crate_universe/MODULE.bazel @@ -72,7 +72,7 @@ alias_rule_global_alias_annotation_none.spec( repositories = ["alias_rule_global_alias_annotation_none"], version = "0.1.0", ) -alias_rule_global_alias_annotation_none.from_cargo( +alias_rule_global_alias_annotation_none.from_specs( name = "alias_rule_global_alias_annotation_none", cargo_lockfile = "//alias_rule:Cargo.lock", lockfile = "//alias_rule:cargo-bazel-lock_global_alias_annotation_none.json", diff --git a/rust/private/common.bzl b/rust/private/common.bzl index a34d6b145d..c0e56f882d 100644 --- a/rust/private/common.bzl +++ b/rust/private/common.bzl @@ -33,7 +33,7 @@ load(":providers.bzl", "CrateGroupInfo", "CrateInfo", "DepInfo", "DepVariantInfo # you remove it or change its format, you will also need to update that code. DEFAULT_RUST_VERSION = "1.84.1" -DEFAULT_NIGHTLY_ISO_DATE = "2024-11-28" +DEFAULT_NIGHTLY_ISO_DATE = "2025-01-30" def _create_crate_info(**kwargs): """A constructor for a `CrateInfo` provider