Deleted "extra workspace member" functionality from crate_universe (#1406)

* Deleted "extra workspace member" functionality from crate_universe

* Regenerate documentation

Author: UebelAndre

.bazelci/presubmit.yml

@@ -333,15 +333,13 @@ tasks:
       # failing. This is likely an issue with the windows workers in CI.
       - "-//vendor_local_pkgs/..."
       - "-//vendor_remote_pkgs/..."
-      - "-//extra_workspace_members/..."
       - "-//no_cargo_manifests/..."
     test_targets:
       - "//..."
       # TODO: Temporarily disable this windows build as it recently started
       # failing. This is likely an issue with the windows workers in CI.
       - "-//vendor_local_pkgs/..."
       - "-//vendor_remote_pkgs/..."
-      - "-//extra_workspace_members/..."
       - "-//no_cargo_manifests/..."
 buildifier:
   version: latest

crate_universe/private/crates_repository.bzl

@@ -175,18 +175,6 @@ CARGO_BAZEL_REPIN=1 bazel sync --only=crate_index
         "cargo_config": attr.label(
             doc = "A [Cargo configuration](https://doc.rust-lang.org/cargo/reference/config.html) file",
         ),
-        "extra_workspace_member_url_template": attr.string(
-            doc = "The registry url to use when fetching extra workspace members",
-            default = "https://crates.io/api/v1/crates/{name}/{version}/download",
-        ),
-        "extra_workspace_members": attr.string_dict(
-            doc = (
-                "Additional crates to download and include as a workspace member. This is unfortunately required in " +
-                "order to add information about \"binary-only\" crates so that a `rust_binary` may be generated for " +
-                "it. [rust-lang/cargo#9096](https://github.com/rust-lang/cargo/issues/9096) tracks an RFC which may " +
-                "solve for this."
-            ),
-        ),
         "generate_build_scripts": attr.bool(
             doc = (
                 "Whether or not to generate " +

crate_universe/private/crates_vendor.bzl

@@ -120,20 +120,6 @@ def _write_splicing_manifest(ctx):
     runfiles = [manifest] + ctx.files.manifests + ([ctx.file.cargo_config] if ctx.attr.cargo_config else [])
     return args, runfiles
 
-def _write_extra_manifests_manifest(ctx):
-    manifest = _write_data_file(
-        ctx = ctx,
-        name = "cargo-bazel-extra-manifests-manifest.json",
-        data = json.encode(struct(
-            # TODO: This is for extra workspace members
-            manifests = [],
-        )),
-    )
-    is_windows = _is_windows(ctx)
-    args = ["--extra-manifests-manifest", _runfiles_path(manifest.short_path, is_windows)]
-    runfiles = [manifest]
-    return args, runfiles
-
 def _write_config_file(ctx):
     rendering_config = dict(json.decode(render_config()))
 
@@ -219,11 +205,6 @@ def _crates_vendor_impl(ctx):
     args.extend(splicing_manifest_args)
     cargo_bazel_runfiles.extend(splicing_manifest_runfiles)
 
-    # Generate extra-manifests manifest
-    extra_manifests_manifest_args, extra_manifests_manifest_runfiles = _write_extra_manifests_manifest(ctx)
-    args.extend(extra_manifests_manifest_args)
-    cargo_bazel_runfiles.extend(extra_manifests_manifest_runfiles)
-
     # Optionally include buildifier
     if ctx.attr.buildifier:
         args.extend(["--buildifier", _runfiles_path(ctx.executable.buildifier.short_path, is_windows)])

crate_universe/private/splicing_utils.bzl

@@ -21,63 +21,6 @@ def splicing_config(resolver_version = "1"):
         resolver_version = resolver_version,
     ))
 
-def download_extra_workspace_members(repository_ctx, cache_dir, render_template_registry_url):
-    """Download additional workspace members for use in splicing.
-
-    Args:
-        repository_ctx (repository_ctx): The rule's context object.
-        cache_dir (path): A directory in which to download and extract extra workspace members
-        render_template_registry_url (str): The base template to use for determining the crate's registry URL.
-
-    Returns:
-        list: A list of information related to the downloaded crates
-            - manifest: The path of the manifest.
-            - url: The url the manifest came from.
-            - sha256: The sha256 checksum of the new manifest.
-    """
-    manifests = []
-    extra_workspace_members = repository_ctx.attr.extra_workspace_members
-    if extra_workspace_members:
-        repository_ctx.report_progress("Downloading extra workspace members.")
-
-    for name, spec in repository_ctx.attr.extra_workspace_members.items():
-        spec = struct(**json.decode(spec))
-
-        url = render_template_registry_url
-        url = url.replace("{name}", name)
-        url = url.replace("{version}", spec.version)
-
-        if spec.sha256:
-            result = repository_ctx.download_and_extract(
-                output = cache_dir,
-                url = url,
-                sha256 = spec.sha256,
-                type = "tar.gz",
-            )
-        else:
-            result = repository_ctx.download_and_extract(
-                output = cache_dir,
-                url = url,
-                type = "tar.gz",
-            )
-
-        manifest = repository_ctx.path("{}/{}-{}/Cargo.toml".format(
-            cache_dir,
-            name,
-            spec.version,
-        ))
-
-        if not manifest.exists:
-            fail("Extra workspace member '{}' has no root Cargo.toml file".format(name))
-
-        manifests.append(struct(
-            manifest = str(manifest),
-            url = url,
-            sha256 = result.sha256,
-        ))
-
-    return manifests
-
 def kebab_case_keys(data):
     """Ensure the key value of the data given are kebab-case
 
@@ -183,22 +126,6 @@ def splice_workspace_manifest(repository_ctx, generator, lockfile, splicing_mani
     repository_ctx.report_progress("Splicing Cargo workspace.")
     repo_dir = repository_ctx.path(".")
 
-    # Download extra workspace members
-    crates_cache_dir = repository_ctx.path("{}/.crates_cache".format(repo_dir))
-    extra_manifest_info = download_extra_workspace_members(
-        repository_ctx = repository_ctx,
-        cache_dir = crates_cache_dir,
-        render_template_registry_url = repository_ctx.attr.extra_workspace_member_url_template,
-    )
-
-    extra_manifests_manifest = repository_ctx.path("{}/extra_manifests_manifest.json".format(repo_dir))
-    repository_ctx.file(
-        extra_manifests_manifest,
-        json.encode_indent(struct(
-            manifests = extra_manifest_info,
-        ), indent = " " * 4),
-    )
-
     splicing_output_dir = repository_ctx.path("splicing-output")
 
     # Generate a workspace root which contains all workspace members
@@ -209,8 +136,6 @@ def splice_workspace_manifest(repository_ctx, generator, lockfile, splicing_mani
         splicing_output_dir,
         "--splicing-manifest",
         splicing_manifest,
-        "--extra-manifests-manifest",
-        extra_manifests_manifest,
         "--cargo",
         cargo,
         "--rustc",

crate_universe/src/cli/splice.rs

@@ -7,9 +7,7 @@ use clap::Parser;
 
 use crate::cli::Result;
 use crate::metadata::{write_metadata, Generator, MetadataGenerator};
-use crate::splicing::{
-    generate_lockfile, ExtraManifestsManifest, Splicer, SplicingManifest, WorkspaceMetadata,
-};
+use crate::splicing::{generate_lockfile, Splicer, SplicingManifest, WorkspaceMetadata};
 
 /// Command line options for the `splice` subcommand
 #[derive(Parser, Debug)]
@@ -19,10 +17,6 @@ pub struct SpliceOptions {
     #[clap(long)]
     pub splicing_manifest: PathBuf,
 
-    /// A generated manifest of "extra workspace members"
-    #[clap(long)]
-    pub extra_manifests_manifest: PathBuf,
-
     /// A Cargo lockfile (Cargo.lock).
     #[clap(long)]
     pub cargo_lockfile: Option<PathBuf>,
@@ -57,8 +51,6 @@ pub struct SpliceOptions {
 pub fn splice(opt: SpliceOptions) -> Result<()> {
     // Load the all config files required for splicing a workspace
     let splicing_manifest = SplicingManifest::try_from_path(&opt.splicing_manifest)?;
-    let extra_manifests_manifest =
-        ExtraManifestsManifest::try_from_path(opt.extra_manifests_manifest)?;
 
     // Determine the splicing workspace
     let temp_dir;
@@ -71,7 +63,7 @@ pub fn splice(opt: SpliceOptions) -> Result<()> {
     };
 
     // Generate a splicer for creating a Cargo workspace manifest
-    let splicer = Splicer::new(splicing_dir, splicing_manifest, extra_manifests_manifest)?;
+    let splicer = Splicer::new(splicing_dir, splicing_manifest)?;
 
     // Splice together the manifest
     let manifest_path = splicer.splice_workspace()?;

crate_universe/src/cli/vendor.rs

@@ -14,9 +14,7 @@ use crate::context::Context;
 use crate::metadata::{Annotations, VendorGenerator};
 use crate::metadata::{Generator, MetadataGenerator};
 use crate::rendering::{render_module_label, write_outputs, Renderer};
-use crate::splicing::{
-    generate_lockfile, ExtraManifestsManifest, Splicer, SplicingManifest, WorkspaceMetadata,
-};
+use crate::splicing::{generate_lockfile, Splicer, SplicingManifest, WorkspaceMetadata};
 
 /// Command line options for the `vendor` subcommand
 #[derive(Parser, Debug)]
@@ -55,10 +53,6 @@ pub struct VendorOptions {
     #[clap(long)]
     pub metadata: Option<PathBuf>,
 
-    /// A generated manifest of "extra workspace members"
-    #[clap(long)]
-    pub extra_manifests_manifest: PathBuf,
-
     /// The path to a bazel binary
     #[clap(long, env = "BAZEL_REAL", default_value = "bazel")]
     pub bazel: PathBuf,
@@ -117,18 +111,12 @@ pub fn vendor(opt: VendorOptions) -> Result<()> {
     // Load the all config files required for splicing a workspace
     let splicing_manifest = SplicingManifest::try_from_path(&opt.splicing_manifest)?
         .resolve(&opt.workspace_dir, &output_base);
-    let extra_manifests_manifest =
-        ExtraManifestsManifest::try_from_path(opt.extra_manifests_manifest)?.absolutize();
 
     let temp_dir = tempfile::tempdir().context("Failed to create temporary directory")?;
 
     // Generate a splicer for creating a Cargo workspace manifest
-    let splicer = Splicer::new(
-        PathBuf::from(temp_dir.as_ref()),
-        splicing_manifest,
-        extra_manifests_manifest,
-    )
-    .context("Failed to create splicer")?;
+    let splicer = Splicer::new(PathBuf::from(temp_dir.as_ref()), splicing_manifest)
+        .context("Failed to create splicer")?;
 
     // Splice together the manifest
     let manifest_path = splicer

crate_universe/src/splicing.rs

@@ -21,20 +21,9 @@ use crate::utils::starlark::Label;
 use self::cargo_config::CargoConfig;
 pub use self::splicer::*;
 
-#[derive(Debug, Default, Serialize, Deserialize)]
-pub struct ExtraManifestInfo {
-    // The path to a Cargo Manifest
-    pub manifest: PathBuf,
-
-    // The URL where the manifest's package can be downloaded
-    pub url: String,
-
-    // The Sha256 checksum of the downloaded package located at `url`.
-    pub sha256: String,
-}
-
 type DirectPackageManifest = BTreeMap<String, cargo_toml::DependencyDetail>;
 
+/// A collection of information used for splicing together a new Cargo manifest.
 #[derive(Debug, Default, Serialize, Deserialize, Clone)]
 #[serde(deny_unknown_fields)]
 pub struct SplicingManifest {
@@ -104,6 +93,7 @@ impl SplicingManifest {
     }
 }
 
+/// The result of fully resolving a [SplicingManifest] in preparation for splicing.
 #[derive(Debug, Serialize, Default)]
 pub struct SplicingMetadata {
     /// A set of all packages directly written to the rule
@@ -149,32 +139,6 @@ impl TryFrom<SplicingManifest> for SplicingMetadata {
     }
 }
 
-/// A collection of information required for reproducible "extra worksspace members".
-#[derive(Debug, Default, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub struct ExtraManifestsManifest {
-    pub manifests: Vec<ExtraManifestInfo>,
-}
-
-impl FromStr for ExtraManifestsManifest {
-    type Err = serde_json::Error;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        serde_json::from_str(s)
-    }
-}
-
-impl ExtraManifestsManifest {
-    pub fn try_from_path<T: AsRef<Path>>(path: T) -> Result<Self> {
-        let content = fs::read_to_string(path.as_ref())?;
-        Self::from_str(&content).context("Failed to load ExtraManifestsManifest")
-    }
-
-    pub fn absolutize(self) -> Self {
-        self
-    }
-}
-
 #[derive(Debug, Default, Serialize, Deserialize, Clone)]
 pub struct SourceInfo {
     /// A url where to a `.crate` file.
@@ -229,30 +193,9 @@ impl TryFrom<serde_json::Value> for WorkspaceMetadata {
 impl WorkspaceMetadata {
     fn new(
         splicing_manifest: &SplicingManifest,
-        extra_manifests_manifest: &ExtraManifestsManifest,
-        injected_manifests: HashMap<&PathBuf, String>,
+        member_manifests: HashMap<&PathBuf, String>,
     ) -> Result<Self> {
-        let mut sources = BTreeMap::new();
-
-        for config in extra_manifests_manifest.manifests.iter() {
-            let package = match read_manifest(&config.manifest) {
-                Ok(manifest) => match manifest.package {
-                    Some(pkg) => pkg,
-                    None => continue,
-                },
-                Err(e) => return Err(e),
-            };
-
-            let id = CrateId::new(package.name, package.version);
-            let info = SourceInfo {
-                url: config.url.clone(),
-                sha256: config.sha256.clone(),
-            };
-
-            sources.insert(id, info);
-        }
-
-        let mut package_prefixes: BTreeMap<String, String> = injected_manifests
+        let mut package_prefixes: BTreeMap<String, String> = member_manifests
             .iter()
             .filter_map(|(original_manifest, cargo_pkg_name)| {
                 let label = match splicing_manifest.manifests.get(*original_manifest) {
@@ -287,7 +230,7 @@ impl WorkspaceMetadata {
             .collect();
 
         Ok(Self {
-            sources,
+            sources: BTreeMap::new(),
             workspace_prefix,
             package_prefixes,
         })