Add no-merge-sources of cargo vendor to handle duplicates

Author: junjihashimoto

src/bin/cargo/commands/vendor.rs

@@ -32,7 +32,7 @@ pub fn cli() -> Command {
             "versioned-dirs",
             "Always include version in subdir name",
         ))
-        .arg(unsupported("no-merge-sources"))
+        .arg(flag("no-merge-sources", "Keep sources separate"))
         .arg(unsupported("relative-path"))
         .arg(unsupported("only-git-deps"))
         .arg(unsupported("disallow-duplicates"))
@@ -79,6 +79,7 @@ pub fn exec(config: &mut Config, args: &ArgMatches) -> CliResult {
                 .unwrap_or_default()
                 .cloned()
                 .collect(),
+            no_merge_sources: args.flag("no-merge-sources"),
         },
     )?;
     Ok(())

src/cargo/ops/vendor.rs

@@ -1,6 +1,6 @@
 use crate::core::package::MANIFEST_PREAMBLE;
 use crate::core::shell::Verbosity;
-use crate::core::{GitReference, Package, Workspace};
+use crate::core::{GitReference, Package, SourceId, Workspace};
 use crate::ops;
 use crate::sources::path::PathSource;
 use crate::sources::CRATES_IO_REGISTRY;
@@ -9,18 +9,23 @@ use crate::util::{try_canonicalize, CargoResult, Config};
 use anyhow::{bail, Context as _};
 use cargo_util::{paths, Sha256};
 use serde::Serialize;
+use std::collections::hash_map::DefaultHasher;
 use std::collections::HashSet;
 use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::ffi::OsStr;
 use std::fs::{self, File, OpenOptions};
+use std::hash::Hasher;
 use std::io::{Read, Write};
 use std::path::{Path, PathBuf};
 
+const SOURCES_FILE_NAME: &str = ".sources";
+
 pub struct VendorOptions<'a> {
     pub no_delete: bool,
     pub versioned_dirs: bool,
     pub destination: &'a Path,
     pub extra: Vec<PathBuf>,
+    pub no_merge_sources: bool,
 }
 
 pub fn vendor(ws: &Workspace<'_>, opts: &VendorOptions<'_>) -> CargoResult<()> {
@@ -88,8 +93,16 @@ fn sync(
     let canonical_destination = try_canonicalize(opts.destination);
     let canonical_destination = canonical_destination.as_deref().unwrap_or(opts.destination);
     let dest_dir_already_exists = canonical_destination.exists();
+    let merge_sources = !opts.no_merge_sources;
+    let sources_file = canonical_destination.join(SOURCES_FILE_NAME);
 
     paths::create_dir_all(&canonical_destination)?;
+
+    if !merge_sources {
+        let mut file = File::create(sources_file)?;
+        file.write_all(serde_json::json!([]).to_string().as_bytes())?;
+    }
+
     let mut to_remove = HashSet::new();
     if !opts.no_delete {
         for entry in canonical_destination.read_dir()? {
@@ -176,8 +189,9 @@ fn sync(
     let mut versions = HashMap::new();
     for id in ids.keys() {
         let map = versions.entry(id.name()).or_insert_with(BTreeMap::default);
-        if let Some(prev) = map.get(&id.version()) {
-            bail!(
+
+        match map.get(&id.version()) {
+            Some(prev) if merge_sources => bail!(
                 "found duplicate version of package `{} v{}` \
                  vendored from two sources:\n\
                  \n\
@@ -187,7 +201,8 @@ fn sync(
                 id.version(),
                 prev,
                 id.source_id()
-            );
+            ),
+            _ => {}
         }
         map.insert(id.version(), id.source_id());
     }
@@ -211,7 +226,17 @@ fn sync(
         };
 
         sources.insert(id.source_id());
-        let dst = canonical_destination.join(&dst_name);
+        let source_dir = if merge_sources {
+            PathBuf::from(canonical_destination).clone()
+        } else {
+            PathBuf::from(canonical_destination).join(source_id_to_dir_name(id.source_id()))
+        };
+        if sources.insert(id.source_id()) && !merge_sources {
+            if fs::create_dir_all(&source_dir).is_err() {
+                panic!("failed to create: `{}`", source_dir.display())
+            }
+        }
+        let dst = source_dir.join(&dst_name);
         to_remove.remove(&dst);
         let cksum = dst.join(".cargo-checksum.json");
         if dir_has_version_suffix && cksum.exists() {
@@ -248,6 +273,31 @@ fn sync(
         }
     }
 
+    if !merge_sources {
+        let sources_file = PathBuf::from(canonical_destination).join(SOURCES_FILE_NAME);
+        let file = File::open(&sources_file)?;
+        let mut new_sources: BTreeSet<String> = sources
+            .iter()
+            .map(|src_id| source_id_to_dir_name(*src_id))
+            .collect();
+        let old_sources: BTreeSet<String> = serde_json::from_reader::<_, BTreeSet<String>>(file)?
+            .difference(&new_sources)
+            .map(|e| e.clone())
+            .collect();
+        for dir_name in old_sources {
+            let path = PathBuf::from(canonical_destination).join(dir_name.clone());
+            if path.is_dir() {
+                if path.read_dir()?.next().is_none() {
+                    fs::remove_dir(path)?;
+                } else {
+                    new_sources.insert(dir_name.clone());
+                }
+            }
+        }
+        let file = File::create(sources_file)?;
+        serde_json::to_writer(file, &new_sources)?;
+    }
+
     // add our vendored source
     let mut config = BTreeMap::new();
 
@@ -263,16 +313,42 @@ fn sync(
             source_id.without_precise().as_url().to_string()
         };
 
+        let replace_name = if !merge_sources {
+            format!("vendor+{}", name)
+        } else {
+            merged_source_name.to_string()
+        };
+
+        if !merge_sources {
+            let src_id_string = source_id_to_dir_name(source_id);
+            let src_dir = PathBuf::from(canonical_destination).join(src_id_string.clone());
+            match src_dir.to_str() {
+                Some(s) => {
+                    let string = s.to_string();
+                    config.insert(
+                        replace_name.clone(),
+                        VendorSource::Directory { directory: string },
+                    );
+                }
+                None => println!("PathBuf of src_dir contains invalid UTF-8 characters"),
+            }
+        }
+
+        // if source id is a path, skip the source replacement
+        if source_id.is_path() {
+            continue;
+        }
+
         let source = if source_id.is_crates_io() {
             VendorSource::Registry {
                 registry: None,
-                replace_with: merged_source_name.to_string(),
+                replace_with: replace_name,
             }
         } else if source_id.is_remote_registry() {
             let registry = source_id.url().to_string();
             VendorSource::Registry {
                 registry: Some(registry),
-                replace_with: merged_source_name.to_string(),
+                replace_with: replace_name,
             }
         } else if source_id.is_git() {
             let mut branch = None;
@@ -291,7 +367,7 @@ fn sync(
                 branch,
                 tag,
                 rev,
-                replace_with: merged_source_name.to_string(),
+                replace_with: replace_name,
             }
         } else {
             panic!("Invalid source ID: {}", source_id)
@@ -400,6 +476,42 @@ fn cp_sources(
     Ok(())
 }
 
+fn source_id_to_dir_name(src_id: SourceId) -> String {
+    let src_type = if src_id.is_registry() {
+        "registry"
+    } else if src_id.is_git() {
+        "git"
+    } else {
+        panic!()
+    };
+    let mut hasher = DefaultHasher::new();
+    src_id.stable_hash(Path::new(""), &mut hasher);
+    let src_hash = hasher.finish();
+    let mut bytes = [0; 8];
+    for i in 0..7 {
+        bytes[i] = (src_hash >> i * 8) as u8
+    }
+    format!("{}-{}", src_type, hex(&bytes))
+}
+
+fn hex(bytes: &[u8]) -> String {
+    let mut s = String::with_capacity(bytes.len() * 2);
+    for &byte in bytes {
+        s.push(hex((byte >> 4) & 0xf));
+        s.push(hex((byte >> 0) & 0xf));
+    }
+
+    return s;
+
+    fn hex(b: u8) -> char {
+        if b < 10 {
+            (b'0' + b) as char
+        } else {
+            (b'a' + b - 10) as char
+        }
+    }
+}
+
 fn copy_and_checksum<T: Read>(
     dst_path: &Path,
     dst_opts: &mut OpenOptions,

tests/testsuite/cargo_vendor/help/stdout.log

@@ -10,6 +10,7 @@ Options:
   -s, --sync <TOML>            Additional `Cargo.toml` to sync and vendor
       --respect-source-config  Respect `[source]` config in `.cargo/config`
       --versioned-dirs         Always include version in subdir name
+      --no-merge-sources       Keep sources separate
   -v, --verbose...             Use verbose output (-vv very verbose/build.rs output)
   -q, --quiet                  Do not print cargo log messages
       --color <WHEN>           Coloring: auto, always, never

tests/testsuite/vendor.rs

@@ -1151,3 +1151,76 @@ fn vendor_crate_with_ws_inherit() {
         .with_stderr_contains("[..]foo/vendor/bar/src/lib.rs[..]")
         .run();
 }
+
+#[cargo_test]
+fn replace_section() {
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.1.0"
+
+                [dependencies]
+                libc = "0.2.43"
+                [replace."libc:0.2.43"]
+                git = "https://github.com/rust-lang/libc"
+                rev = "add1a320b4e1b454794a034e3f4218f877c393fc"
+            "#,
+        )
+        .file("src/lib.rs", "")
+        .build();
+
+    Package::new("libc", "0.2.43").publish();
+
+    let output = p
+        .cargo("vendor --no-merge-sources")
+        .exec_with_output()
+        .unwrap();
+    p.change_file(".cargo/config", &String::from_utf8(output.stdout).unwrap());
+    assert!(p.root().join("vendor/.sources").exists());
+    p.cargo("check").run();
+}
+
+#[cargo_test]
+fn switch_merged_source() {
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.1.0"
+                [dependencies]
+                log = "0.3.5"
+            "#,
+        )
+        .file("src/lib.rs", "")
+        .build();
+
+    Package::new("log", "0.3.5").publish();
+
+    // Start with multi sources
+    let output = p
+        .cargo("vendor --no-merge-sources")
+        .exec_with_output()
+        .unwrap();
+    assert!(p.root().join("vendor/.sources").exists());
+    p.change_file(".cargo/config", &String::from_utf8(output.stdout).unwrap());
+    p.cargo("check").run();
+
+    // Switch to merged source
+    let output = p.cargo("vendor").exec_with_output().unwrap();
+    p.change_file(".cargo/config", &String::from_utf8(output.stdout).unwrap());
+    p.cargo("check").run();
+
+    // Switch back to multi sources
+    let output = p
+        .cargo("vendor --no-merge-sources")
+        .exec_with_output()
+        .unwrap();
+    p.change_file(".cargo/config", &String::from_utf8(output.stdout).unwrap());
+    assert!(p.root().join("vendor/.sources").exists());
+    p.cargo("check").run();
+}