diff --git a/Cargo.lock b/Cargo.lock index 55a3ecc0d5..bb1b924183 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -126,6 +126,7 @@ dependencies = [ "assert_cli", "atty", "cargo_metadata", + "crates-index", "dirs-next", "env_proxy", "error-chain", @@ -244,6 +245,24 @@ dependencies = [ "unreachable", ] +[[package]] +name = "crates-index" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ad4af5c8dd9940a497ef4473e6e558b660a4a1b6e5ce2cb9d85454e2aaaf947" +dependencies = [ + "git2", + "glob", + "hex", + "home", + "memchr", + "semver", + "serde", + "serde_derive", + "serde_json", + "smartstring", +] + [[package]] name = "ctor" version = "0.1.18" @@ -364,9 +383,9 @@ checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" [[package]] name = "git2" -version = "0.13.17" +version = "0.13.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d250f5f82326884bd39c2853577e70a121775db76818ffa452ed1e80de12986" +checksum = "659cd14835e75b64d9dba5b660463506763cf0aa6cb640aeeb0e98d841093490" dependencies = [ "bitflags", "libc", @@ -377,6 +396,12 @@ dependencies = [ "url", ] +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "heck" version = "0.3.2" @@ -397,9 +422,21 @@ dependencies = [ [[package]] name = "hex" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +dependencies = [ + "serde", +] + +[[package]] +name = "home" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2456aef2e6b6a9784192ae780c0f15bc57df0e918585282325e8c8ac27737654" +dependencies = [ + "winapi 0.3.9", +] [[package]] name = "idna" @@ -450,9 +487,9 @@ checksum = "1cca32fa0182e8c0989459524dc356b8f2b5c10f1b9eb521b7d182c03cf8c5ff" [[package]] name = "libgit2-sys" -version = "0.12.18+1.1.0" +version = "0.12.22+1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3da6a42da88fc37ee1ecda212ffa254c25713532980005d5f7c0b0fbe7e6e885" +checksum = "89c53ac117c44f7042ad8d8f5681378dfbc6010e49ec2c0d1f11dfedc7a4a1c3" dependencies = [ "cc", "libc", @@ -511,9 +548,9 @@ checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] name = "memchr" -version = "2.3.4" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" [[package]] name = "miniz_oxide" @@ -822,18 +859,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.123" +version = "1.0.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" +checksum = "d1f72836d2aa753853178eda473a3b9d8e4eefdaf20523b919677e6de489f8f1" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.123" +version = "1.0.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" +checksum = "e57ae87ad533d9a56427558b516d0adac283614e347abf85b0dc0cbbf0a249f3" dependencies = [ "proc-macro2", "quote", @@ -842,15 +879,25 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.61" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fceb2595057b6891a4ee808f70054bd2d12f0e97f1cbb78689b59f676df325a" +checksum = "336b10da19a12ad094b59d870ebde26a45402e5b470add4b5fd03c5048a32127" dependencies = [ "itoa", "ryu", "serde", ] +[[package]] +name = "smartstring" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31aa6a31c0c2b21327ce875f7e8952322acfcfd0c27569a6e18a647281352c9b" +dependencies = [ + "serde", + "static_assertions", +] + [[package]] name = "socks" version = "0.3.3" @@ -869,6 +916,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strsim" version = "0.8.0" diff --git a/Cargo.toml b/Cargo.toml index 4fb854c000..844e3ca74d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ required-features = ["upgrade"] [dependencies] atty = { version = "0.2.14", optional = true } cargo_metadata = "0.14.0" +crates-index = "0.17.0" dirs-next = "2.0.0" env_proxy = "0.4.1" error-chain = "0.12.4" diff --git a/src/errors.rs b/src/errors.rs index 6d30599413..91bdf0cb1f 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -5,6 +5,8 @@ error_chain! { Io(::std::io::Error) #[doc = "An error from the std::io module"]; Git(::git2::Error)#[doc = "An error from the git2 crate"]; CargoMetadata(::failure::Compat<::cargo_metadata::Error>)#[doc = "An error from the cargo_metadata crate"]; + Semver(::semver::Error)#[doc = "An error from the semver crate"]; + CratesIndex(::crates_index::Error)#[doc = "An error from the crates-index crate"]; } errors { diff --git a/src/fetch.rs b/src/fetch.rs index 7dde3b88fa..6bfee89da5 100644 --- a/src/fetch.rs +++ b/src/fetch.rs @@ -1,19 +1,18 @@ use crate::errors::*; -use crate::registry::{registry_path, registry_path_from_url}; +use crate::registry::registry_url; use crate::VersionExt; use crate::{Dependency, Manifest}; use regex::Regex; use std::env; use std::io::Write; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::time::Duration; use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor}; use url::Url; -#[derive(Deserialize)] +#[derive(Debug)] struct CrateVersion { name: String, - #[serde(rename = "vers")] version: semver::Version, yanked: bool, } @@ -54,12 +53,12 @@ pub fn get_latest_dependency( return Err(ErrorKind::EmptyCrateName.into()); } - let registry_path = match registry { - Some(url) => registry_path_from_url(url)?, - None => registry_path(manifest_path, None)?, + let registry = match registry { + Some(url) => url.clone(), + None => registry_url(manifest_path, None)?, }; - let crate_versions = fuzzy_query_registry_index(crate_name, ®istry_path)?; + let crate_versions = fuzzy_query_registry_index(crate_name, ®istry)?; let dep = read_latest_version(&crate_versions, flag_allow_prerelease)?; @@ -94,8 +93,6 @@ fn read_latest_version( /// update registry index for given project pub fn update_registry_index(registry: &Url, quiet: bool) -> Result<()> { - let registry_path = registry_path_from_url(registry)?; - let colorchoice = if atty::is(atty::Stream::Stdout) { ColorChoice::Auto } else { @@ -103,19 +100,8 @@ pub fn update_registry_index(registry: &Url, quiet: bool) -> Result<()> { }; let mut output = StandardStream::stdout(colorchoice); - if !registry_path.as_path().exists() { - output.set_color(ColorSpec::new().set_fg(Some(Color::Green)).set_bold(true))?; - write!(output, "{:>12}", "Initializing")?; - output.reset()?; - writeln!(output, " '{}' index", registry)?; - - let mut opts = git2::RepositoryInitOptions::new(); - opts.bare(true); - git2::Repository::init_opts(®istry_path, &opts)?; - return Ok(()); - } - - let repo = git2::Repository::open(®istry_path)?; + let index = crates_index::BareIndex::from_url(registry.as_str())?; + let mut index = index.open_or_clone()?; if !quiet { output.set_color(ColorSpec::new().set_fg(Some(Color::Green)).set_bold(true))?; write!(output, "{:>12}", "Updating")?; @@ -123,62 +109,55 @@ pub fn update_registry_index(registry: &Url, quiet: bool) -> Result<()> { writeln!(output, " '{}' index", registry)?; } - let refspec = format!( - "refs/heads/{0}:refs/remotes/origin/{0}", - get_checkout_name(registry_path)? - ); - fetch_with_cli(&repo, registry.as_str(), &refspec)?; + while let Err(err) = index.retrieve() { + if need_retry(err)? { + registry_blocked_message(&mut output)?; + std::thread::sleep(REGISTRY_BACKOFF); + } else { + break; + } + } Ok(()) } -// https://github.com/rust-lang/cargo/blob/57986eac7157261c33f0123bade7ccd20f15200f/src/cargo/sources/git/utils.rs#L758 -fn fetch_with_cli(repo: &git2::Repository, url: &str, refspec: &str) -> Result<()> { - let cmd = subprocess::Exec::shell("git") - .arg("fetch") - .arg("--tags") // fetch all tags - .arg("--force") // handle force pushes - .arg("--update-head-ok") // see discussion in rust-lang/cargo#2078 - .arg(url) - .arg(refspec) - // If cargo is run by git (for example, the `exec` command in `git - // rebase`), the GIT_DIR is set by git and will point to the wrong - // location (this takes precedence over the cwd). Make sure this is - // unset so git will look at cwd for the repo. - .env_remove("GIT_DIR") - // The reset of these may not be necessary, but I'm including them - // just to be extra paranoid and avoid any issues. - .env_remove("GIT_WORK_TREE") - .env_remove("GIT_INDEX_FILE") - .env_remove("GIT_OBJECT_DIRECTORY") - .env_remove("GIT_ALTERNATE_OBJECT_DIRECTORIES") - .cwd(repo.path()); - - let _ = cmd.capture().map_err(|e| match e { - subprocess::PopenError::IoError(io) => ErrorKind::Io(io), - _ => unreachable!("expected only io error"), - })?; +const REGISTRY_BACKOFF: Duration = Duration::from_secs(1); + +fn need_retry(err: crates_index::Error) -> Result { + match err { + crates_index::Error::Git(err) => { + if err.class() == git2::ErrorClass::Index && err.code() == git2::ErrorCode::Locked { + Ok(true) + } else { + Err(crates_index::Error::Git(err).into()) + } + } + err => Err(err.into()), + } +} + +fn registry_blocked_message(output: &mut StandardStream) -> Result<()> { + output.set_color(ColorSpec::new().set_fg(Some(Color::Green)).set_bold(true))?; + write!(output, "{:>12}", "Blocking")?; + output.reset()?; + writeln!(output, " waiting for lock on registry index")?; Ok(()) } #[test] -fn get_latest_stable_version_from_json() { - let versions: Vec = serde_json::from_str( - r#"[ - { - "name": "foo", - "vers": "0.6.0-alpha", - "yanked": false +fn get_latest_stable_version() { + let versions = vec![ + CrateVersion { + name: "foo".into(), + version: "0.6.0-alpha".parse().unwrap(), + yanked: false, }, - { - "name": "foo", - "vers": "0.5.0", - "yanked": false - } - ]"#, - ) - .expect("crate version is correctly parsed"); - + CrateVersion { + name: "foo".into(), + version: "0.5.0".parse().unwrap(), + yanked: false, + }, + ]; assert_eq!( read_latest_version(&versions, false) .unwrap() @@ -189,23 +168,19 @@ fn get_latest_stable_version_from_json() { } #[test] -fn get_latest_unstable_or_stable_version_from_json() { - let versions: Vec = serde_json::from_str( - r#"[ - { - "name": "foo", - "vers": "0.6.0-alpha", - "yanked": false +fn get_latest_unstable_or_stable_version() { + let versions = vec![ + CrateVersion { + name: "foo".into(), + version: "0.6.0-alpha".parse().unwrap(), + yanked: false, }, - { - "name": "foo", - "vers": "0.5.0", - "yanked": false - } - ]"#, - ) - .expect("crate version is correctly parsed"); - + CrateVersion { + name: "foo".into(), + version: "0.5.0".parse().unwrap(), + yanked: false, + }, + ]; assert_eq!( read_latest_version(&versions, true) .unwrap() @@ -216,23 +191,19 @@ fn get_latest_unstable_or_stable_version_from_json() { } #[test] -fn get_latest_version_from_json_test() { - let versions: Vec = serde_json::from_str( - r#"[ - { - "name": "treexml", - "vers": "0.3.1", - "yanked": true +fn get_latest_version_with_yanked() { + let versions = vec![ + CrateVersion { + name: "treexml".into(), + version: "0.3.1".parse().unwrap(), + yanked: true, }, - { - "name": "treexml", - "vers": "0.3.0", - "yanked": false - } - ]"#, - ) - .expect("crate version is correctly parsed"); - + CrateVersion { + name: "true".into(), + version: "0.3.0".parse().unwrap(), + yanked: false, + }, + ]; assert_eq!( read_latest_version(&versions, false) .unwrap() @@ -244,61 +215,30 @@ fn get_latest_version_from_json_test() { #[test] fn get_no_latest_version_from_json_when_all_are_yanked() { - let versions: Vec = serde_json::from_str( - r#"[ - { - "name": "treexml", - "vers": "0.3.1", - "yanked": true + let versions = vec![ + CrateVersion { + name: "treexml".into(), + version: "0.3.1".parse().unwrap(), + yanked: true, }, - { - "name": "treexml", - "vers": "0.3.0", - "yanked": true - } - ]"#, - ) - .expect("crate version is correctly parsed"); - + CrateVersion { + name: "true".into(), + version: "0.3.0".parse().unwrap(), + yanked: true, + }, + ]; assert!(read_latest_version(&versions, false).is_err()); } -/// Gets the checkedout branch name of .cargo/registry/index/github.com-*/.git/refs or -/// .cargo/registry/index/github.com-*/refs for bare git repository -fn get_checkout_name(registry_path: impl AsRef) -> Result { - let checkout_dir = registry_path - .as_ref() - .join(".git") - .join("refs/remotes/origin/"); - let bare_checkout_dir = registry_path.as_ref().join("refs/remotes/origin/"); - - Ok(checkout_dir - .read_dir() // .git repo - .or_else(|_| bare_checkout_dir.read_dir())? // there's no .git, it's bare one - .next() //Is there always only one branch? (expecting either master og HEAD) - .ok_or(ErrorKind::MissingRegistraryCheckout(checkout_dir))?? - .file_name() - .into_string() - .map_err(|_| ErrorKind::NonUnicodeGitPath)?) -} - /// Fuzzy query crate from registry index fn fuzzy_query_registry_index( crate_name: impl Into, - registry_path: impl AsRef, + registry: &Url, ) -> Result> { - let crate_name = crate_name.into(); - let remotes = PathBuf::from("refs/remotes/origin/"); - let repo = git2::Repository::open(®istry_path)?; - let tree = repo - .find_reference( - remotes - .join(get_checkout_name(®istry_path)?) - .to_str() - .ok_or(ErrorKind::NonUnicodeGitPath)?, - )? - .peel_to_tree()?; + let index = crates_index::BareIndex::from_url(registry.as_str())?; + let index = index.open_or_clone()?; + let crate_name = crate_name.into(); let mut names = gen_fuzzy_crate_names(crate_name.clone())?; if let Some(index) = names.iter().position(|x| *x == crate_name) { // ref: https://github.com/killercup/cargo-edit/pull/317#discussion_r307365704 @@ -306,20 +246,21 @@ fn fuzzy_query_registry_index( } for the_name in names { - let file = match tree.get_path(&PathBuf::from(summary_raw_path(&the_name))) { - Ok(x) => x.to_object(&repo)?.peel_to_blob()?, - Err(_) => continue, + let crate_ = match index.crate_(&the_name) { + Some(crate_) => crate_, + None => continue, }; - let content = String::from_utf8(file.content().to_vec()) - .map_err(|_| ErrorKind::InvalidSummaryJson)?; - - return content - .lines() - .map(|line: &str| { - serde_json::from_str::(line) - .map_err(|_| ErrorKind::InvalidSummaryJson.into()) + return crate_ + .versions() + .iter() + .map(|v| { + Ok(CrateVersion { + name: v.name().to_owned(), + version: v.version().parse()?, + yanked: v.is_yanked(), + }) }) - .collect::>>(); + .collect(); } Err(ErrorKind::NoCrate(crate_name).into()) } @@ -490,24 +431,3 @@ fn test_gen_fuzzy_crate_names() { &["DC__janus", "DC_-janus", "DC-_janus", "DC--janus"], ); } - -fn summary_raw_path(crate_name: &str) -> String { - let crate_name = crate_name.to_ascii_lowercase(); - match crate_name.len() { - 0 => unreachable!("we check that crate_name is not empty here"), - 1 => format!("1/{}", crate_name), - 2 => format!("2/{}", crate_name), - 3 => format!("3/{}/{}", &crate_name[..1], crate_name), - _ => format!("{}/{}/{}", &crate_name[..2], &crate_name[2..4], crate_name), - } -} - -#[test] -fn test_summary_raw_path() { - assert_eq!(summary_raw_path("a"), "1/a"); - assert_eq!(summary_raw_path("ab"), "2/ab"); - assert_eq!(summary_raw_path("abc"), "3/a/abc"); - assert_eq!(summary_raw_path("abcd"), "ab/cd/abcd"); - assert_eq!(summary_raw_path("abcdefg"), "ab/cd/abcdefg"); - assert_eq!(summary_raw_path("Inflector"), "in/fl/inflector"); -} diff --git a/src/registry.rs b/src/registry.rs index 756f4555b2..bd3c367626 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -1,4 +1,3 @@ -use self::code_from_cargo::Kind; use crate::errors::*; use std::collections::HashMap; use std::path::{Path, PathBuf}; @@ -7,17 +6,6 @@ use url::Url; const CRATES_IO_INDEX: &str = "https://github.com/rust-lang/crates.io-index"; const CRATES_IO_REGISTRY: &str = "crates-io"; -pub fn registry_path(manifest_path: &Path, registry: Option<&str>) -> Result { - registry_path_from_url(®istry_url(manifest_path, registry)?) -} - -pub fn registry_path_from_url(registry: &Url) -> Result { - Ok(cargo_home()? - .join("registry") - .join("index") - .join(short_name(registry))) -} - #[derive(Debug, Deserialize)] struct Source { #[serde(rename = "replace-with")] @@ -130,33 +118,6 @@ pub fn registry_url(manifest_path: &Path, registry: Option<&str>) -> Result Ok(registry_url) } -fn short_name(registry: &Url) -> String { - // ref: https://github.com/rust-lang/cargo/blob/4c1fa54d10f58d69ac9ff55be68e1b1c25ecb816/src/cargo/sources/registry/mod.rs#L386-L390 - #![allow(deprecated)] - use std::hash::{Hash, Hasher, SipHasher}; - - let mut hasher = SipHasher::new(); - Kind::Registry.hash(&mut hasher); - registry.as_str().hash(&mut hasher); - let hash = hex::encode(hasher.finish().to_le_bytes()); - - let ident = registry.host_str().unwrap_or("").to_string(); - - format!("{}-{}", ident, hash) -} - -#[cfg_attr(target_pointer_width = "64", test)] -fn test_short_name() { - fn test_helper(url: &str, name: &str) { - let url = Url::parse(url).unwrap(); - assert_eq!(short_name(&url), name); - } - test_helper( - "https://github.com/rust-lang/crates.io-index", - "github.com-1ecc6299db9ec823", - ); -} - mod code_from_cargo { #![allow(dead_code)]