diff --git a/examples/sparse_http_reqwest.rs b/examples/sparse_http_reqwest.rs index 509f3112..94151f07 100644 --- a/examples/sparse_http_reqwest.rs +++ b/examples/sparse_http_reqwest.rs @@ -1,4 +1,4 @@ -use crates_index::{SparseIndex}; +use crates_index::SparseIndex; /// /// **important**:
@@ -18,18 +18,18 @@ fn main() { print_crate(&mut index); } -fn print_crate(index: &mut SparseIndex){ +fn print_crate(index: &mut SparseIndex) { match index.crate_from_cache(CRATE_TO_FETCH) { Ok(krate) => { println!("{:?}", krate.highest_normal_version().unwrap().version()); } Err(_err) => { - println!("could not find crate {}",CRATE_TO_FETCH) + println!("could not find crate {}", CRATE_TO_FETCH) } } } -fn update(index: &mut SparseIndex){ +fn update(index: &mut SparseIndex) { let req = index.make_cache_request(CRATE_TO_FETCH).unwrap().body(()).unwrap(); let (parts, _) = req.into_parts(); @@ -41,9 +41,7 @@ fn update(index: &mut SparseIndex){ let res = client.execute(req).unwrap(); - let mut builder = http::Response::builder() - .status(res.status()) - .version(res.version()); + let mut builder = http::Response::builder().status(res.status()).version(res.version()); builder .headers_mut() @@ -54,4 +52,4 @@ fn update(index: &mut SparseIndex){ let res = builder.body(body.to_vec()).unwrap(); index.parse_cache_response(CRATE_TO_FETCH, res, true).unwrap(); -} \ No newline at end of file +} diff --git a/examples/sparse_http_ureq.rs b/examples/sparse_http_ureq.rs index 42a59e86..3f939b9c 100644 --- a/examples/sparse_http_ureq.rs +++ b/examples/sparse_http_ureq.rs @@ -1,5 +1,5 @@ +use crates_index::SparseIndex; use std::io; -use crates_index::{SparseIndex}; /// /// **important**:
@@ -19,27 +19,28 @@ fn main() { print_crate(&mut index); } -fn print_crate(index: &mut SparseIndex){ +fn print_crate(index: &mut SparseIndex) { match index.crate_from_cache(CRATE_TO_FETCH) { Ok(krate) => { println!("{:?}", krate.highest_normal_version().unwrap().version()); } Err(_err) => { - println!("could not find crate {}",CRATE_TO_FETCH) + println!("could not find crate {}", CRATE_TO_FETCH) } } } -fn update(index: &mut SparseIndex){ +fn update(index: &mut SparseIndex) { let request: ureq::Request = index.make_cache_request(CRATE_TO_FETCH).unwrap().into(); let response: http::Response = request .call() - .map_err(|_e| io::Error::new(io::ErrorKind::InvalidInput, "connection error")).unwrap() + .map_err(|_e| io::Error::new(io::ErrorKind::InvalidInput, "connection error")) + .unwrap() .into(); let (parts, body) = response.into_parts(); let response = http::Response::from_parts(parts, body.into_bytes()); index.parse_cache_response(CRATE_TO_FETCH, response, true).unwrap(); -} \ No newline at end of file +} diff --git a/examples/update_and_get_latest.rs b/examples/update_and_get_latest.rs index 14c1605d..c625a728 100644 --- a/examples/update_and_get_latest.rs +++ b/examples/update_and_get_latest.rs @@ -3,12 +3,16 @@ fn main() -> Result<(), Box> { let mut index = crates_index::Index::new_cargo_default()?; println!("Updating index…"); index.update()?; - + let limit = 10; println!("The most recent {limit} changes:\n"); for change in index.changes()?.take(limit) { let change = change?; - println!("{name} changed in {commit}", name = change.crate_name(), commit = change.commit_hex()); + println!( + "{name} changed in {commit}", + name = change.crate_name(), + commit = change.commit_hex() + ); } Ok(()) -} \ No newline at end of file +} diff --git a/rustfmt.toml b/rustfmt.toml index c7ad93ba..0a453277 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1 +1,2 @@ -disable_all_formatting = true +max_width = 120 +disable_all_formatting = false diff --git a/src/bare_index/changes.rs b/src/bare_index/changes.rs index 1d609b0c..a460458a 100644 --- a/src/bare_index/changes.rs +++ b/src/bare_index/changes.rs @@ -68,7 +68,15 @@ impl<'repo> Iterator for ChangesIter<'repo> { }; let parent_tree = parent.tree().ok()?; let time = SystemTime::UNIX_EPOCH + Duration::from_secs(self.current.time().ok()?.seconds.max(0) as _); - Self::tree_additions(&self.repo, &mut self.out, time, &self.current.id(), &self.current_tree, &parent_tree).ok()?; + Self::tree_additions( + &self.repo, + &mut self.out, + time, + &self.current.id(), + &self.current_tree, + &parent_tree, + ) + .ok()?; self.current_tree = parent_tree; self.current = parent; } @@ -78,7 +86,11 @@ impl<'repo> Iterator for ChangesIter<'repo> { impl<'repo> ChangesIter<'repo> { pub(crate) fn new(index: &'repo Index) -> Result { - let current = index.repo.find_object(index.head_commit)?.peel_to_kind(gix::object::Kind::Commit)?.into_commit(); + let current = index + .repo + .find_object(index.head_commit)? + .peel_to_kind(gix::object::Kind::Commit)? + .into_commit(); let current_tree = current.tree()?; Ok(Self { @@ -90,7 +102,13 @@ impl<'repo> ChangesIter<'repo> { } fn get_parent(&self) -> Result>, GixError> { - match self.current.parent_ids().next().map(|id| id.try_object()).transpose()?.flatten() + match self + .current + .parent_ids() + .next() + .map(|id| id.try_object()) + .transpose()? + .flatten() { Some(obj) => Ok(Some(obj.try_into_commit()?)), None => { @@ -134,13 +152,12 @@ impl<'repo> ChangesIter<'repo> { // Recurse only into crate subdirs, and they all happen to be 1 or 2 letters long let is_crates_subdir = name.len() <= 2 && name.iter().copied().all(valid_crate_name_char); let old_obj = if is_crates_subdir { - old.bisect_entry(name, true) - .map(|entry| entry.attach(repo)) + old.bisect_entry(name, true).map(|entry| entry.attach(repo)) } else { None } - .map(|o| o.object()) - .transpose()?; + .map(|o| o.object()) + .transpose()?; let old_tree = match old_obj.and_then(|o| o.try_into_tree().ok()) { Some(t) => t, None => repo.empty_tree(), @@ -168,7 +185,10 @@ fn valid_crate_name_char(c: u8) -> bool { } fn oid_and_branch_from_commit_message(msg: &str) -> Option<(gix::ObjectId, &str)> { - let hash_start = msg.split_once("Previous HEAD was ")?.1.trim_start_matches(|c: char| !c.is_ascii_hexdigit()); + let hash_start = msg + .split_once("Previous HEAD was ")? + .1 + .trim_start_matches(|c: char| !c.is_ascii_hexdigit()); let (hash_str, rest) = hash_start.split_once(|c: char| !c.is_ascii_hexdigit())?; let hash = gix::ObjectId::from_hex(hash_str.as_bytes()).ok()?; let snapshot_start = rest.find("snapshot-")?; @@ -179,7 +199,7 @@ fn oid_and_branch_from_commit_message(msg: &str) -> Option<(gix::ObjectId, &str) #[cfg(test)] pub(crate) mod test { - use super::{oid_and_branch_from_commit_message}; + use super::oid_and_branch_from_commit_message; #[test] fn changes_parse_split_message() { @@ -191,7 +211,7 @@ More information about this change can be found [online] and on [this issue]. [online]: https://internals.rust-lang.org/t/cargos-crate-index-upcoming-squash-into-one-commit/8440 [this issue]: https://github.com/rust-lang/crates-io-cargo-teams/issues/47", ) - .unwrap(); + .unwrap(); assert_eq!("4181c62812c70fafb2b56cbbd66c31056671b445", id.to_string()); assert_eq!("snapshot-2021-07-02", branch); } diff --git a/src/bare_index/config.rs b/src/bare_index/config.rs index 6ddeb8d9..6b6a0580 100644 --- a/src/bare_index/config.rs +++ b/src/bare_index/config.rs @@ -15,10 +15,7 @@ fn read_cargo_config( ) -> Result, Error> { use std::borrow::Cow; - if let Some(mut path) = root - .map(PathBuf::from) - .or_else(|| std::env::current_dir().ok()) - { + if let Some(mut path) = root.map(PathBuf::from).or_else(|| std::env::current_dir().ok()) { loop { path.push(".cargo/config.toml"); if let Some(toml) = try_read_toml(&path)? { diff --git a/src/bare_index/mod.rs b/src/bare_index/mod.rs index 428d8db4..080f0b47 100644 --- a/src/bare_index/mod.rs +++ b/src/bare_index/mod.rs @@ -92,27 +92,24 @@ impl Index { fn from_path_and_url(path: PathBuf, url: String) -> Result { let mut mapping = gix::sec::trust::Mapping::default(); - let open_with_complete_config = - gix::open::Options::default().permissions(gix::open::Permissions { - config: gix::open::permissions::Config { - // Be sure to get all configuration, some of which is only known by the git binary. - // That way we are sure to see all the systems credential helpers - git_binary: true, - ..Default::default() - }, + let open_with_complete_config = gix::open::Options::default().permissions(gix::open::Permissions { + config: gix::open::permissions::Config { + // Be sure to get all configuration, some of which is only known by the git binary. + // That way we are sure to see all the systems credential helpers + git_binary: true, ..Default::default() - }); + }, + ..Default::default() + }); mapping.reduced = open_with_complete_config.clone(); mapping.full = open_with_complete_config.clone(); let _lock = gix::lock::Marker::acquire_to_hold_resource( path.with_extension("crates-index"), - gix::lock::acquire::Fail::AfterDurationWithBackoff( - std::time::Duration::from_secs(60 * 10), - ), + gix::lock::acquire::Fail::AfterDurationWithBackoff(std::time::Duration::from_secs(60 * 10)), Some(PathBuf::from_iter(Some(std::path::Component::RootDir))), ) - .map_err(GixError::from)?; + .map_err(GixError::from)?; let repo = gix::ThreadSafeRepository::discover_opts( &path, gix::discover::upwards::Options::default().apply_environment(), @@ -131,12 +128,10 @@ impl Index { let repo = match repo { Some(repo) => repo, - None => { - match gix::open_opts(&path, open_with_complete_config).ok() { - None => clone_url(&url, &path)?, - Some(repo) => repo, - } - } + None => match gix::open_opts(&path, open_with_complete_config).ok() { + None => clone_url(&url, &path)?, + Some(repo) => repo, + }, }; let head_commit = Self::find_repo_head(&repo, &path)?; @@ -150,11 +145,7 @@ impl Index { } fn tree(&self) -> Result, GixError> { - Ok(self - .repo - .find_object(self.head_commit)? - .try_into_commit()? - .tree()?) + Ok(self.repo.find_object(self.head_commit)?.try_into_commit()?.tree()?) } #[doc(hidden)] @@ -180,15 +171,14 @@ impl Index { /// method will mean no cache entries will be used, if a new commit is fetched /// from the repository, as their commit version will no longer match. pub fn update(&mut self) -> Result<(), Error> { - let mut remote = self.repo.find_remote("origin").ok().unwrap_or_else(|| { - self.repo - .remote_at(self.url.as_str()) - .expect("own URL is always valid") - }); - fetch_remote(&mut remote, &[ - "HEAD:refs/remotes/origin/HEAD", - "master:refs/remotes/origin/master", - ] + let mut remote = self + .repo + .find_remote("origin") + .ok() + .unwrap_or_else(|| self.repo.remote_at(self.url.as_str()).expect("own URL is always valid")); + fetch_remote( + &mut remote, + &["HEAD:refs/remotes/origin/HEAD", "master:refs/remotes/origin/master"], )?; let head_commit = Self::find_repo_head(&self.repo, &self.path)?; @@ -197,7 +187,7 @@ impl Index { Ok(()) } - + /// Reads a crate from the index, it will attempt to use a cached entry if /// one is available, otherwise it will fallback to reading the crate /// directly from the git blob containing the crate information. @@ -212,15 +202,12 @@ impl Index { // mechanism and can fail for a few reasons that are non-fatal { // avoid realloc on each push - let mut cache_path = - PathBuf::with_capacity(path_max_byte_len(&self.path) + 8 + rel_path.len()); + let mut cache_path = PathBuf::with_capacity(path_max_byte_len(&self.path) + 8 + rel_path.len()); cache_path.push(&self.path); cache_path.push(".cache"); cache_path.push(&rel_path); if let Ok(cache_bytes) = std::fs::read(&cache_path) { - if let Ok(krate) = - Crate::from_cache_slice(&cache_bytes, Some(&self.head_commit_hex)) - { + if let Ok(krate) = Crate::from_cache_slice(&cache_bytes, Some(&self.head_commit_hex)) { return Some(krate); } } @@ -259,8 +246,7 @@ impl Index { #[must_use] pub fn crates_parallel( &self, - ) -> impl rayon::iter::ParallelIterator> + '_ - { + ) -> impl rayon::iter::ParallelIterator> + '_ { use rayon::iter::{IntoParallelIterator, ParallelIterator}; let tree_oids = match self.crates_top_level_ids() { Ok(objs) => objs, @@ -344,16 +330,16 @@ impl Index { } /// Find the most recent commit of `repo` at `path`. - /// + /// /// This is complicated by a few specialities of the cargo git index. - /// - /// * it's possible for `origin/HEAD` and `origin/master` to be stalled and out of date if they have been fetched with + /// + /// * it's possible for `origin/HEAD` and `origin/master` to be stalled and out of date if they have been fetched with /// non-force refspecs. /// This was done by this crate as well, but is not done by cargo. /// * if `origin/master` is out of date, `FETCH_HEAD` is the only chance for getting the most recent commit. /// * if `gix` is updating the index, `FETCH_HEAD` will not be written at all, *only* the references are. Note that /// `cargo` does not rely on `FETCH_HEAD`, but relies on `origin/master` directly. - /// + /// /// This, we get a list of candidates and use the most recent commit. fn find_repo_head(repo: &gix::Repository, path: &Path) -> Result { #[rustfmt::skip] @@ -400,11 +386,7 @@ fn is_top_level_dir(entry: &gix::object::tree::EntryRef<'_, '_>) -> bool { fn with_delta_cache(mut repo: gix::Repository) -> gix::Repository { if repo .config_snapshot() - .integer_by_key( - gix::config::tree::Core::DELTA_BASE_CACHE_LIMIT - .logical_name() - .as_str(), - ) + .integer_by_key(gix::config::tree::Core::DELTA_BASE_CACHE_LIMIT.logical_name().as_str()) .is_none() { let mut config = repo.config_snapshot_mut(); @@ -417,10 +399,7 @@ fn with_delta_cache(mut repo: gix::Repository) -> gix::Repository { } pub(crate) fn fetch_remote(remote: &mut gix::Remote<'_>, refspecs: &[&str]) -> Result<(), GixError> { - remote.replace_refspecs( - refspecs, - gix::remote::Direction::Fetch, - )?; + remote.replace_refspecs(refspecs, gix::remote::Direction::Fetch)?; remote .connect(gix::remote::Direction::Fetch)? @@ -435,10 +414,7 @@ fn clone_url(url: &str, destination: &Path) -> Result .with_remote_name("origin")? .configure_remote(|remote| { Ok(remote.with_refspecs( - [ - "+HEAD:refs/remotes/origin/HEAD", - "+master:refs/remotes/origin/master", - ], + ["+HEAD:refs/remotes/origin/HEAD", "+master:refs/remotes/origin/master"], gix::remote::Direction::Fetch, )?) }) @@ -470,8 +446,7 @@ impl Iterator for CratesTreesToBlobs { if obj.kind.is_tree() { let tree = gix::objs::TreeRef::from_bytes(&obj.data).unwrap(); for entry in tree.entries.into_iter().rev() { - self.stack - .push(self.repo.find_object(entry.oid).unwrap().detach()); + self.stack.push(self.repo.find_object(entry.oid).unwrap().detach()); } continue; } else { diff --git a/src/bare_index/test.rs b/src/bare_index/test.rs index 315012c7..093b2e5f 100644 --- a/src/bare_index/test.rs +++ b/src/bare_index/test.rs @@ -1,5 +1,5 @@ -use gix::bstr::ByteSlice; use super::*; +use gix::bstr::ByteSlice; #[test] #[cfg_attr(debug_assertions, ignore = "too slow in debug mode")] @@ -35,8 +35,7 @@ fn parse_all_blobs() { fn shared_index() -> Index { let index_path = "tests/fixtures/git-registry"; if is_ci::cached() { - Index::new_cargo_default() - .expect("CI has just cloned this index and its ours and valid") + Index::new_cargo_default().expect("CI has just cloned this index and its ours and valid") } else { Index::with_path(index_path, INDEX_GIT_URL).expect("clone works and there is no racing") } diff --git a/src/config.rs b/src/config.rs index b83c57d6..0264a806 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,5 +1,5 @@ -use serde_derive::Deserialize; use crate::dirs::crate_prefix; +use serde_derive::Deserialize; /// Global configuration of an index, reflecting the [contents of config.json](https://doc.rust-lang.org/cargo/reference/registries.html#index-format). #[derive(Clone, Debug, Deserialize)] diff --git a/src/dedupe.rs b/src/dedupe.rs index a60f79d4..877ae05e 100644 --- a/src/dedupe.rs +++ b/src/dedupe.rs @@ -25,7 +25,8 @@ impl DedupeContext { if let Some(has_feats) = self.features.get(&features_to_dedupe) { *features = Arc::clone(&has_feats.map); } else { - if self.features.len() > 16384 { // keeps peak memory low (must clear, remove is leaving tombstones) + if self.features.len() > 16384 { + // keeps peak memory low (must clear, remove is leaving tombstones) self.features.clear(); } self.features.insert(features_to_dedupe); @@ -36,7 +37,8 @@ impl DedupeContext { if let Some(has_deps) = self.deps.get(&*deps) { *deps = Arc::clone(has_deps); } else { - if self.deps.len() > 16384 { // keeps peak memory low (must clear, remove is leaving tombstones) + if self.deps.len() > 16384 { + // keeps peak memory low (must clear, remove is leaving tombstones) self.deps.clear(); } self.deps.insert(Arc::clone(deps)); @@ -52,7 +54,10 @@ pub struct HashableHashMap { } impl Hash for HashableHashMap { - fn hash(&self, hasher: &mut H) where H: Hasher { + fn hash(&self, hasher: &mut H) + where + H: Hasher, + { hasher.write_u64(self.hash); } } diff --git a/src/dirs.rs b/src/dirs.rs index 27fb923a..3591d410 100644 --- a/src/dirs.rs +++ b/src/dirs.rs @@ -31,12 +31,30 @@ pub(crate) fn crate_prefix(accumulator: &mut String, crate_name: &str, separator 3 => { accumulator.push('3'); accumulator.push(separator); - accumulator.extend(crate_name.as_bytes().get(0..1)?.iter().map(|c| c.to_ascii_lowercase() as char)); + accumulator.extend( + crate_name + .as_bytes() + .get(0..1)? + .iter() + .map(|c| c.to_ascii_lowercase() as char), + ); } _ => { - accumulator.extend(crate_name.as_bytes().get(0..2)?.iter().map(|c| c.to_ascii_lowercase() as char)); + accumulator.extend( + crate_name + .as_bytes() + .get(0..2)? + .iter() + .map(|c| c.to_ascii_lowercase() as char), + ); accumulator.push(separator); - accumulator.extend(crate_name.as_bytes().get(2..4)?.iter().map(|c| c.to_ascii_lowercase() as char)); + accumulator.extend( + crate_name + .as_bytes() + .get(2..4)? + .iter() + .map(|c| c.to_ascii_lowercase() as char), + ); } }; Some(()) @@ -52,7 +70,6 @@ pub(crate) fn crate_name_to_relative_path(crate_name: &str, separator: Option Result<(String, String), Error> { @@ -184,39 +201,26 @@ mod test { // that one includes a secret key as part of the url which would allow // anyone to publish to the registry, so uhh...here's a fake one instead assert_eq!( - super::url_to_local_dir( - "https://dl.cloudsmith.io/aBcW1234aBcW1234/embark/rust/cargo/index.git" - ) - .unwrap(), + super::url_to_local_dir("https://dl.cloudsmith.io/aBcW1234aBcW1234/embark/rust/cargo/index.git").unwrap(), ( "dl.cloudsmith.io-ff79e51ddd2b38fd".to_owned(), "https://dl.cloudsmith.io/aBcW1234aBcW1234/embark/rust/cargo/index.git".to_owned() ) ); } - + #[test] fn git_url_matches_cargo() { assert_eq!( crate::dirs::url_to_local_dir(INDEX_GIT_URL).unwrap(), - ( - "github.com-1ecc6299db9ec823".to_owned(), - INDEX_GIT_URL.to_owned() - ) + ("github.com-1ecc6299db9ec823".to_owned(), INDEX_GIT_URL.to_owned()) ); // Ensure we actually strip off the irrelevant parts of a url, note that // the .git suffix is not part of the canonical url, but *is* used when hashing assert_eq!( - crate::dirs::url_to_local_dir(&format!( - "registry+{}.git?one=1&two=2#fragment", - INDEX_GIT_URL - )) - .unwrap(), - ( - "github.com-c786010fb7ef2e6e".to_owned(), - INDEX_GIT_URL.to_owned() - ) + crate::dirs::url_to_local_dir(&format!("registry+{}.git?one=1&two=2#fragment", INDEX_GIT_URL)).unwrap(), + ("github.com-c786010fb7ef2e6e".to_owned(), INDEX_GIT_URL.to_owned()) ); } } diff --git a/src/error.rs b/src/error.rs index b1e0d8f3..dea1ea24 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,5 +1,5 @@ pub use serde_json::Error as SerdeJsonError; -use std::{io}; +use std::io; use std::path::PathBuf; pub use toml::de::Error as TomlDeError; @@ -12,7 +12,7 @@ pub enum Error { Git(#[from] GixError), #[error("{0}")] Url(String), - #[error("Could not obtain the most recent head commit in repo at {}. Tried {}, had {} available", repo_path.display(), refs_tried.join(", "), refs_available.join(", "))] + #[error("Could not obtain the most recent head commit in repo at {}. Tried {}, had {} available", repo_path.display(), refs_tried.join(", "), refs_available.join(", "))] MissingHead { /// The references we tried to get commits for. refs_tried: &'static [&'static str], @@ -49,9 +49,7 @@ pub enum GixError { #[error(transparent)] IntoObjectKind(#[from] gix::object::try_into::Error), #[error("The '{}' file is missing at the root of the tree of the crates index", path.display())] - PathMissing { - path: std::path::PathBuf - }, + PathMissing { path: std::path::PathBuf }, #[error(transparent)] LockAcquire(#[from] gix::lock::acquire::Error), #[error(transparent)] diff --git a/src/lib.rs b/src/lib.rs index afe264c8..a6b7c5da 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -52,7 +52,7 @@ //! # Ok::<_, crates_index::Error>(()) //! ``` //! -//! ### Getting most recently published or yanked crates +//! ### Getting most recently published or yanked crates //! //! ```rust //! # { @@ -66,40 +66,40 @@ //! # } //! # Ok::<_, crates_index::Error>(()) //! ``` -//! +//! //! ## Auto-cloning and parallelism -//! -//! When using any means of instantiating the [`Index`] type, we will +//! +//! When using any means of instantiating the [`Index`] type, we will //! clone the default crates index (or the given one) if it no git //! repository is present at the destination path. -//! -//! In order to protect from parallel operations of this kind, a +//! +//! In order to protect from parallel operations of this kind, a //! file-based lock is used. When interrupting the program with `Ctrl + C`, //! by default the program will be aborted which won't run destructors. //! This will cause the file lock to be stranded, causing all future operations //! to fail. -//! +//! //! To prevent this issue, the application must integrate with the //! [`gix-tempfile` signal handler](https://docs.rs/gix-tempfile/latest/gix_tempfile/#initial-setup), //! which allows locks to be deleted when typical signals are received. //! //! ## Git Repository Performance //! -//! By default, `gix` is compiled with `max-performance-safe`, which maximizes support for compilation environments but which +//! By default, `gix` is compiled with `max-performance-safe`, which maximizes support for compilation environments but which //! may be slower as it uses a pure-Rust Zlib implementation. //! To get best possible performance, use the `git-index-performance` feature toggle. -//! +//! //! ## Using `rustls` instead of `openssl` when using the `https` feature in applications -//! +//! //! When using the `https` feature, a choice will be made for you that involves selecting the `curl` backend for making //! the `https` protocol available. As using a different backend isn't additive, as cargo features should be, one will have //! to resort to the following. -//! +//! //! * Change the `crates-index` dependency to not use any default features with `default-features = false`, and turn on //! `features = ["git-index", …(everything else *but* "https")]` //! * Add the `gix` dependency with `default-features = false` and `features = ["blocking-http-transport-reqwest-rust-tls"]`. //! Consider renaming the crate to `gix-for-configuration-only = { package = "gix", … }` to make the intend clear. -//! +//! //! Please note that this should only be done in application manifests, who have the final say over the protocol and backend choices. #![forbid(unsafe_code)] #![deny(missing_docs)] @@ -119,15 +119,13 @@ mod dirs; /// Re-exports in case you want to inspect specific error details pub mod error; #[doc(hidden)] -pub use error::{Error, CratesIterError}; +pub use error::{CratesIterError, Error}; mod sparse_index; pub use sparse_index::{Index as SparseIndex, CRATES_IO_HTTP_INDEX}; - mod types; -pub use types::{Crate, Version, Dependency, DependencyKind}; - +pub use types::{Crate, Dependency, DependencyKind, Version}; pub(crate) fn split(haystack: &[u8], needle: u8) -> impl Iterator + '_ { struct Split<'a> { diff --git a/src/sparse_index.rs b/src/sparse_index.rs index 5eda1b6d..b5a475b1 100644 --- a/src/sparse_index.rs +++ b/src/sparse_index.rs @@ -1,8 +1,8 @@ use std::io; use std::path::{Path, PathBuf}; -use crate::{Crate, dirs::get_index_details, Error, IndexConfig, path_max_byte_len}; use crate::dirs::crate_name_to_relative_path; +use crate::{dirs::get_index_details, path_max_byte_len, Crate, Error, IndexConfig}; /// The default URL of the crates.io HTTP index, see [`Index::from_url`] and [`Index::new_cargo_default`] pub const CRATES_IO_HTTP_INDEX: &str = "sparse+https://index.crates.io/"; @@ -73,7 +73,8 @@ impl Index { /// Reads a crate from the local cache of the index. There are no guarantees around freshness, /// and if the crate is not known in the cache, no fetch will be performed. pub fn crate_from_cache(&self, name: &str) -> Result { - let cache_path = self.cache_path(name) + let cache_path = self + .cache_path(name) .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "bad name"))?; let cache_bytes = std::fs::read(cache_path)?; @@ -113,7 +114,7 @@ impl Index { } /// Reads the version of the cache entry for the specified crate, if it exists - /// + /// /// The version is of the form `key:value`, where, currently, the key is either /// `etag` or `last-modified` #[cfg(feature = "sparse-http")] @@ -137,28 +138,28 @@ impl Index { } let rest = &rest[4..]; - let version = crate::split(rest, 0).next().and_then(|version| { - std::str::from_utf8(version).ok().map(String::from) - }); + let version = crate::split(rest, 0) + .next() + .and_then(|version| std::str::from_utf8(version).ok().map(String::from)); version } /// Creates an HTTP request that can be sent via your HTTP client of choice /// to retrieve the current metadata for the specified crate - /// + /// /// See [`Self::parse_cache_response`] processing the response from the remote /// index - /// + /// /// It is highly recommended to assume HTTP/2 when making requests to remote /// indices, at least crates.io #[cfg(feature = "sparse-http")] pub fn make_cache_request(&self, name: &str) -> Result { use http::header; - let url = self.crate_url(name).ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "crate name is invalid") - })?; + let url = self + .crate_url(name) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "crate name is invalid"))?; let cache_version = self.read_cache_version(name); @@ -169,15 +170,9 @@ impl Index { // AFAICT this does not affect responses at the moment, but could in the future // if there are changes - headers.insert( - "cargo-protocol", - header::HeaderValue::from_static("version=1"), - ); + headers.insert("cargo-protocol", header::HeaderValue::from_static("version=1")); // All index entries are just files with lines of JSON - headers.insert( - header::ACCEPT, - header::HeaderValue::from_static("text/plain"), - ); + headers.insert(header::ACCEPT, header::HeaderValue::from_static("text/plain")); // We need to accept both identity and gzip, as otherwise cloudfront will // always respond to requests with strong etag's, which will differ from // cache entries generated by cargo @@ -212,19 +207,24 @@ impl Index { } /// Process the response to a request created by [`Self::make_cache_request`] - /// + /// /// This handles both the scenario where the local cache is missing the specified /// crate, or it is out of date, as well as the local entry being up to date /// and can just be read from disk - /// + /// /// You may specify whether an updated index entry is written locally to the /// cache or not - /// + /// /// Note that responses from sparse HTTP indices, at least crates.io, may /// send responses with `gzip` compression, it is your responsibility to /// decompress it before sending to this function #[cfg(feature = "sparse-http")] - pub fn parse_cache_response(&self, name: &str, response: http::Response>, write_cache_entry: bool) -> Result, Error> { + pub fn parse_cache_response( + &self, + name: &str, + response: http::Response>, + write_cache_entry: bool, + ) -> Result, Error> { use http::{header, StatusCode}; let (parts, body) = response.into_parts(); @@ -260,20 +260,20 @@ impl Index { } // The local cache entry is up to date with the latest entry on the // server, we can just return the local one - StatusCode::NOT_MODIFIED => { - self.crate_from_cache(name).map(Option::Some) - } + StatusCode::NOT_MODIFIED => self.crate_from_cache(name).map(Option::Some), // The server requires authorization but the user didn't provide it StatusCode::UNAUTHORIZED => { Err(io::Error::new(io::ErrorKind::PermissionDenied, "the request was not authorized").into()) } // The crate does not exist, or has been removed - StatusCode::NOT_FOUND | StatusCode::GONE | StatusCode::UNAVAILABLE_FOR_LEGAL_REASONS => { - Ok(None) - } - other => { - Err(io::Error::new(io::ErrorKind::Unsupported, format!("the server responded with status code '{other}', which is not supported in the current protocol")).into()) - } + StatusCode::NOT_FOUND | StatusCode::GONE | StatusCode::UNAVAILABLE_FOR_LEGAL_REASONS => Ok(None), + other => Err(io::Error::new( + io::ErrorKind::Unsupported, + format!( + "the server responded with status code '{other}', which is not supported in the current protocol" + ), + ) + .into()), } } } @@ -281,15 +281,17 @@ impl Index { #[cfg(test)] #[cfg(feature = "sparse-http")] mod tests { - use http::header; use crate::SparseIndex; - + use http::header; + #[inline] fn crates_io() -> SparseIndex { SparseIndex::with_path( - std::path::Path::new(&std::env::var_os("CARGO_MANIFEST_DIR").unwrap()).join("tests/fixtures/sparse_registry_cache/cargo_home"), - crate::CRATES_IO_HTTP_INDEX - ).unwrap() + std::path::Path::new(&std::env::var_os("CARGO_MANIFEST_DIR").unwrap()) + .join("tests/fixtures/sparse_registry_cache/cargo_home"), + crate::CRATES_IO_HTTP_INDEX, + ) + .unwrap() } // curl -v -H 'accept-encoding: gzip,identity' https://index.crates.io/cr/at/crates-index @@ -309,9 +311,13 @@ mod tests { let response = http::Response::builder() .status(http::StatusCode::OK) .header(header::ETAG, "W/\"7fbfc422231ec53a9283f2eb2fb4f459\"") - .body(CRATES_INDEX_INDEX_ENTRY.to_vec()).unwrap(); + .body(CRATES_INDEX_INDEX_ENTRY.to_vec()) + .unwrap(); - let http_krate = index.parse_cache_response("crates-index", response, true /* write cache entry */).unwrap().unwrap(); + let http_krate = index + .parse_cache_response("crates-index", response, true /* write cache entry */) + .unwrap() + .unwrap(); assert!(cache_path.is_file(), "the cache entry was indeed written"); let cache_krate = index.crate_from_cache("crates-index").unwrap(); @@ -319,4 +325,4 @@ mod tests { assert_eq!(http.version(), cache.version()); } } -} \ No newline at end of file +} diff --git a/src/types.rs b/src/types.rs index 5424e5a3..287e1272 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,13 +1,13 @@ use crate::dedupe::DedupeContext; -use std::sync::Arc; +use crate::IndexConfig; use semver::Version as SemverVersion; use serde_derive::{Deserialize, Serialize}; use smol_str::SmolStr; use std::collections::HashMap; use std::io; use std::path::Path; -use crate::IndexConfig; +use std::sync::Arc; /// A single version of a crate (package) published to the index #[derive(Serialize, Deserialize, Clone, Debug)] @@ -75,7 +75,7 @@ impl Version { /// combines features and features2 /// /// dedupes dependencies and features - fn build_data(&mut self, dedupe: &mut DedupeContext){ + fn build_data(&mut self, dedupe: &mut DedupeContext) { if let Some(features2) = self.features2.take() { if let Some(f1) = Arc::get_mut(&mut self.features) { for (key, mut val) in features2.into_iter() { @@ -266,8 +266,8 @@ impl Crate { let num_versions = bytes.split(is_newline).count(); let mut versions = Vec::with_capacity(num_versions); for line in bytes.split(is_newline) { - let mut version: Version = serde_json::from_slice(line) - .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + let mut version: Version = + serde_json::from_slice(line).map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; version.build_data(dedupe); @@ -303,8 +303,10 @@ impl Crate { let index_v_bytes = rest.get(..4).ok_or(io::ErrorKind::UnexpectedEof)?; let index_v = u32::from_le_bytes(index_v_bytes.try_into().unwrap()); if index_v != CURRENT_INDEX_FORMAT_VERSION { - return Err(io::Error::new(io::ErrorKind::Unsupported, - format!("wrong index format version: {index_v} (expected {CURRENT_INDEX_FORMAT_VERSION}))"))); + return Err(io::Error::new( + io::ErrorKind::Unsupported, + format!("wrong index format version: {index_v} (expected {CURRENT_INDEX_FORMAT_VERSION}))"), + )); } rest = &rest[4..]; } @@ -347,9 +349,7 @@ impl Crate { Self::from_version_entries_iter(iter) } - pub(crate) fn from_version_entries_iter<'a, I: Iterator + 'a>( - mut iter: I, - ) -> io::Result { + pub(crate) fn from_version_entries_iter<'a, I: Iterator + 'a>(mut iter: I) -> io::Result { let mut versions = Vec::new(); let mut dedupe = DedupeContext::new(); @@ -357,8 +357,8 @@ impl Crate { // Each entry is a tuple of (semver, version_json) while let Some(_version) = iter.next() { let version_slice = iter.next().ok_or(io::ErrorKind::UnexpectedEof)?; - let mut version: Version = serde_json::from_slice(version_slice) - .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + let mut version: Version = + serde_json::from_slice(version_slice).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; version.build_data(&mut dedupe); diff --git a/tests/bare_index/mod.rs b/tests/bare_index/mod.rs index 3676a867..a3cf0c30 100644 --- a/tests/bare_index/mod.rs +++ b/tests/bare_index/mod.rs @@ -1,8 +1,8 @@ #[cfg(feature = "https")] -pub(crate) mod with_https { - use std::time::SystemTime; +pub(crate) mod with_https { use crates_index::{Index, INDEX_GIT_URL}; - + use std::time::SystemTime; + #[test] fn changes() { let index = shared_index(); @@ -49,13 +49,10 @@ pub(crate) mod with_https { let tmp_dir = tempfile::TempDir::new().unwrap(); let path = tmp_dir.path().join("some/sub/dir/testing/abc"); - let mut repo = - Index::with_path(path, INDEX_GIT_URL).expect("Failed to clone crates.io index"); + let mut repo = Index::with_path(path, INDEX_GIT_URL).expect("Failed to clone crates.io index"); fn test_sval(repo: &Index) { - let krate = repo - .crate_("sval") - .expect("Could not find the crate sval in the index"); + let krate = repo.crate_("sval").expect("Could not find the crate sval in the index"); let version = krate .versions() @@ -67,10 +64,7 @@ pub(crate) mod with_https { .iter() .find(|d| d.name() == "serde_lib") .expect("sval does not have expected dependency?"); - assert_ne!( - dep_with_package_name.name(), - dep_with_package_name.package().unwrap() - ); + assert_ne!(dep_with_package_name.name(), dep_with_package_name.package().unwrap()); assert_eq!( dep_with_package_name.crate_name(), dep_with_package_name.package().unwrap() @@ -89,9 +83,7 @@ pub(crate) mod with_https { fn opens_bare_index_and_can_update_it() { let mut repo = shared_index(); fn test_sval(repo: &Index) { - let krate = repo - .crate_("sval") - .expect("Could not find the crate sval in the index"); + let krate = repo.crate_("sval").expect("Could not find the crate sval in the index"); let version = krate .versions() @@ -103,10 +95,7 @@ pub(crate) mod with_https { .iter() .find(|d| d.name() == "serde_lib") .expect("sval does not have expected dependency?"); - assert_ne!( - dep_with_package_name.name(), - dep_with_package_name.package().unwrap() - ); + assert_ne!(dep_with_package_name.name(), dep_with_package_name.package().unwrap()); assert_eq!( dep_with_package_name.crate_name(), dep_with_package_name.package().unwrap() @@ -147,10 +136,7 @@ pub(crate) mod with_https { .iter() .find(|d| d.name() == "serde_lib") .expect("sval does not have expected dependency?"); - assert_ne!( - dep_with_package_name.name(), - dep_with_package_name.package().unwrap() - ); + assert_ne!(dep_with_package_name.name(), dep_with_package_name.package().unwrap()); assert_eq!( dep_with_package_name.crate_name(), dep_with_package_name.package().unwrap() @@ -163,13 +149,7 @@ pub(crate) mod with_https { let mut index = shared_index(); index .update() - .map_err(|e| { - format!( - "could not fetch cargo's index in {}: {}", - index.path().display(), - e - ) - }) + .map_err(|e| format!("could not fetch cargo's index in {}: {}", index.path().display(), e)) .unwrap(); assert!(index.crate_("crates-index").is_some()); assert!(index.crate_("toml").is_some()); @@ -182,8 +162,7 @@ pub(crate) mod with_https { pub(crate) fn shared_index() -> Index { let index_path = "tests/fixtures/git-registry"; if is_ci::cached() { - Index::new_cargo_default() - .expect("CI has just cloned this index and its ours and valid") + Index::new_cargo_default().expect("CI has just cloned this index and its ours and valid") } else { Index::with_path(index_path, INDEX_GIT_URL).expect("clone works and there is no racing") } diff --git a/tests/mem.rs b/tests/mem.rs index e83e714b..f042e551 100644 --- a/tests/mem.rs +++ b/tests/mem.rs @@ -1,9 +1,9 @@ #[cfg(all(feature = "parallel", feature = "git-index"))] mod mem { + use bytesize::ByteSize; use cap::Cap; use std::alloc; use std::time::Instant; - use bytesize::ByteSize; #[global_allocator] static ALLOCATOR: Cap = Cap::new(alloc::System, usize::max_value()); @@ -33,6 +33,9 @@ mod mem { total = ByteSize(ALLOCATOR.total_allocated() as u64), peak = ByteSize(ALLOCATOR.max_allocated() as u64), ); - assert!(per_crate < 6300, "per crate limit {per_crate}B should remain below memory limit"); + assert!( + per_crate < 6300, + "per crate limit {per_crate}B should remain below memory limit" + ); } } diff --git a/tests/sparse_index/mod.rs b/tests/sparse_index/mod.rs index d2ac6cdc..1f18cf0a 100644 --- a/tests/sparse_index/mod.rs +++ b/tests/sparse_index/mod.rs @@ -1,9 +1,11 @@ #[test] fn crate_from_cache() { let index = crates_index::SparseIndex::with_path( - std::path::Path::new(&std::env::var_os("CARGO_MANIFEST_DIR").unwrap()).join("tests/fixtures/sparse_registry_cache/cargo_home"), - crates_index::CRATES_IO_HTTP_INDEX - ).unwrap(); + std::path::Path::new(&std::env::var_os("CARGO_MANIFEST_DIR").unwrap()) + .join("tests/fixtures/sparse_registry_cache/cargo_home"), + crates_index::CRATES_IO_HTTP_INDEX, + ) + .unwrap(); let crate_ = index.crate_from_cache("autocfg").unwrap(); @@ -20,14 +22,16 @@ mod with_sparse_http_feature { #[inline] fn crates_io() -> SparseIndex { SparseIndex::with_path( - std::path::Path::new(&std::env::var_os("CARGO_MANIFEST_DIR").unwrap()).join("tests/fixtures/sparse_registry_cache/cargo_home"), - crates_index::CRATES_IO_HTTP_INDEX - ).unwrap() + std::path::Path::new(&std::env::var_os("CARGO_MANIFEST_DIR").unwrap()) + .join("tests/fixtures/sparse_registry_cache/cargo_home"), + crates_index::CRATES_IO_HTTP_INDEX, + ) + .unwrap() } mod make_cache_request { - use http::{header, Request}; use crate::sparse_index::with_sparse_http_feature::crates_io; + use http::{header, Request}; // Validates that a valid request is generated when there is no cache entry // for a crate @@ -41,7 +45,12 @@ mod with_sparse_http_feature { assert!(req.headers().get(header::IF_NONE_MATCH).is_none()); assert!(req.headers().get(header::IF_MODIFIED_SINCE).is_none()); assert_eq!(req.headers().get(header::ACCEPT_ENCODING).unwrap(), "gzip,identity"); - assert_eq!(req.headers().get(header::HeaderName::from_static("cargo-protocol")).unwrap(), "version=1"); + assert_eq!( + req.headers() + .get(header::HeaderName::from_static("cargo-protocol")) + .unwrap(), + "version=1" + ); assert_eq!(req.headers().get(header::ACCEPT).unwrap(), "text/plain"); } @@ -54,14 +63,17 @@ mod with_sparse_http_feature { let req: Request> = builder.body(vec![]).unwrap(); assert_eq!(req.uri(), format!("{}au/to/autocfg", index.url()).as_str()); - assert_eq!(req.headers().get(header::IF_NONE_MATCH).unwrap(), "W/\"aa975a09419f9c8f61762a3d06fdb67d\""); + assert_eq!( + req.headers().get(header::IF_NONE_MATCH).unwrap(), + "W/\"aa975a09419f9c8f61762a3d06fdb67d\"" + ); assert!(req.headers().get(header::IF_MODIFIED_SINCE).is_none()); } } - + mod parse_cache_response { - use http::header; use crate::sparse_index::with_sparse_http_feature::crates_io; + use http::header; // curl -v -H 'accept-encoding: gzip,identity' -H 'if-none-match: W/"aa975a09419f9c8f61762a3d06fdb67d"' https://index.crates.io/au/to/autocfg // as of 2023-06-15 @@ -74,7 +86,8 @@ mod with_sparse_http_feature { let response = http::Response::builder() .status(http::StatusCode::OK) .header(header::ETAG, "W/\"5f15de4a723e10b3f9eaf048d693cccc\"") - .body(AUTOCFG_INDEX_ENTRY.to_vec()).unwrap(); + .body(AUTOCFG_INDEX_ENTRY.to_vec()) + .unwrap(); let krate = index.parse_cache_response("autocfg", response, false).unwrap().unwrap(); assert_eq!(krate.highest_version().version(), "1.1.0"); @@ -88,7 +101,8 @@ mod with_sparse_http_feature { let response = http::Response::builder() .status(http::StatusCode::NOT_MODIFIED) .header(header::ETAG, "W/\"5f15de4a723e10b3f9eaf048d693cccc\"") - .body(Vec::new()).unwrap(); + .body(Vec::new()) + .unwrap(); let krate = index.parse_cache_response("autocfg", response, false).unwrap().unwrap(); assert_eq!(krate.name(), "autocfg"); @@ -104,7 +118,8 @@ mod with_sparse_http_feature { let index = crates_io(); let response = http::Response::builder() .status(http::StatusCode::NOT_FOUND) - .body(Vec::new()).unwrap(); + .body(Vec::new()) + .unwrap(); assert!(index.parse_cache_response("serde", response, false).unwrap().is_none()); }