Implement a local registry type

This flavor of registry is intended to behave very similarly to the standard
remote registry, except everything is contained locally on the filesystem
instead. There are a few components to this new flavor of registry:

1. The registry itself is rooted at a particular directory, owning all structure
   beneath it.
2. There is an `index` folder with the same structure as the crates.io index
   describing the local registry (e.g. contents, versions, checksums, etc).
3. Inside the root will also be a list of `.crate` files which correspond to
   those described in the index. All crates must be of the form
   `name-version.crate` and be the same `.crate` files from crates.io itself.

This support can currently be used via the previous implementation of source
overrides with the new type:

```toml
[source.crates-io]
replace-with = 'my-awesome-registry'

[source.my-awesome-registry]
local-registry = 'path/to/registry'
```

I will soon follow up with a tool which can be used to manage these local
registries externally.

Author: alexcrichton

src/cargo/core/resolver/mod.rs

@@ -217,7 +217,6 @@ struct Context {
     activations: HashMap<(String, SourceId), Vec<Rc<Summary>>>,
     resolve_graph: Graph<PackageId>,
     resolve_features: HashMap<PackageId, HashSet<String>>,
-    visited: HashSet<PackageId>,
 }
 
 /// Builds the list of all packages required to build the first argument.
@@ -233,6 +232,7 @@ pub fn resolve(summary: &Summary, method: &Method, registry: &mut Registry)
     let _p = profile::start(format!("resolving: {}", summary.package_id()));
     let cx = try!(activate_deps_loop(cx, registry, Rc::new(summary.clone()),
                                      method));
+    try!(check_cycles(&cx, summary.package_id()));
 
     let mut resolve = Resolve {
         graph: cx.resolve_graph,
@@ -247,8 +247,6 @@ pub fn resolve(summary: &Summary, method: &Method, registry: &mut Registry)
         resolve.checksums.insert(summary.package_id().clone(), cksum);
     }
 
-    try!(check_cycles(&cx));
-
     trace!("resolved: {:?}", resolve);
     Ok(resolve)
 }
@@ -841,18 +839,18 @@ impl Context {
     }
 }
 
-fn check_cycles(cx: &Context) -> CargoResult<()> {
+fn check_cycles(cx: &Context, root: &PackageId) -> CargoResult<()> {
     let mut summaries = HashMap::new();
     for summary in cx.activations.values().flat_map(|v| v) {
         summaries.insert(summary.package_id(), &**summary);
     }
-    return visit(&cx.resolve,
-                 cx.resolve.root(),
+    return visit(&cx.resolve_graph,
+                 root,
                  &summaries,
                  &mut HashSet::new(),
                  &mut HashSet::new());
 
-    fn visit<'a>(resolve: &'a Resolve,
+    fn visit<'a>(resolve: &'a Graph<PackageId>,
                  id: &'a PackageId,
                  summaries: &HashMap<&'a PackageId, &Summary>,
                  visited: &mut HashSet<&'a PackageId>,
@@ -873,7 +871,7 @@ fn check_cycles(cx: &Context) -> CargoResult<()> {
         // dependencies.
         if checked.insert(id) {
             let summary = summaries[id];
-            for dep in resolve.deps(id).into_iter().flat_map(|a| a) {
+            for dep in resolve.edges(id).into_iter().flat_map(|a| a) {
                 let is_transitive = summary.dependencies().iter().any(|d| {
                     d.matches_id(dep) && d.is_transitive()
                 });

src/cargo/core/source.rs

@@ -63,6 +63,8 @@ enum Kind {
     Path,
     /// represents the central registry
     Registry,
+    /// represents a local filesystem-based registry
+    LocalRegistry,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
@@ -107,7 +109,7 @@ impl SourceId {
     /// use cargo::core::SourceId;
     /// SourceId::from_url("git+https://github.com/alexcrichton/\
     ///                     libssh2-static-sys#80e71a3021618eb05\
-    ///                     656c58fb7c5ef5f12bc747f".to_string());
+    ///                     656c58fb7c5ef5f12bc747f");
     /// ```
     pub fn from_url(string: &str) -> CargoResult<SourceId> {
         let mut parts = string.splitn(2, '+');
@@ -169,6 +171,9 @@ impl SourceId {
             SourceIdInner { kind: Kind::Registry, ref url, .. } => {
                 format!("registry+{}", url)
             }
+            SourceIdInner { kind: Kind::LocalRegistry, ref url, .. } => {
+                format!("local-registry+{}", url)
+            }
         }
     }
 
@@ -186,6 +191,11 @@ impl SourceId {
         SourceId::new(Kind::Registry, url.clone())
     }
 
+    pub fn for_local_registry(path: &Path) -> CargoResult<SourceId> {
+        let url = try!(path.to_url());
+        Ok(SourceId::new(Kind::LocalRegistry, url))
+    }
+
     /// Returns the `SourceId` corresponding to the main repository.
     ///
     /// This is the main cargo registry by default, but it can be overridden in
@@ -210,7 +220,9 @@ impl SourceId {
 
     pub fn url(&self) -> &Url { &self.inner.url }
     pub fn is_path(&self) -> bool { self.inner.kind == Kind::Path }
-    pub fn is_registry(&self) -> bool { self.inner.kind == Kind::Registry }
+    pub fn is_registry(&self) -> bool {
+        self.inner.kind == Kind::Registry || self.inner.kind == Kind::LocalRegistry
+    }
 
     pub fn is_git(&self) -> bool {
         match self.inner.kind {
@@ -232,6 +244,13 @@ impl SourceId {
                 Box::new(PathSource::new(&path, self, config))
             }
             Kind::Registry => Box::new(RegistrySource::remote(self, config)),
+            Kind::LocalRegistry => {
+                let path = match self.inner.url.to_file_path() {
+                    Ok(p) => p,
+                    Err(()) => panic!("path sources cannot be remote"),
+                };
+                Box::new(RegistrySource::local(self, &path, config))
+            }
         }
     }
 
@@ -317,7 +336,8 @@ impl fmt::Display for SourceId {
                 }
                 Ok(())
             }
-            SourceIdInner { kind: Kind::Registry, ref url, .. } => {
+            SourceIdInner { kind: Kind::Registry, ref url, .. } |
+            SourceIdInner { kind: Kind::LocalRegistry, ref url, .. } => {
                 write!(f, "registry {}", url)
             }
         }

src/cargo/sources/config.rs

@@ -128,11 +128,21 @@ a lock file compatible with `{orig}` cannot be generated in this situation
             let url = try!(url(val, &format!("source.{}.registry", name)));
             srcs.push(SourceId::for_registry(&url));
         }
+        if let Some(val) = table.get("local-registry") {
+            let (s, path) = try!(val.string(&format!("source.{}.local-registry",
+                                                     name)));
+            let mut path = path.to_path_buf();
+            path.pop();
+            path.pop();
+            path.push(s);
+            srcs.push(try!(SourceId::for_local_registry(&path)));
+        }
 
         let mut srcs = srcs.into_iter();
         let src = try!(srcs.next().chain_error(|| {
-            human(format!("no source URL specified for `source.{}`, needs \
-                           `registry` defined", name))
+            human(format!("no source URL specified for `source.{}`, need \
+                           either `registry` or `local-registry` defined",
+                          name))
         }));
         if srcs.next().is_some() {
             return Err(human(format!("more than one source URL specified for \

src/cargo/sources/registry/local.rs

@@ -0,0 +1,89 @@
+use std::fs::{self, File};
+use std::io::prelude::*;
+use std::path::{PathBuf, Path};
+
+use rustc_serialize::hex::ToHex;
+
+use core::PackageId;
+use sources::registry::{RegistryData, RegistryConfig};
+use util::{Config, CargoResult, ChainError, human, Sha256};
+
+pub struct LocalRegistry<'cfg> {
+    index_path: PathBuf,
+    root: PathBuf,
+    src_path: PathBuf,
+    config: &'cfg Config,
+}
+
+impl<'cfg> LocalRegistry<'cfg> {
+    pub fn new(root: &Path,
+               config: &'cfg Config,
+               name: &str) -> LocalRegistry<'cfg> {
+        LocalRegistry {
+            src_path: config.registry_source_path().join(name),
+            index_path: root.join("index"),
+            root: root.to_path_buf(),
+            config: config,
+        }
+    }
+}
+
+impl<'cfg> RegistryData for LocalRegistry<'cfg> {
+    fn index_path(&self) -> &Path {
+        &self.index_path
+    }
+
+    fn config(&self) -> CargoResult<Option<RegistryConfig>> {
+        // Local registries don't have configuration for remote APIs or anything
+        // like that
+        Ok(None)
+    }
+
+    fn update_index(&mut self) -> CargoResult<()> {
+        // Nothing to update, we just use what's on disk. Verify it actually
+        // exists though
+        if !self.root.is_dir() {
+            bail!("local registry path is not a directory: {}",
+                  self.root.display())
+        }
+        if !self.index_path.is_dir() {
+            bail!("local registry index path is not a directory: {}",
+                  self.index_path.display())
+        }
+        Ok(())
+    }
+
+    fn download(&mut self, pkg: &PackageId, checksum: &str)
+                -> CargoResult<PathBuf> {
+        let crate_file = format!("{}-{}.crate", pkg.name(), pkg.version());
+        let crate_file = self.root.join(&crate_file);
+
+        // If we've already got an unpacked version of this crate, then skip the
+        // checksum below as it is in theory already verified.
+        let dst = format!("{}-{}", pkg.name(), pkg.version());
+        let dst = self.src_path.join(&dst);
+        if fs::metadata(&dst).is_ok() {
+            return Ok(crate_file)
+        }
+
+        try!(self.config.shell().status("Unpacking", pkg));
+
+        // We don't actually need to download anything per-se, we just need to
+        // verify the checksum matches the .crate file itself.
+        let mut file = try!(File::open(&crate_file).chain_error(|| {
+            human(format!("failed to read `{}` for `{}`", crate_file.display(),
+                          pkg))
+        }));
+        let mut data = Vec::new();
+        try!(file.read_to_end(&mut data).chain_error(|| {
+            human(format!("failed to read `{}`", crate_file.display()))
+        }));
+        let mut state = Sha256::new();
+        state.update(&data);
+        if state.finish().to_hex() != checksum {
+            bail!("failed to verify the checksum of `{}`", pkg)
+        }
+
+        Ok(crate_file)
+    }
+}

src/cargo/sources/registry/mod.rs

@@ -226,6 +226,7 @@ pub trait RegistryData {
 
 mod index;
 mod remote;
+mod local;
 
 fn short_name(id: &SourceId) -> String {
     let hash = hex::short_hash(id);
@@ -241,6 +242,14 @@ impl<'cfg> RegistrySource<'cfg> {
         RegistrySource::new(source_id, config, &name, Box::new(ops))
     }
 
+    pub fn local(source_id: &SourceId,
+                 path: &Path,
+                 config: &'cfg Config) -> RegistrySource<'cfg> {
+        let name = short_name(source_id);
+        let ops = local::LocalRegistry::new(path, config, &name);
+        RegistrySource::new(source_id, config, &name, Box::new(ops))
+    }
+
     fn new(source_id: &SourceId,
            config: &'cfg Config,
            name: &str,

tests/support/mod.rs

@@ -666,3 +666,4 @@ pub static UPLOADING:   &'static str = "   Uploading";
 pub static VERIFYING:   &'static str = "   Verifying";
 pub static ARCHIVING:   &'static str = "   Archiving";
 pub static INSTALLING:  &'static str = "  Installing";
+pub static UNPACKING:   &'static str = "   Unpacking";

tests/support/registry.rs

@@ -24,6 +24,7 @@ pub struct Package {
     deps: Vec<(String, String, &'static str, String)>,
     files: Vec<(String, String)>,
     yanked: bool,
+    local: bool,
 }
 
 pub fn init() {
@@ -62,9 +63,15 @@ impl Package {
             deps: Vec::new(),
             files: Vec::new(),
             yanked: false,
+            local: false,
         }
     }
 
+    pub fn local(&mut self, local: bool) -> &mut Package {
+        self.local = local;
+        self
+    }
+
     pub fn file(&mut self, name: &str, contents: &str) -> &mut Package {
         self.files.push((name.to_string(), contents.to_string()));
         self
@@ -96,7 +103,6 @@ impl Package {
         self
     }
 
-    #[allow(deprecated)] // connect => join in 1.3
     pub fn publish(&self) {
         self.make_archive();
 
@@ -109,7 +115,7 @@ impl Package {
                        \"target\":{},\
                        \"optional\":false,\
                        \"kind\":\"{}\"}}", name, req, target, kind)
-        }).collect::<Vec<_>>().connect(",");
+        }).collect::<Vec<_>>().join(",");
         let cksum = {
             let mut c = Vec::new();
             File::open(&self.archive_dst()).unwrap()
@@ -128,28 +134,34 @@ impl Package {
         };
 
         // Write file/line in the index
-        let dst = registry_path().join(&file);
+        let dst = if self.local {
+            registry_path().join("index").join(&file)
+        } else {
+            registry_path().join(&file)
+        };
         let mut prev = String::new();
         let _ = File::open(&dst).and_then(|mut f| f.read_to_string(&mut prev));
         fs::create_dir_all(dst.parent().unwrap()).unwrap();
         File::create(&dst).unwrap()
              .write_all((prev + &line[..] + "\n").as_bytes()).unwrap();
 
         // Add the new file to the index
-        let repo = git2::Repository::open(&registry_path()).unwrap();
-        let mut index = repo.index().unwrap();
-        index.add_path(Path::new(&file)).unwrap();
-        index.write().unwrap();
-        let id = index.write_tree().unwrap();
-
-        // Commit this change
-        let tree = repo.find_tree(id).unwrap();
-        let sig = repo.signature().unwrap();
-        let parent = repo.refname_to_id("refs/heads/master").unwrap();
-        let parent = repo.find_commit(parent).unwrap();
-        repo.commit(Some("HEAD"), &sig, &sig,
-                    "Another commit", &tree,
-                    &[&parent]).unwrap();
+        if !self.local {
+            let repo = git2::Repository::open(&registry_path()).unwrap();
+            let mut index = repo.index().unwrap();
+            index.add_path(Path::new(&file)).unwrap();
+            index.write().unwrap();
+            let id = index.write_tree().unwrap();
+
+            // Commit this change
+            let tree = repo.find_tree(id).unwrap();
+            let sig = repo.signature().unwrap();
+            let parent = repo.refname_to_id("refs/heads/master").unwrap();
+            let parent = repo.find_commit(parent).unwrap();
+            repo.commit(Some("HEAD"), &sig, &sig,
+                        "Another commit", &tree,
+                        &[&parent]).unwrap();
+        }
     }
 
     fn make_archive(&self) {
@@ -199,7 +211,12 @@ impl Package {
     }
 
     pub fn archive_dst(&self) -> PathBuf {
-        dl_path().join(&self.name).join(&self.vers).join("download")
+        if self.local {
+            registry_path().join(format!("{}-{}.crate", self.name,
+                                         self.vers))
+        } else {
+            dl_path().join(&self.name).join(&self.vers).join("download")
+        }
     }
 }