diff --git a/Cargo.lock b/Cargo.lock index 75c6ee4..ea78be7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -241,6 +241,7 @@ name = "db" version = "0.1.0" dependencies = [ "dockerclient 0.1.0", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "walkdir 2.2.7 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/db/Cargo.lock b/db/Cargo.lock index 6472c2b..a2a8a98 100644 --- a/db/Cargo.lock +++ b/db/Cargo.lock @@ -138,6 +138,7 @@ name = "db" version = "0.1.0" dependencies = [ "dockerclient 0.1.0", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "walkdir 2.2.7 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/db/Cargo.toml b/db/Cargo.toml index 26f6709..685acdf 100644 --- a/db/Cargo.toml +++ b/db/Cargo.toml @@ -6,4 +6,5 @@ authors = ["root"] [dependencies] dockerclient = { path = "../dockerclient" } sha2 = "0.8.0" -walkdir = "2.2.7" \ No newline at end of file +walkdir = "2.2.7" +hex = "0.3.2" \ No newline at end of file diff --git a/db/src/containeritem.rs b/db/src/containeritem.rs new file mode 100644 index 0000000..a1b058b --- /dev/null +++ b/db/src/containeritem.rs @@ -0,0 +1,88 @@ +use crate::utils::get_or_run; +use hex::FromHex; +use std::io; +use std::path::Path; +#[derive(Debug)] +pub enum Error { + BadPath(String), + IOError(io::Error), + DefaultError(String), +} + +pub struct ContainerItem<'a, 'b> { + path: &'a Path, + id: String, + dockercli: &'b dockerclient::DockerClient, + image: String, +} + +impl<'a, 'b> ContainerItem<'a, 'b> { + pub fn new( + path: &'a Path, + dockercli: &'b dockerclient::DockerClient, + image: &str, + ) -> Result { + let mut filename = "".into(); + match path.file_name() { + None => { + return Err(Error::BadPath(path.display().to_string())); + } + Some(s) => filename = s.to_string_lossy().to_owned(), + } + let id = format!("{:?}", filename); + let image = image.into(); + Ok(ContainerItem { + path, + id, + dockercli, + image, + }) + } +} + +impl<'a, 'b> crate::Item for ContainerItem<'a, 'b> { + fn hash(&mut self) -> Vec { + let c = get_or_run(self.dockercli, &self.image).expect("get or run container fail"); + let (out, err) = self + .dockercli + .exec( + &c.id, + &format!( + "cat {}", + Path::new("/checksum") + .join(self.path.strip_prefix("/").unwrap()) + .display() + ), + ) + .map(|(out, err)| { + ( + (String::from_utf8_lossy(&out).to_owned().to_string()), + (String::from_utf8_lossy(&err).to_owned().to_string()), + ) + }) + .expect("get checksum file fail"); + + if err != "" { + panic!(format!( + "out:{}, err:{}, {}", + out, + err, + format!( + "cat {}", + Path::new("/checksum") + .join(self.path.strip_prefix("/").unwrap()) + .display() + ) + )); + } + Vec::from_hex(out.trim()).expect("from_hex") + } + + fn id(&self) -> &str { + &self.id + } + + fn srcpath(&self) -> &Path { + self.path + } +} diff --git a/db/src/dir_item.rs b/db/src/dir_item.rs deleted file mode 100644 index 2ec9510..0000000 --- a/db/src/dir_item.rs +++ /dev/null @@ -1,57 +0,0 @@ -use sha2::{Digest, Sha256}; -use std::fs::File; -use std::io; -use std::io::{copy, Seek, SeekFrom}; -use std::path::Path; -#[derive(Debug)] -pub enum Error { - BadPath(String), - IOError(io::Error), - DefaultError(String), -} - -pub struct DirItem<'a> { - path: &'a Path, - fs: Vec, - id: String, -} - -impl<'a> DirItem<'a> { - pub fn new(path: &'a Path) -> Result { - if !path.is_dir() || !path.exists() { - return Err(Error::BadPath(path.display().to_string())); - } - let mut fs = vec![]; - for e in walkdir::WalkDir::new(path) - .follow_links(true) - .sort_by(|a, b| a.file_name().cmp(b.file_name())) - { - let entry = e.map_err(|e| Error::DefaultError(e.to_string()))?; - if !entry.path().is_dir() { - let f = File::open(entry.path()).map_err(Error::IOError)?; - fs.push(f); - } - } - let id = format!("DIR: {:?}", path.file_name().unwrap()); - Ok(DirItem { path, fs, id }) - } -} - -impl<'a> crate::Item for DirItem<'a> { - fn hash(&mut self) -> Vec { - let mut hasher = Sha256::new(); - for f in &mut self.fs { - f.seek(SeekFrom::Start(0)).unwrap(); - copy(f, &mut hasher).unwrap(); - } - hasher.result().to_vec() - } - - fn id(&self) -> &str { - &self.id - } - - fn srcpath(&self) -> &Path { - self.path - } -} diff --git a/db/src/docker.rs b/db/src/docker.rs index 294ce6a..4c2d072 100644 --- a/db/src/docker.rs +++ b/db/src/docker.rs @@ -1,14 +1,16 @@ +use crate::utils::get_or_run; use crate::*; +use hex; use std::fmt; -use std::ops::DerefMut; use std::path::Path; #[derive(Debug)] pub enum Error { + NotExistItem(String), NotFoundEntry(String), DockerError(dockerclient::Error), ExecError(String), - FileItemError(file_item::Error), - DirItemError(dir_item::Error), + HostItemError(hostitem::Error), + ContainerItemError(containeritem::Error), } impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { @@ -54,13 +56,12 @@ impl DB for ImageDrive { } fn add(&self, entry: &str, itempath: &Path) -> Result { - let mut item: Box = if itempath.is_file() { - Box::new(file_item::FileItem::new(itempath).map_err(Error::FileItemError)?) - } else { - Box::new(dir_item::DirItem::new(itempath).map_err(Error::DirItemError)?) - }; + if !itempath.exists() { + return Err(Error::NotExistItem(format!("{}", itempath.display()))); + } + let mut item = hostitem::HostItem::new(itempath).map_err(Error::HostItemError)?; - let c = get_or_run(&self.dockercli, &self.image_name)?; + let c = get_or_run(&self.dockercli, &self.image_name).map_err(Error::DockerError)?; let path = Path::new("/data").join(entry); let ls_entry = format!( "mkdir -p {} && ls {}", @@ -83,55 +84,58 @@ impl DB for ImageDrive { let items: Vec<&str> = if out.trim() == "" { vec![] } else { - out.trim().split(" ").collect() + out.trim().split_whitespace().collect() }; let mut ps = vec![]; let mut fileitems = vec![]; - let mut diritems = vec![]; for item in items { - ps.push(Path::new("/data").join(item)); + ps.push(Path::new("/data").join(entry).join(item)); } for p in &ps { - if p.is_file() { - fileitems.push(file_item::FileItem::new(&p).map_err(Error::FileItemError)?); - } else { - diritems.push(dir_item::DirItem::new(&p).map_err(Error::DirItemError)?); - }; + fileitems.push( + containeritem::ContainerItem::new(&p, &self.dockercli, &self.image_name) + .map_err(Error::ContainerItemError)?, + ); } for mut fi in fileitems { - if compare_items(&mut fi, item.deref_mut()) { + if compare_items(&mut fi, &mut item) { return Ok(AddResult::ExistedItem(fi.id().into())); } } - for mut di in diritems { - if compare_items(&mut di, item.deref_mut()) { - return Ok(AddResult::ExistedItem(di.id().into())); - } - } let dstpath = Path::new("/data").join(entry).join(item.id()); self.dockercli .copy_in(&c.id, item.srcpath(), &dstpath) .map_err(Error::DockerError)?; + + self.dockercli + .exec( + &c.id, + &format!( + "mkdir -p {} && echo {} > {}", + Path::new("/checksum") + .join(dstpath.parent().unwrap().strip_prefix("/").unwrap()) + .display(), + hex::encode(item.hash()), + Path::new("/checksum") + .join(dstpath.strip_prefix("/").unwrap()) + .display() + ), + ) + .expect("write checksum"); Ok(AddResult::Succ) }) } - fn delete(&self, entry: &str, itempath: &Path) -> Result<(), Error> { - let item: Box = if itempath.is_file() { - Box::new(file_item::FileItem::new(itempath).map_err(Error::FileItemError)?) - } else { - Box::new(dir_item::DirItem::new(itempath).map_err(Error::DirItemError)?) - }; - - let c = get_or_run(&self.dockercli, &self.image_name)?; - let dstpath = Path::new("/data").join(entry).join(item.id()); + fn delete(&self, entry: &str, item: &str) -> Result<(), Error> { + let c = get_or_run(&self.dockercli, &self.image_name).map_err(Error::DockerError)?; + let dstpath = Path::new("/data").join(entry).join(item); self.dockercli .remove_file(&c.id, &dstpath) .map_err(|e| Error::ExecError(format!("{:?}", e))) } fn export_to_dir(&self, dir: &Path, entry: &str) -> Result<(), Error> { - let c = get_or_run(&self.dockercli, &self.image_name)?; + let c = get_or_run(&self.dockercli, &self.image_name).map_err(Error::DockerError)?; let srcpath = Path::new("/data").join(entry); self.dockercli .copy_out(&c.id, &srcpath, dir) @@ -144,7 +148,7 @@ impl DB for ImageDrive { // image existed, so push to registry // 1. commit all changed data in container to image - let c = get_or_run(&self.dockercli, &self.image_name)?; + let c = get_or_run(&self.dockercli, &self.image_name).map_err(Error::DockerError)?; if self.dockercli.diff_container_image_content(&c.id) { let _ = self .dockercli @@ -170,26 +174,8 @@ impl DB for ImageDrive { } } -fn get_or_run( - cli: &dockerclient::DockerClient, - image: &str, -) -> Result { - let cs = cli.ps(false).map_err(Error::DockerError)?; - for c in cs { - if c.image.split(":").collect::>()[0] - == image.split(":").collect::>()[0] - { - return Ok(c); - } - } - // not found - let c = cli.create(image).map_err(Error::DockerError)?; - cli.start(&c.id).map_err(Error::DockerError)?; - Ok(c) -} - fn ls(cli: &dockerclient::DockerClient, image: &str, dir: &Path) -> Result, Error> { - let c = get_or_run(cli, image)?; + let c = get_or_run(cli, image).map_err(Error::DockerError)?; cli.exec(&c.id, &format!("ls {}", dir.display())) .map_err(Error::DockerError) .map(|(out, err)| { @@ -205,7 +191,7 @@ fn ls(cli: &dockerclient::DockerClient, image: &str, dir: &Path) -> Result = out.trim().split(" ").collect(); + let dirs: Vec<&str> = out.trim().split_whitespace().collect(); let mut r = vec![]; for dir in dirs { r.push(dir.trim().to_owned().to_string()); diff --git a/db/src/file_item.rs b/db/src/file_item.rs deleted file mode 100644 index d1692fe..0000000 --- a/db/src/file_item.rs +++ /dev/null @@ -1,49 +0,0 @@ -use sha2::{Digest, Sha256}; -use std::fs::File; -use std::io; -use std::io::{copy, Seek, SeekFrom}; -use std::path::Path; - -#[derive(Debug)] -pub enum Error { - BadPath(String), - IOError(io::Error), -} - -pub struct FileItem<'a> { - path: &'a Path, - f: File, - id: String, -} - -impl<'a> FileItem<'a> { - pub fn new(path: &'a Path) -> Result { - if !path.is_file() || !path.exists() { - return Err(Error::BadPath(path.display().to_string())); - } - let f = File::open(path).map_err(Error::IOError)?; - let id = path - .file_name() - .unwrap() - .to_string_lossy() - .to_owned() - .to_string(); - Ok(FileItem { path, f, id }) - } -} - -impl<'a> crate::Item for FileItem<'a> { - fn hash(&mut self) -> Vec { - self.f.seek(SeekFrom::Start(0)).unwrap(); - let mut hasher = Sha256::new(); - copy(&mut self.f, &mut hasher).unwrap(); - hasher.result().to_vec() - } - fn id(&self) -> &str { - &self.id - } - - fn srcpath(&self) -> &Path { - self.path - } -} diff --git a/db/src/hostitem.rs b/db/src/hostitem.rs new file mode 100644 index 0000000..d8a8af4 --- /dev/null +++ b/db/src/hostitem.rs @@ -0,0 +1,70 @@ +use sha2::{Digest, Sha256}; +use std::fs::File; +use std::io; +use std::io::{copy, Seek, SeekFrom}; +use std::path::Path; + +#[derive(Debug)] +pub enum Error { + BadPath(String), + IOError(io::Error), + DefaultError(String), +} + +pub struct HostItem<'a> { + path: &'a Path, + id: String, + fs: Vec, // if item is file, only 1 elem in vec +} + +impl<'a> HostItem<'a> { + pub fn new(path: &'a Path) -> Result { + let mut filename = "".into(); + match path.file_name() { + None => { + return Err(Error::BadPath(path.display().to_string())); + } + Some(s) => filename = s.to_string_lossy().to_owned(), + } + let id = format!("{:?}", filename); + let mut fs = vec![]; + // host file or dir + if path.is_file() { + let f = File::open(path).map_err(Error::IOError)?; + fs.push(f); + } else if path.is_dir() { + for e in walkdir::WalkDir::new(path) + .follow_links(true) + .sort_by(|a, b| a.file_name().cmp(b.file_name())) + { + let entry = e.map_err(|err| Error::DefaultError(err.to_string()))?; + if entry.path().is_dir() { + let f = File::open(entry.path()).map_err(Error::IOError)?; + fs.push(f); + } + } + } else { + return Err(Error::BadPath(path.display().to_string())); + } + Ok(HostItem { path, id, fs }) + } +} + +impl<'a> crate::Item for HostItem<'a> { + fn hash(&mut self) -> Vec { + let mut hasher = Sha256::new(); + for f in &mut self.fs { + f.seek(SeekFrom::Start(0)).unwrap(); + copy(f, &mut hasher).unwrap(); + } + hasher.result().to_vec() + } + + fn id(&self) -> &str { + &self.id + } + + fn srcpath(&self) -> &Path { + self.path + } +} diff --git a/db/src/lib.rs b/db/src/lib.rs index 7566685..febba53 100644 --- a/db/src/lib.rs +++ b/db/src/lib.rs @@ -1,16 +1,18 @@ extern crate dockerclient; +extern crate hex; extern crate sha2; extern crate walkdir; -mod dir_item; +mod containeritem; pub mod docker; -mod file_item; +mod hostitem; +mod utils; pub trait Item { /// Hash compute item's hash value fn hash(&mut self) -> Vec; /// id is Item's unqiue name fn id(&self) -> &str; - /// path is item's host path + /// path is item's path fn srcpath(&self) -> &std::path::Path; } @@ -36,7 +38,7 @@ where /// add `item` to DB under `entry` fn add(&self, entry: &str, itempath: &std::path::Path) -> Result; /// delete item from DB, which is located by entry and reference - fn delete(&self, entry: &str, itempath: &std::path::Path) -> Result<(), E>; + fn delete(&self, entry: &str, item: &str) -> Result<(), E>; /// export_to_dir export `entry` to `dir` fn export_to_dir(&self, dir: &std::path::Path, entry: &str) -> Result<(), E>; /// sync local DB to remote DB diff --git a/db/src/utils.rs b/db/src/utils.rs new file mode 100644 index 0000000..0358c3a --- /dev/null +++ b/db/src/utils.rs @@ -0,0 +1,19 @@ +use crate::*; + +pub fn get_or_run( + cli: &dockerclient::DockerClient, + image: &str, +) -> Result { + let cs = cli.ps(false)?; + for c in cs { + if c.image.split(":").collect::>()[0] + == image.split(":").collect::>()[0] + { + return Ok(c); + } + } + // not found + let c = cli.create(image)?; + cli.start(&c.id)?; + Ok(c) +}