From 25c7d0d4707ecd31b83fc14ae0611a06bedae338 Mon Sep 17 00:00:00 2001 From: Draklaw Date: Sat, 21 Oct 2023 19:55:17 +0200 Subject: [PATCH] Mostly improved tree walking I guess. --- Cargo.lock | 73 ++++++++++- cas-core/src/cas.rs | 11 +- cas-core/src/error.rs | 12 ++ cas-core/src/lib.rs | 6 +- cas-core/src/object_metadata.rs | 2 +- cas-simple/Cargo.toml | 4 +- cas-simple/src/cas.rs | 182 ++++++++++++++-------------- cas-simple/src/lib.rs | 2 +- cas-simple/src/utils.rs | 102 ++++++++-------- libbsv/Cargo.toml | 4 +- libbsv/src/config.rs | 31 ----- libbsv/src/ignore.rs | 2 +- libbsv/src/lib.rs | 2 - libbsv/src/path_map.rs | 66 ++++++---- libbsv/src/repository.rs | 206 +++++++++++++++++++------------- libbsv/src/tree_item.rs | 121 ++++++++++++++----- libbsv/src/tree_walker.rs | 72 +++++++++-- 17 files changed, 553 insertions(+), 345 deletions(-) delete mode 100644 libbsv/src/config.rs diff --git a/Cargo.lock b/Cargo.lock index e42182d..31d5ffb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -93,6 +93,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.2" @@ -130,6 +136,22 @@ dependencies = [ "version_check", ] +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "libbsv" version = "0.1.0" @@ -225,9 +247,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.127" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f03b9878abf6d14e6779d3f24f07b2cfa90352cfec4acc5aab8f1ac7f146fae8" +checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" + +[[package]] +name = "serde_spanned" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +dependencies = [ + "serde", +] [[package]] name = "sha2" @@ -288,11 +319,36 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.8" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542" dependencies = [ "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", ] [[package]] @@ -378,3 +434,12 @@ name = "windows_x86_64_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + +[[package]] +name = "winnow" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5504cc7644f4b593cbc05c4a55bf9bd4e94b867c3c0bd440934174d50482427d" +dependencies = [ + "memchr", +] diff --git a/cas-core/src/cas.rs b/cas-core/src/cas.rs index f2c1085..743b59b 100644 --- a/cas-core/src/cas.rs +++ b/cas-core/src/cas.rs @@ -63,7 +63,10 @@ pub trait Cas { } pub trait RefStore { - fn get_ref>(&self, key: P) -> Result; - fn set_ref>(&mut self, key: P, value: &ObjectId) -> Result<()>; - fn remove_ref>(&mut self, key: P) -> Result<()>; -} \ No newline at end of file + fn get_ref(&self, key: &Utf8Path) -> Result; + fn set_ref(&mut self, key: &Utf8Path, value: &ObjectId) -> Result<()>; + fn remove_ref(&mut self, key: &Utf8Path) -> Result<()>; +} + +pub trait CasWithRef: Cas + RefStore { +} diff --git a/cas-core/src/error.rs b/cas-core/src/error.rs index ef04d84..e6fb4f3 100644 --- a/cas-core/src/error.rs +++ b/cas-core/src/error.rs @@ -16,6 +16,10 @@ // use std::path::Utf8PathBuf; +use std::path::StripPrefixError; + +use crate::ObjectId; + /// Result type used through cas-core. pub type Result = std::result::Result; @@ -87,6 +91,9 @@ pub enum Error { #[error("non-unicode file name: '{0}'")] NonUnicodeFileName(String), + #[error("object {0} does not exists")] + ObjectDoesNotExists(ObjectId), + #[error("{0}")] UnknownError(String), } @@ -129,6 +136,11 @@ impl Error { } } +impl From for Error { + fn from(value: StripPrefixError) -> Self { + Error::unknown(format!("Invalid operation: {}", value)) + } +} // fn format_optional_path(maybe_path: &Option) -> String { // match maybe_path { diff --git a/cas-core/src/lib.rs b/cas-core/src/lib.rs index c53577c..137be03 100644 --- a/cas-core/src/lib.rs +++ b/cas-core/src/lib.rs @@ -35,10 +35,10 @@ mod cas; pub use crate::{ error::{Error, Result}, object_id::{ObjectId, hex, write_hex}, - object_type::{ObjectType}, - object_metadata::{ObjectMetadata}, + object_type::ObjectType, + object_metadata::ObjectMetadata, pipeline::{Pipeline, DefaultPipeline, Reader, Writer, ReadWrapper, WriteWrapper}, - cas::{Cas, RefStore}, + cas::{Cas, CasWithRef, RefStore}, }; diff --git a/cas-core/src/object_metadata.rs b/cas-core/src/object_metadata.rs index 68c3fe1..5afbc25 100644 --- a/cas-core/src/object_metadata.rs +++ b/cas-core/src/object_metadata.rs @@ -14,7 +14,7 @@ // along with bsv. If not, see . -use super::object_type::{ObjectType}; +use super::object_type::ObjectType; #[derive(Clone, Eq, PartialEq)] diff --git a/cas-simple/Cargo.toml b/cas-simple/Cargo.toml index 4a941d8..63bd3a3 100644 --- a/cas-simple/Cargo.toml +++ b/cas-simple/Cargo.toml @@ -11,8 +11,6 @@ license = "AGPL-3.0-or-later" digest = { version = "0.9.0", features = ["alloc"] } sha2 = "0.9.5" camino = { version = "1.0.7" } -toml = "0.5.8" +toml = "0.7.6" cas-core = { path = "../cas-core" } tempfile = "3.2.0" - -[dev-dependencies] diff --git a/cas-simple/src/cas.rs b/cas-simple/src/cas.rs index cef3358..8bd55ab 100644 --- a/cas-simple/src/cas.rs +++ b/cas-simple/src/cas.rs @@ -17,60 +17,75 @@ use std::str::FromStr; use digest::DynDigest; -use toml::Value; +use toml::{Value, Table}; use camino::{Utf8Path, Utf8PathBuf}; use cas_core::{ Cas, DefaultPipeline, err, Error, ObjectId, ObjectMetadata, ObjectType, - Pipeline, Reader, ReadWrapper, RefStore, Result, Writer, + Pipeline, Reader, ReadWrapper, RefStore, Result, Writer, CasWithRef, }; use crate::utils::{ obj_dir, obj_path, ref_dir, tmp_dir, - read_config, write_config, read_metadata, write_metadata, new_digest, }; use crate::wfile::WFile; +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SimpleCasConfig { + pub digest_id: String, +} + +impl SimpleCasConfig { + pub fn new(digest_id: String) -> Self { + Self { + digest_id, + } + } + + pub fn from_toml_value(value: &Value) -> Result { + if let Value::Table(ref table) = value { + let engine = + table.get("engine") + .ok_or_else(|| Error::unknown("missing engine field in cas config"))? + .as_str() + .ok_or_else(|| Error::unknown("cas engine must be a string"))?; + if engine != "simple" { + return err!("expected simple cas engine"); + } + let digest_id = + table.get("digest") + .ok_or_else(|| Error::unknown("missing digest field in cas config"))? + .as_str() + .ok_or_else(|| Error::unknown("cas digest must be a string"))?; + Ok(Self { + digest_id: digest_id.to_string(), + }) + } + else { + err!("cas config must be a table") + } + } + + pub fn to_toml_value(&self) -> Result { + let mut table = Table::new(); + table["engine"] = "simple".into(); + table["digest"] = "sha256".into(); + return Ok(Value::Table(table)) + } +} + pub struct SimpleCas { + config: SimpleCasConfig, db_path: Utf8PathBuf, digest: Box, pipeline: DefaultPipeline, - // config: Value, } impl SimpleCas { - pub fn create(db_path: Utf8PathBuf, mut config: Value) -> Result { - if !config.is_table() { - return Error::err("invalid config object: must be table"); - } - - let maybe_engine = config.as_table_mut().unwrap() - .entry("cas") - .or_insert_with(|| toml::value::Table::new().into()) - .as_table_mut().unwrap() - .entry("engine") - .or_insert("simple".into()) - .as_str(); - match maybe_engine { - Some(engine) if engine != "simple" => { - return err!("invalid cas.engine in config: got {}, expected simple", engine); - }, - None => { - return Error::err("invalid casengine in config: expected String"); - }, - _ => {} - } - - let digest_id = config["cas"]["digest"].as_str() - .ok_or_else(|| Error::unknown( - "mandatory cas.digest value is invalid or missing from config" - ))?; - let digest = new_digest(digest_id)?; - let pipeline = DefaultPipeline::new(digest.box_clone()); - + pub fn create(db_path: Utf8PathBuf, config: SimpleCasConfig) -> Result { if db_path.exists() { return err!( "failed to create SimpleCas: target directory already exists ({})", @@ -83,8 +98,8 @@ impl SimpleCas { &obj_dir(&db_path), &ref_dir(&db_path), &tmp_dir(&db_path), - ] { - std::fs::create_dir(path).or_else(|e| + ] { + std::fs::create_dir(path).or_else(|e| err!( "failed to create directory ({}): {}", path, e, @@ -92,37 +107,24 @@ impl SimpleCas { )?; } - write_config(&config, &db_path)?; - - Ok(SimpleCas { - db_path, - digest, - pipeline, - // config, - }) + Self::open(db_path, config) } - pub fn open(db_path: Utf8PathBuf) -> Result { - let config = read_config(&db_path)?; - - let digest_id = config["cas"]["digest"].as_str() - .ok_or_else(|| Error::unknown( - "mandatory cas.digest value is invalid or missing from config" - ))?; - let digest = new_digest(digest_id)?; + pub fn open(db_path: Utf8PathBuf, config: SimpleCasConfig) -> Result { + let digest = new_digest(&config.digest_id)?; let pipeline = DefaultPipeline::new(digest.box_clone()); Ok(SimpleCas { + config, db_path, digest, pipeline, - // config, }) } - // pub fn save_config(&self) -> Result<()> { - // write_config(&self.config, &self.db_path) - // } + pub fn get_config(&self) -> &SimpleCasConfig { + &self.config + } } @@ -152,7 +154,7 @@ impl Cas for SimpleCas { fn open_object(&self, oid: &ObjectId) -> Result<(ObjectMetadata, Box)> { let opath = obj_path(&self.db_path, oid); if !opath.is_file() { - return err!("object not found: {}", oid); + return Err(Error::ObjectDoesNotExists(oid.clone())); } let file = std::fs::File::open(opath).or_else(|err| @@ -184,56 +186,59 @@ impl Cas for SimpleCas { } impl RefStore for SimpleCas { - fn get_ref>(&self, key: P) -> Result { - let path = ref_dir(&self.db_path).join(key.as_ref()); + fn get_ref(&self, key: &Utf8Path) -> Result { + let path = ref_dir(&self.db_path).join(key); if !path.exists() { - err!("reference {} does not exists", key.as_ref()) + err!("reference {} does not exists", key) } else if !path.is_file() { - err!("reference {} is not a file", key.as_ref()) + err!("reference {} is not a file", key) } else { let file = std::fs::read(path).or_else(|err| - err!("failed to read reference file for {}: {}", key.as_ref(), err) + err!("failed to read reference file for {}: {}", key, err) )?; Ok( ObjectId::from_str( std::str::from_utf8(&file).or_else(|err| - err!("invalid reference file at {}: {}", key.as_ref(), err) + err!("invalid reference file at {}: {}", key, err) )? )? ) } } - fn set_ref>(&mut self, key: P, value: &ObjectId) -> Result<()> { - let path = ref_dir(&self.db_path).join(key.as_ref()); + fn set_ref(&mut self, key: &Utf8Path, value: &ObjectId) -> Result<()> { + let path = ref_dir(&self.db_path).join(key); std::fs::create_dir_all(path.parent().ok_or_else(|| - Error::unknown(format!("reference file {} has no parent dir?", key.as_ref())) + Error::unknown(format!("reference file {} has no parent dir?", key)) )?).or_else(|err| - err!("failed to create reference dir for {}: {}", key.as_ref(), err) + err!("failed to create reference dir for {}: {}", key, err) )?; std::fs::write(path, value.to_string()).or_else(|err| - err!("failed to write reference {}: {}", key.as_ref(), err) + err!("failed to write reference {}: {}", key, err) ) } - fn remove_ref>(&mut self, key: P) -> Result<()> { - let path = ref_dir(&self.db_path).join(key.as_ref()); + fn remove_ref(&mut self, key: &Utf8Path) -> Result<()> { + let path = ref_dir(&self.db_path).join(key); if !path.exists() { - err!("reference {} does not exists", key.as_ref()) + err!("reference {} does not exists", key) } else if !path.is_file() { - err!("reference {} is not a file", key.as_ref()) + err!("reference {} is not a file", key) } else { std::fs::remove_file(path).or_else(|err| - err!("failed to remove reference file {}: {}", key.as_ref(), err) + err!("failed to remove reference file {}: {}", key, err) ) } } } +impl CasWithRef for SimpleCas { +} + pub struct ObjectIdIterator { root_dirs: Vec, @@ -387,13 +392,6 @@ mod tests { use super::*; - fn get_config() -> Value { - toml::toml!( - [cas] - digest = "sha256" - ) - } - fn get_cas_path(dir: &Utf8Path) -> Utf8PathBuf { let mut cas_path = dir.to_path_buf(); cas_path.push(".bsv"); @@ -404,7 +402,7 @@ mod tests { fn test_create_simple_cas() { let dir = tempfile::tempdir().expect("failed to create temp test dir"); let cas_path = get_cas_path(Utf8Path::from_path(dir.path()).unwrap()); - let config = get_config(); + let config = SimpleCasConfig::new("sha256".into()); let cas = SimpleCas::create(cas_path, config) .expect("failed to create SimpleCas object"); @@ -418,7 +416,7 @@ mod tests { fn test_write_object() { let dir = tempfile::tempdir().expect("failed to create temp test dir"); let cas_path = get_cas_path(dir.path().try_into().unwrap()); - let config = get_config(); + let config = SimpleCasConfig::new("sha256".into()); let otype = ObjectType::new(b"blob").expect("failed to create object type"); let payload = b"Hello World!"; @@ -448,7 +446,7 @@ mod tests { let dir = tempfile::tempdir().expect("failed to create temp test dir"); let cas_path = get_cas_path(dir.path().try_into().unwrap()); - let config = get_config(); + let config = SimpleCasConfig::new("sha256".into()); let otype = ObjectType::new(b"blob").expect("failed to create object type"); let payload = b"Hello World!"; @@ -487,7 +485,7 @@ mod tests { fn test_read_write_object() { let dir = tempfile::tempdir().expect("failed to create temp test dir"); let cas_path = get_cas_path(dir.path().try_into().unwrap()); - let config = get_config(); + let config = SimpleCasConfig::new("sha256".into()); let otype = ObjectType::new(b"blob").expect("failed to create object type"); let payload = b"This is a test."; @@ -507,7 +505,7 @@ mod tests { fn test_remove_object() { let dir = tempfile::tempdir().expect("failed to create temp test dir"); let cas_path = get_cas_path(dir.path().try_into().unwrap()); - let config = get_config(); + let config = SimpleCasConfig::new("sha256".into()); let otype = ObjectType::new(b"blob").expect("failed to create object type"); let payload = b"This is a test."; @@ -528,7 +526,7 @@ mod tests { fn test_object_id_iterator() { let dir = tempfile::tempdir().expect("failed to create temp test dir"); let cas_path = get_cas_path(dir.path().try_into().unwrap()); - let config = get_config(); + let config = SimpleCasConfig::new("sha256".into()); let mut cas = SimpleCas::create(cas_path.clone(), config) .expect("failed to create SimpleCas object"); @@ -642,21 +640,21 @@ mod tests { fn test_reference() { let dir = tempfile::tempdir().expect("failed to create temp test dir"); let cas_path = get_cas_path(dir.path().try_into().unwrap()); - let config = get_config(); + let config = SimpleCasConfig::new("sha256".into()); let mut cas = SimpleCas::create(cas_path.clone(), config) .expect("failed to create SimpleCas object"); let oid_0 = ObjectId::from_str("f731f6bc6a6a73bad170e56452473ef6930b7a0ab33cc54be44221a89b49d786").unwrap(); - assert!(cas.get_ref("foo/bar").is_err()); - assert!(cas.remove_ref("foo/bar").is_err()); + assert!(cas.get_ref("foo/bar".into()).is_err()); + assert!(cas.remove_ref("foo/bar".into()).is_err()); - cas.set_ref("foo/bar", &oid_0).unwrap(); - assert_eq!(cas.get_ref("foo/bar").unwrap(), oid_0); + cas.set_ref("foo/bar".into(), &oid_0).unwrap(); + assert_eq!(cas.get_ref("foo/bar".into()).unwrap(), oid_0); - cas.remove_ref("foo/bar").unwrap(); + cas.remove_ref("foo/bar".into()).unwrap(); - assert!(cas.get_ref("foo/bar").is_err()); + assert!(cas.get_ref("foo/bar".into()).is_err()); } } diff --git a/cas-simple/src/lib.rs b/cas-simple/src/lib.rs index 7086404..69d5546 100644 --- a/cas-simple/src/lib.rs +++ b/cas-simple/src/lib.rs @@ -32,4 +32,4 @@ mod wfile; mod cas; -pub use cas::SimpleCas; +pub use cas::{SimpleCas, SimpleCasConfig}; diff --git a/cas-simple/src/utils.rs b/cas-simple/src/utils.rs index 7f08daa..c7bb09a 100644 --- a/cas-simple/src/utils.rs +++ b/cas-simple/src/utils.rs @@ -16,7 +16,7 @@ use digest::DynDigest; -use toml::Value; +// use toml::Value; use camino::{Utf8Path, Utf8PathBuf}; use cas_core::{ @@ -49,39 +49,39 @@ pub fn config_path(cas_path: &Utf8Path) -> Utf8PathBuf { } -pub fn read_config(db_path: &Utf8Path) -> Result { - use std::io::Read; - - let mut file = std::fs::File::open(config_path(db_path)).or_else(|err| - err!("invalid repository: no config file: {}", err) - )?; - let mut config_str = String::new(); - file.read_to_string(&mut config_str).or_else(|err| - err!("cannot read config file: {}", err) - )?; - let config = toml::from_str(&config_str).or_else(|err| - err!("error while reading config file: {}", err) - )?; - Ok(config) -} - -pub fn write_config(config: &Value, db_path: &Utf8Path) -> Result<()> { - use std::io::Write; - - let config_str = toml::to_string_pretty(config).or_else(|err| - err!("failed to serialize config: {}", err) - )?; - let mut file = tempfile::NamedTempFile::new_in(tmp_dir(db_path)).or_else(|err| - err!("cannot create temp config file: {}", err) - )?; - file.write_all(config_str.as_bytes()).or_else(|err| - err!("failed to write to temp config: {}", err) - )?; - file.persist(config_path(db_path)).or_else(|err| - err!("failed to (over)write config: {}", err) - )?; - Ok(()) -} +// pub fn read_config(db_path: &Utf8Path) -> Result { +// use std::io::Read; + +// let mut file = std::fs::File::open(config_path(db_path)).or_else(|err| +// err!("invalid repository: no config file: {}", err) +// )?; +// let mut config_str = String::new(); +// file.read_to_string(&mut config_str).or_else(|err| +// err!("cannot read config file: {}", err) +// )?; +// let config = toml::from_str(&config_str).or_else(|err| +// err!("error while reading config file: {}", err) +// )?; +// Ok(config) +// } + +// pub fn write_config(config: &Value, db_path: &Utf8Path) -> Result<()> { +// use std::io::Write; + +// let config_str = toml::to_string_pretty(config).or_else(|err| +// err!("failed to serialize config: {}", err) +// )?; +// let mut file = tempfile::NamedTempFile::new_in(tmp_dir(db_path)).or_else(|err| +// err!("cannot create temp config file: {}", err) +// )?; +// file.write_all(config_str.as_bytes()).or_else(|err| +// err!("failed to write to temp config: {}", err) +// )?; +// file.persist(config_path(db_path)).or_else(|err| +// err!("failed to (over)write config: {}", err) +// )?; +// Ok(()) +// } pub fn read_metadata(read: &mut dyn std::io::Read) -> Result { @@ -151,28 +151,28 @@ mod tests { ); } - #[test] - fn test_read_write_config() { - let config = toml::toml!{ - [cas] - path = "/foo/bar" - digest = "sha1" + // #[test] + // fn test_read_write_config() { + // let config = toml::toml!{ + // [cas] + // path = "/foo/bar" + // digest = "sha1" - [extra] - test = 42 - }; + // [extra] + // test = 42 + // }; - let dir = tempfile::TempDir::new() - .expect("failed to create tmp dir"); - std::fs::create_dir(tmp_dir(Utf8Path::from_path(dir.path()).unwrap())).expect("failed to create db/tmp dir"); + // let dir = tempfile::TempDir::new() + // .expect("failed to create tmp dir"); + // std::fs::create_dir(tmp_dir(Utf8Path::from_path(dir.path()).unwrap())).expect("failed to create db/tmp dir"); - write_config(&config, Utf8Path::from_path(dir.path()).unwrap()).expect("failed to write config"); - let config2 = read_config(Utf8Path::from_path(dir.path()).unwrap()).expect("failed to read config"); + // write_config(&config, Utf8Path::from_path(dir.path()).unwrap()).expect("failed to write config"); + // let config2 = read_config(Utf8Path::from_path(dir.path()).unwrap()).expect("failed to read config"); - assert_eq!(config2, config); + // assert_eq!(config2, config); - dir.close().expect("failed to close tmp dir") - } + // dir.close().expect("failed to close tmp dir") + // } #[test] fn test_read_write_metadata() { diff --git a/libbsv/Cargo.toml b/libbsv/Cargo.toml index b38f087..5cdfac4 100644 --- a/libbsv/Cargo.toml +++ b/libbsv/Cargo.toml @@ -6,11 +6,9 @@ edition = "2021" license = "AGPL-3.0-or-later" [dependencies] -toml = "0.5.8" +toml = { version = "0.7.6", features = ["parse"] } camino = "1.0.7" regex = "1.6.0" cas-core = { path = "../cas-core" } cas-simple = { path = "../cas-simple" } - -[dev-dependencies] tempfile = "3.7.1" diff --git a/libbsv/src/config.rs b/libbsv/src/config.rs deleted file mode 100644 index 9f39a6c..0000000 --- a/libbsv/src/config.rs +++ /dev/null @@ -1,31 +0,0 @@ -// This file is part of bsv. -// -// bsv is free software: you can redistribute it and/or modify it under the -// terms of the GNU Affero General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// bsv is distributed in the hope that it will be useful, but WITHOUT ANY -// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -// FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for -// more details. -// -// You should have received a copy of the Affero GNU General Public License -// along with bsv. If not, see . - - -use std::collections::{HashMap}; -use camino::Utf8PathBuf; - -use cas_core::{Cas, Result}; - - -pub trait CasConfig { - fn build_cas(&self) -> Result>; -} - - -pub struct BsvConfig { - cas: Box, - dir_map: HashMap, -} diff --git a/libbsv/src/ignore.rs b/libbsv/src/ignore.rs index 38b9a24..a91847a 100644 --- a/libbsv/src/ignore.rs +++ b/libbsv/src/ignore.rs @@ -27,7 +27,7 @@ pub enum IgnoreAction { } -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct IgnoreRules { patterns: RegexSet, actions: Vec, diff --git a/libbsv/src/lib.rs b/libbsv/src/lib.rs index 2311357..b554bde 100644 --- a/libbsv/src/lib.rs +++ b/libbsv/src/lib.rs @@ -17,8 +17,6 @@ extern crate toml; extern crate camino; extern crate regex; - -#[cfg(test)] extern crate tempfile; extern crate cas_core; diff --git a/libbsv/src/path_map.rs b/libbsv/src/path_map.rs index d22b4d9..5149c85 100644 --- a/libbsv/src/path_map.rs +++ b/libbsv/src/path_map.rs @@ -15,7 +15,7 @@ use camino::{Utf8Path, Utf8PathBuf}; -use toml::Value; +use toml::{Value, Table}; use cas_core::{err, Error, Result}; @@ -30,10 +30,28 @@ impl PathPair { pub fn new(logic: Utf8PathBuf, physic: Utf8PathBuf) -> Self { Self { logic, physic } } + + pub fn physic_from_logic>(&self, logic_path: P) -> Result { + let mut physic = self.physic.clone(); + physic.push( + logic_path.as_ref().strip_prefix(&self.logic)? + ); + + Ok(physic) + } + + pub fn logic_from_physic>(&self, physic_path: P) -> Result { + let mut logic = self.logic.clone(); + logic.push( + physic_path.as_ref().strip_prefix(&self.physic)? + ); + + Ok(logic) + } } -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct PathMap { pairs: Vec, logic_order: Vec, @@ -113,38 +131,42 @@ impl PathMap { Self::from_vec(map) } - pub fn physic_from_logic>(&self, logic_path: P) -> Option { + pub fn to_toml_value(&self) -> Result { + let mut table = Table::new(); + for pair in self.pairs.iter() { + table[pair.logic.as_str()] = pair.physic.as_str().into(); + } + Ok(Value::Table(table)) + } + + pub fn path_pair_from_logic>(&self, logic_path: P) -> Option<&PathPair> { let path = logic_path.as_ref(); - let pair = self.logic_order.iter() + Some(self.logic_order.iter() .map(|&i| &self.pairs[i]) .filter(|p| path.starts_with(&p.logic)) - .next()?; - - let mut physic = pair.physic.clone(); - physic.push( - path.strip_prefix(&pair.logic) - .ok()? - ); - - return Some(physic) + .next()? + ) } - pub fn logic_from_physic>(&self, physic_path: P) -> Option { + pub fn path_pair_from_physic>(&self, physic_path: P) -> Option<&PathPair> { let path = physic_path.as_ref(); - let pair = self.physic_order.iter() + Some(self.physic_order.iter() .map(|&i| &self.pairs[i]) .filter(|p| path.starts_with(&p.physic)) - .next()?; + .next()? + ) + } - let mut logic = pair.logic.clone(); - logic.push( - path.strip_prefix(&pair.physic) - .ok()? - ); + pub fn physic_from_logic>(&self, logic_path: P) -> Option { + self.path_pair_from_logic(&logic_path)? + .physic_from_logic(&logic_path).ok() + } - return Some(logic) + pub fn logic_from_physic>(&self, physic_path: P) -> Option { + self.path_pair_from_physic(&physic_path)? + .logic_from_physic(&physic_path).ok() } } diff --git a/libbsv/src/repository.rs b/libbsv/src/repository.rs index a16de7d..12fc6b1 100644 --- a/libbsv/src/repository.rs +++ b/libbsv/src/repository.rs @@ -14,100 +14,118 @@ // along with bsv. If not, see . +use std::{fs::create_dir_all, io::BufReader}; + use camino::{Utf8Path, Utf8PathBuf}; -use toml::Value; +use cas_simple::utils::config_path; +use toml::{Value, Table}; -use cas_core::{Cas, err, Error, ObjectId, Result}; -use cas_simple::SimpleCas; +use cas_core::{err, Error, ObjectId, Result, ObjectType, CasWithRef}; +use cas_simple::{SimpleCas, SimpleCasConfig}; +use crate::PathMap; pub use crate::permissions::Permissions; +use crate::tree_item::Deserialize; pub use crate::tree_item::{Serialize, TreeItem}; -pub fn create_cas(path: Utf8PathBuf, config: Value) -> Result> { - let engine = config - .get("cas") - .ok_or_else(|| Error::unknown("config must have a cas item"))? - .get("engine") - .ok_or_else(|| Error::unknown("config must have a cas.engine item"))? - .as_str() - .ok_or_else(|| Error::unknown("cas.engine must be a string"))?; - - match engine { - "simple" => { SimpleCas::create(path, config).map(|cas| Box::new(cas) as Box) } - _ => { err!("unknown cas engine {}", engine) } - } +#[derive(Clone, Debug)] +pub struct Config { + pub device_name: String, + pub cas: SimpleCasConfig, + pub path_map: PathMap, } +impl Config { + pub fn from_toml_value(value: &Value) -> Result { + let bsv_value = value.get("bsv") + .ok_or_else(|| Error::unknown("bsv section missing from config"))?; + if !bsv_value.is_table() { + return err!("bsv section must be a table"); + } -pub fn open_cas(path: Utf8PathBuf, config: &Value) -> Result> { - let engine = config - .get("cas") - .ok_or_else(|| Error::unknown("config must have a cas item"))? - .get("engine") - .ok_or_else(|| Error::unknown("config must have a cas.engine item"))? - .as_str() - .ok_or_else(|| Error::unknown("cas.engine must be a string"))?; + Ok(Self { + device_name: bsv_value.get("device_name") + .ok_or_else(|| Error::unknown("bsv.device_name missing from config"))? + .as_str() + .ok_or_else(|| Error::unknown("bsv.device_name must be a string"))? + .to_string(), + cas: SimpleCasConfig::from_toml_value( + value.get("cas").ok_or_else(|| Error::unknown("config must have a cas section"))? + )?, + path_map: PathMap::from_toml_value( + value.get("mapping").ok_or_else(|| Error::unknown("config must have a mapping section"))? + )?, + }) + } - match engine { - "simple" => { SimpleCas::open(path).map(|cas| Box::new(cas) as Box) } - _ => { err!("unknown cas engine {}", engine) } + pub fn from_toml_file(config_path: &Utf8Path) -> Result { + use std::io::Read; + + let mut file = std::fs::File::open(config_path).or_else(|err| + err!("invalid repository: failed to read config file: {}", err) + )?; + let mut config_str = String::new(); + file.read_to_string(&mut config_str).or_else(|err| + err!("failed to read config file: {}", err) + )?; + let value = config_str.parse::().or_else(|err| + err!("parse error while reading config file: {}", err) + )?; + Self::from_toml_value(&value) } -} + pub fn to_toml_value(&self) -> Result { + let mut bsv = Table::new(); + bsv["device_name"] = Value::String(self.device_name.clone()); + + let mut table = Table::new(); + table["bsv"] = Value::Table(bsv); + table["cas"] = self.cas.to_toml_value()?; + table["mapping"] = self.path_map.to_toml_value()?; + + Ok(Value::Table(table)) + } -// pub trait FsVisitor { -// fn accept(&self, path: &Utf8Path, metadata: &Metadata) -> bool; -// fn handle_error(&mut self, error: Error) -> Option; - -// fn handle_result(&mut self, result: Result) -> Result -// { -// result.map_err(|error| -// self.handle_error(error) -// .unwrap_or(Error::Skipped) -// ) -// } -// } - - -// fn read_path_map(config: &Value) -> Result> { -// if let Some(mapping) = config.get("mapping") { -// if let Some(ref table) = mapping.as_table() { -// table.iter() -// .map(|(k, v)| Ok(PathPair{ -// logic: k.into(), -// physic: v.as_str() -// .ok_or_else(|| Error::unknown("mapping values must be strings"))? -// .into() -// })) -// .collect() -// } -// else { -// err!("mapping must be a table, got {}", mapping) -// } -// } -// else { -// Ok(vec![]) -// } -// } + pub fn write_toml_file(&self, config_path: &Utf8Path) -> Result<()> { + use std::io::Write; + + let value = self.to_toml_value()?; + let config_str = toml::to_string_pretty(&value).or_else(|err| + err!("failed to serialize config: {}", err) + )?; + let mut file = tempfile::NamedTempFile::new_in(config_path.parent().unwrap()).or_else(|err| + err!("cannot create temp config file: {}", err) + )?; + file.write_all(config_str.as_bytes()).or_else(|err| + err!("failed to write to temp config: {}", err) + )?; + file.persist(config_path).or_else(|err| + err!("failed to (over)write config: {}", err) + )?; + Ok(()) + } +} pub struct Repository { - cas: Box, - // path_map: Vec, + config: Config, + cas: Box, } impl Repository { - pub fn create(path: Utf8PathBuf, config: Value) -> Result { + pub fn create(path: Utf8PathBuf, config: Config) -> Result { if path.exists() { return err!("cannot create bsv repository, path {} already exists", path); } - let cas = create_cas(path, config)?; + create_dir_all(&path)?; + config.write_toml_file(&config_path(&path))?; + let cas = SimpleCas::create(path.to_path_buf(), config.cas.clone())?; Ok(Self { - cas, - // path_map: vec![], + config, + cas: Box::new(cas), }) } @@ -117,39 +135,61 @@ impl Repository { } let config_file = cas_simple::utils::config_path(&path); - let config = cas_simple::utils::read_config(&config_file)?; + let config = Config::from_toml_file(&config_file)?; - let cas = open_cas(path, &config)?; - // let path_map = read_path_map(&config)?; + let cas = SimpleCas::open(path, config.cas.clone())?; Ok(Self { - cas, - // path_map, + config, + cas: Box::new(cas), }) } - pub fn cas(&self) -> &dyn Cas { + pub fn cas(&self) -> &dyn CasWithRef { self.cas.as_ref() } - // pub fn path_pair_from_logic_path>(&self, logic_path: P) -> Option { - // None - // } + pub fn physic_from_logic_path>(&self, logic_path: P) -> Option { + self.config.path_map.physic_from_logic(&logic_path) + } - // pub fn path_pair_from_physic_path>(&self, physic_path: P) -> Option { - // None - // } + pub fn path_pair_from_physic_path>(&self, physic_path: P) -> Option { + self.config.path_map.logic_from_physic(&physic_path) + } - pub fn oid_from_logic_path>(&self, logic_path: P) -> Result { + pub fn oid_from_logic_path>(&self, _logic_path: P) -> Result { + // let snapshot_oid = self.cas.get_ref(Utf8Path::new("latest"))?; + // let snapshot = self.read_snapshot(snapshot_oid)?; err!("not implemented") } pub fn oid_from_physic_path>(&self, physic_path: P) -> Result { - err!("not implemented") + let logic_path = self.config.path_map.logic_from_physic(physic_path) + .ok_or_else(|| Error::unknown("physic path do not map to a logic path"))?; + self.oid_from_logic_path(logic_path) } pub fn read_tree(&self, oid: &ObjectId) -> Result>> { - err!("not implemented") + match self.cas.open_object(oid) { + Ok((metadata, mut reader)) => { + if metadata.otype() != &ObjectType::new(b"tree")? { + err!("object is not a tree") + } + else { + let mut buf_read = BufReader::new(reader.as_mut()); + Ok(Some( + Vec::::deserialize(&mut buf_read)? + .unwrap_or_else(|| Vec::new()) + )) + } + }, + Err(Error::ObjectDoesNotExists(_)) => { + Ok(None) + } + Err(err) => { + Err(err) + } + } } // pub fn add(&mut self, path: P, visitor: &mut V) -> Result diff --git a/libbsv/src/tree_item.rs b/libbsv/src/tree_item.rs index 5e3fbe2..ecb75bc 100644 --- a/libbsv/src/tree_item.rs +++ b/libbsv/src/tree_item.rs @@ -27,10 +27,10 @@ pub trait Serialize { } pub trait Deserialize { - fn deserialize_with_buf(stream: &mut R, buf: &mut Vec) -> Result + fn deserialize_with_buf(stream: &mut R, buf: &mut Vec) -> Result> where Self: Sized; - fn deserialize(stream: &mut R) -> Result + fn deserialize(stream: &mut R) -> Result> where Self: Sized { let mut buf = Vec::new(); @@ -39,6 +39,15 @@ pub trait Deserialize { } +pub fn is_tree_item_name_valid(name: &str) -> bool { + return + !name.contains('/') && + !name.contains('\\') && + !name.contains('\n') && + !name.contains('\0'); +} + + #[derive(Clone, Debug, Eq, PartialEq)] pub struct TreeItem { pub name: String, @@ -52,6 +61,10 @@ pub struct TreeItem { impl TreeItem { pub fn from_metadata(name: String, metadata: &std::fs::Metadata, oid: ObjectId) -> Result { + if !is_tree_item_name_valid(&name) { + return err!("invalid item name {:?}", name); + } + let otype = otype_from_metadata(metadata)?; let permissions = Permissions::from_metadata(metadata)?; @@ -70,7 +83,7 @@ impl TreeItem { impl Serialize for TreeItem { fn serialize(&self, out: &mut W) -> Result<()> { // TODO: Check that name do not contain invalid characters - writeln!(out, "{}\t{}\t{}\t{}\t{}\t{}\t{}/", + writeln!(out, "{}\t{}\t{}\t{}\t{}\t{}\t{}", self.oid, self.otype, self.size, @@ -87,28 +100,39 @@ impl Serialize for TreeItem { } impl Deserialize for TreeItem { - fn deserialize_with_buf(stream: &mut R, buf: &mut Vec) -> Result { - let oid: ObjectId = read_field_parse(stream, buf, "object ID", b'\t')?; + fn deserialize_with_buf(stream: &mut R, buf: &mut Vec) -> Result> { + let oid: ObjectId = if let Some(oid) = read_field_parse(stream, buf, "object ID", b'\t')? { + oid + } + else { + return Ok(None); + }; read_field(stream, buf, "object type", b'\t')?; let otype = ObjectType::new(buf)?; - let size: u64 = read_field_parse(stream, buf, "object type", b'\t')?; - let permissions: Permissions = read_field_parse(stream, buf, "permissions", b'\t')?; + let size: u64 = read_field_parse(stream, buf, "object type", b'\t')? + .ok_or_else(|| Error::unknown("unexpected end-of-file"))?; + let permissions: Permissions = read_field_parse(stream, buf, "permissions", b'\t')? + .ok_or_else(|| Error::unknown("unexpected end-of-file"))?; let created = UNIX_EPOCH + Duration::from_millis( read_field_parse(stream, buf, "creation date", b'\t')? + .ok_or_else(|| Error::unknown("unexpected end-of-file"))? ); let modified = UNIX_EPOCH + Duration::from_millis( read_field_parse(stream, buf, "modification date", b'\t')? + .ok_or_else(|| Error::unknown("unexpected end-of-file"))? ); - let name = read_field_str(stream, buf, "name", b'/')? + let name = read_field_str(stream, buf, "name", b'\n')? + .ok_or_else(|| Error::unknown("unexpected end-of-file"))? .to_string(); - stream.read_exact(&mut buf[..1]) - .or_else(|err| err!("failed to read new line character: {}", err))?; - if buf[0] != b'\n' { - err!("expected new line character, got {:x}", buf[0]) + if name.is_empty() { + err!("tree item name is empty") + } + else if !is_tree_item_name_valid(&name) { + err!("tree item name has invalid character(s)") } else { - Ok(TreeItem { + Ok(Some(TreeItem { name, otype, size, @@ -116,7 +140,7 @@ impl Deserialize for TreeItem { modified, permissions, oid, - }) + })) } } } @@ -133,6 +157,22 @@ impl Serialize for [TreeItem] { } } +impl Deserialize for Vec { + fn deserialize_with_buf(stream: &mut R, buf: &mut Vec) -> Result> { + let mut items = Vec::new(); + while let Some(item) = TreeItem::deserialize_with_buf(stream, buf)? { + items.push(item); + } + + Ok(if items.is_empty() { + None + } + else { + Some(items) + }) + } +} + pub fn otype_from_metadata(metadata: &std::fs::Metadata) -> Result { let file_type = metadata.file_type(); @@ -152,29 +192,45 @@ pub fn otype_from_metadata(metadata: &std::fs::Metadata) -> Result { } -fn read_field(stream: &mut R, buf: &mut Vec, field_name: &str, byte: u8) -> Result<()> { +fn read_field(stream: &mut R, buf: &mut Vec, field_name: &str, byte: u8) -> Result { buf.clear(); stream.read_until(byte, buf) .or_else(|err| err!("failed to read TreeItem {}: {}", field_name, err))?; + if buf.is_empty() { + return Ok(false) + } buf.pop(); - Ok(()) + Ok(true) } -fn read_field_str<'a, R: BufRead>(stream: &mut R, buf: &'a mut Vec, field_name: &str, byte: u8) -> Result<&'a str> { - read_field(stream, buf, field_name, byte)?; - std::str::from_utf8(buf) - .or_else(|err| err!("TreeItem {} is not valid utf-8: {}", field_name, err)) +fn read_field_str<'a, R: BufRead>(stream: &mut R, buf: &'a mut Vec, field_name: &str, byte: u8) -> Result> { + if read_field(stream, buf, field_name, byte)? { + Ok(Some( + std::str::from_utf8(buf) + .or_else(|err| err!("TreeItem {} is not valid utf-8: {}", field_name, err))? + )) + } + else { + Ok(None) + } } -fn read_field_parse(stream: &mut R, buf: &mut Vec, field_name: &str, byte: u8) -> Result +fn read_field_parse(stream: &mut R, buf: &mut Vec, field_name: &str, byte: u8) -> Result> where R: BufRead, I: std::str::FromStr, ::Err: std::fmt::Display { - let int_str = read_field_str(stream, buf, field_name, byte)?; - I::from_str(int_str) - .or_else(|err| err!("failed to parse TreeItem {}: {}", field_name, err)) + let maybe_str = read_field_str(stream, buf, field_name, byte)?; + if let Some(int_str) = maybe_str { + Ok(Some( + I::from_str(int_str) + .or_else(|err| err!("failed to parse TreeItem {}: {}", field_name, err))? + )) + } + else { + Ok(None) + } } @@ -196,7 +252,7 @@ mod tests { permissions: Permissions { read: true, write: false, execute: true }, oid: ObjectId::from_str("0123456789abcdef").unwrap(), }; - let expected = "0123456789abcdef\ttest\t42\tr-x\t1234000\t5678000\tTest $¢ह€한/\n".as_bytes(); + let expected = "0123456789abcdef\ttest\t42\tr-x\t1234000\t5678000\tTest $¢ह€한\n".as_bytes(); let mut result = Vec::new(); item.serialize(&mut result).unwrap(); @@ -208,7 +264,7 @@ mod tests { fn test_deserialize_tree_item() { use std::io::Cursor; - let item_bytes = "0123456789abcdef\ttest\t42\tr-x\t1234000\t5678000\tTest $¢ह€한/\n".as_bytes(); + let item_bytes = "0123456789abcdef\ttest\t42\tr-x\t1234000\t5678000\tTest $¢ह€한\n".as_bytes(); let mut item_cursor = Cursor::new(item_bytes); let expected = TreeItem { name: "Test $¢ह€한".to_string(), @@ -220,19 +276,20 @@ mod tests { oid: ObjectId::from_str("0123456789abcdef").unwrap(), }; - let item = TreeItem::deserialize(&mut item_cursor).unwrap(); + let item = TreeItem::deserialize(&mut item_cursor).unwrap().unwrap(); assert_eq!(item, expected); + assert!(TreeItem::deserialize(&mut Cursor::new("")).unwrap().is_none()); assert!(TreeItem::deserialize(&mut Cursor::new( - "0123456789abcdef\ttest\t42\tr-x\t1234000\t5678000\tTest $¢ह€한/".as_bytes() - )).is_err()); - assert!(TreeItem::deserialize(&mut Cursor::new( - "0123456789abcdef\ttest\t42\tr-x\t1234000\t5678000\tTest $¢ह€한/bar/\n".as_bytes() + "0123456789abcdef\ttest\t42\tr-x\t1234000\t5678000\tTest $¢ह€한".as_bytes() )).is_err()); assert!(TreeItem::deserialize(&mut Cursor::new( - "0123456789abcdef\ttest\t42\tr-x\t1234000\t5678000\tTest $¢ह€한\t\n".as_bytes() + "0123456789abcdef\ttest\t42\tr-x\t1234000\t5678000\tTest $¢ह€한/bar\n".as_bytes() )).is_err()); + // assert!(TreeItem::deserialize(&mut Cursor::new( + // "0123456789abcdef\ttest\t42\tr-x\t1234000\t5678000\tTest $¢ह€한\t\n".as_bytes() + // )).is_err()); assert!(TreeItem::deserialize(&mut Cursor::new( "0123456789abcdef\ttest\t42\tr-x\t5678000\tTest $¢ह€한\t\n".as_bytes() )).is_err()); diff --git a/libbsv/src/tree_walker.rs b/libbsv/src/tree_walker.rs index 505db0a..8983091 100644 --- a/libbsv/src/tree_walker.rs +++ b/libbsv/src/tree_walker.rs @@ -18,10 +18,11 @@ use std::iter::Peekable; use std::fs::read_dir; use std::vec::IntoIter; -use camino::Utf8Path; +use camino::{Utf8Path, Utf8PathBuf}; use cas_core::{Error, ObjectId, Result}; +use crate::{IgnoreAction, IgnoreRules}; use crate::TreeItem; @@ -35,13 +36,19 @@ pub enum Action { } pub struct TreeWalker { + path: Utf8PathBuf, dir_it: Peekable>>, prev_tree_it: Peekable>, + ignore_rules: Option, } impl TreeWalker { - pub fn new>(path: P, prev_tree: Vec) -> Result { + pub fn new>( + path: P, + prev_tree: Vec, + ignore_rules: Option + ) -> Result { let dir_entries = read_dir(path.as_ref().to_path_buf())? .map(|res| res.map_err(|err| err.into())) .collect::>>()?; @@ -70,10 +77,26 @@ impl TreeWalker { }); Ok(Self { + path: path.as_ref().to_path_buf(), dir_it: dir_items.into_iter().peekable(), prev_tree_it: prev_tree.into_iter().peekable(), + ignore_rules: ignore_rules, }) } + + fn test_ignore(&self, item_name: &str, default_action: Action) -> Action { + if let Some(ignore_rules) = self.ignore_rules.as_ref() { + let mut path = self.path.clone(); + path.push(item_name); + match ignore_rules.action_for(path) { + IgnoreAction::Accept => { default_action }, + IgnoreAction::Ignore => { Action::Ignore }, + } + } + else { + default_action + } + } } impl Iterator for TreeWalker { @@ -82,7 +105,8 @@ impl Iterator for TreeWalker { fn next(&mut self) -> Option { match (self.dir_it.peek(), self.prev_tree_it.peek()) { (Some(Err(_)), _) => { - Some(Err(self.dir_it.next().unwrap().unwrap_err())) + let error = self.dir_it.next().unwrap().unwrap_err(); + Some(Err(error)) } (Some(Ok(curr_item)), Some(prev_item)) => { if curr_item.name == prev_item.name { @@ -95,21 +119,28 @@ impl Iterator for TreeWalker { else { Action::Skip }; + let item = self.dir_it.next().unwrap().unwrap(); self.prev_tree_it.next().unwrap(); - Some(Ok((action, self.dir_it.next().unwrap().unwrap()))) + Some(Ok((action, item))) } else if curr_item.name < prev_item.name { - Some(Ok((Action::Add, self.dir_it.next().unwrap().unwrap()))) + let item = self.dir_it.next().unwrap().unwrap(); + let action = self.test_ignore(&item.name, Action::Add); + Some(Ok((action, item))) } else { - Some(Ok((Action::Remove, self.prev_tree_it.next().unwrap()))) + let item = self.prev_tree_it.next().unwrap(); + Some(Ok((Action::Remove, item))) } }, (Some(_), None) => { - Some(Ok((Action::Add, self.dir_it.next().unwrap().unwrap()))) + let item = self.dir_it.next().unwrap().unwrap(); + let action = self.test_ignore(&item.name, Action::Add); + Some(Ok((action, item))) }, (None, Some(_)) => { - Some(Ok((Action::Remove, self.prev_tree_it.next().unwrap()))) + let item = self.prev_tree_it.next().unwrap(); + Some(Ok((Action::Remove, item))) }, (None, None) => None, } @@ -157,11 +188,13 @@ mod tests { let root_dir = tempdir()?; let root = Utf8Path::from_path(root_dir.path()).unwrap(); + let ignore_rules = IgnoreRules::from_source("*.bak", root).unwrap(); + mkdir(root, "test")?; write_file(root, "test/foobar.txt", b"baz")?; write_file(root, "readme", b"hello world!")?; - let items: Vec<_> = TreeWalker::new(root, vec![]).unwrap().collect(); + let items: Vec<_> = TreeWalker::new(root, vec![], Some(ignore_rules.clone())).unwrap().collect(); assert_eq!(items.len(), 2); let (action, item) = items[0].as_ref().unwrap(); assert_eq!(action, &Action::Add); @@ -178,7 +211,7 @@ mod tests { write_file(root, "abc", b"xxxx")?; - let items: Vec<_> = TreeWalker::new(root, to_tree_items(items)).unwrap().collect(); + let items: Vec<_> = TreeWalker::new(root, to_tree_items(items), Some(ignore_rules.clone())).unwrap().collect(); assert_eq!(items.len(), 3); let (action, item) = items[0].as_ref().unwrap(); assert_eq!(action, &Action::Add); @@ -195,7 +228,7 @@ mod tests { remove_file(root, "readme")?; - let items: Vec<_> = TreeWalker::new(root, to_tree_items(items)).unwrap().collect(); + let items: Vec<_> = TreeWalker::new(root, to_tree_items(items), Some(ignore_rules.clone())).unwrap().collect(); assert_eq!(items.len(), 3); let (action, item) = items[0].as_ref().unwrap(); assert_eq!(action, &Action::Skip); @@ -210,7 +243,7 @@ mod tests { write_file(root, "abc", b"ab")?; write_file(root, "test/foobar.txt", b"redacted")?; - let items: Vec<_> = TreeWalker::new(root, to_tree_items(items)).unwrap().collect(); + let items: Vec<_> = TreeWalker::new(root, to_tree_items(items), Some(ignore_rules.clone())).unwrap().collect(); assert_eq!(items.len(), 2); let (action, item) = items[0].as_ref().unwrap(); assert_eq!(action, &Action::Update); @@ -220,6 +253,21 @@ mod tests { assert_eq!(action, &Action::Skip); assert_eq!(item.name, "test"); + write_file(root, "test.bak", b"ignore this")?; + + let items: Vec<_> = TreeWalker::new(root, to_tree_items(items), Some(ignore_rules.clone())).unwrap().collect(); + assert_eq!(items.len(), 3); + let (action, item) = items[0].as_ref().unwrap(); + assert_eq!(action, &Action::Skip); + assert_eq!(item.name, "abc"); + let (action, item) = items[1].as_ref().unwrap(); + assert_eq!(action, &Action::Skip); + assert_eq!(item.name, "test"); + let (action, item) = items[2].as_ref().unwrap(); + assert_eq!(action, &Action::Ignore); + assert_eq!(item.name, "test.bak"); + assert_eq!(item.size, 11); + Ok(()) } }