draklaw/bsv
1
0
Fork 0
Code Issues 6 Pull Requests Projects 1 Releases Wiki Activity

Mostly improved tree walking I guess.

Browse Source
This commit is contained in:
Draklaw
2023-10-21 19:55:17 +02:00
parent 2b27b788e5
commit 25c7d0d470
17 changed files with 554 additions and 346 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cas-simple/Cargo.toml
View File

@@ -11,8 +11,6 @@ license = "AGPL-3.0-or-later"
digest = { version = "0.9.0", features = ["alloc"] }
sha2 = "0.9.5"
camino = { version = "1.0.7" }
toml = "0.5.8"
toml = "0.7.6"
cas-core = { path = "../cas-core" }
tempfile = "3.2.0"
[dev-dependencies]

182
cas-simple/src/cas.rs
View File

@@ -17,60 +17,75 @@
use std::str::FromStr;
use digest::DynDigest;
use toml::Value;
use toml::{Value, Table};
use camino::{Utf8Path, Utf8PathBuf};
use cas_core::{
Cas, DefaultPipeline, err, Error, ObjectId, ObjectMetadata, ObjectType,
Pipeline, Reader, ReadWrapper, RefStore, Result, Writer,
Pipeline, Reader, ReadWrapper, RefStore, Result, Writer, CasWithRef,
};
use crate::utils::{
obj_dir, obj_path, ref_dir, tmp_dir,
read_config, write_config,
read_metadata, write_metadata,
new_digest,
};
use crate::wfile::WFile;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SimpleCasConfig {
pub digest_id: String,
}
impl SimpleCasConfig {
pub fn new(digest_id: String) -> Self {
Self {
digest_id,
}
}
pub fn from_toml_value(value: &Value) -> Result<Self> {
if let Value::Table(ref table) = value {
let engine =
table.get("engine")
.ok_or_else(|| Error::unknown("missing engine field in cas config"))?
.as_str()
.ok_or_else(|| Error::unknown("cas engine must be a string"))?;
if engine != "simple" {
return err!("expected simple cas engine");
}
let digest_id =
table.get("digest")
.ok_or_else(|| Error::unknown("missing digest field in cas config"))?
.as_str()
.ok_or_else(|| Error::unknown("cas digest must be a string"))?;
Ok(Self {
digest_id: digest_id.to_string(),
})
}
else {
err!("cas config must be a table")
}
}
pub fn to_toml_value(&self) -> Result<Value> {
let mut table = Table::new();
table["engine"] = "simple".into();
table["digest"] = "sha256".into();
return Ok(Value::Table(table))
}
}
pub struct SimpleCas {
config: SimpleCasConfig,
db_path: Utf8PathBuf,
digest: Box<dyn DynDigest>,
pipeline: DefaultPipeline,
// config: Value,
}
impl SimpleCas {
pub fn create(db_path: Utf8PathBuf, mut config: Value) -> Result<Self> {
if !config.is_table() {
return Error::err("invalid config object: must be table");
}
let maybe_engine = config.as_table_mut().unwrap()
.entry("cas")
.or_insert_with(|| toml::value::Table::new().into())
.as_table_mut().unwrap()
.entry("engine")
.or_insert("simple".into())
.as_str();
match maybe_engine {
Some(engine) if engine != "simple" => {
return err!("invalid cas.engine in config: got {}, expected simple", engine);
},
None => {
return Error::err("invalid casengine in config: expected String");
},
_ => {}
}
let digest_id = config["cas"]["digest"].as_str()
.ok_or_else(|| Error::unknown(
"mandatory cas.digest value is invalid or missing from config"
))?;
let digest = new_digest(digest_id)?;
let pipeline = DefaultPipeline::new(digest.box_clone());
pub fn create(db_path: Utf8PathBuf, config: SimpleCasConfig) -> Result<Self> {
if db_path.exists() {
return err!(
"failed to create SimpleCas: target directory already exists ({})",
@@ -83,8 +98,8 @@ impl SimpleCas {
&obj_dir(&db_path),
&ref_dir(&db_path),
&tmp_dir(&db_path),
] {
std::fs::create_dir(path).or_else(|e|
] {
std::fs::create_dir(path).or_else(|e|
err!(
"failed to create directory ({}): {}",
path, e,
@@ -92,37 +107,24 @@ impl SimpleCas {
)?;
}
write_config(&config, &db_path)?;
Ok(SimpleCas {
db_path,
digest,
pipeline,
// config,
})
Self::open(db_path, config)
}
pub fn open(db_path: Utf8PathBuf) -> Result<Self> {
let config = read_config(&db_path)?;
let digest_id = config["cas"]["digest"].as_str()
.ok_or_else(|| Error::unknown(
"mandatory cas.digest value is invalid or missing from config"
))?;
let digest = new_digest(digest_id)?;
pub fn open(db_path: Utf8PathBuf, config: SimpleCasConfig) -> Result<Self> {
let digest = new_digest(&config.digest_id)?;
let pipeline = DefaultPipeline::new(digest.box_clone());
Ok(SimpleCas {
config,
db_path,
digest,
pipeline,
// config,
})
}
// pub fn save_config(&self) -> Result<()> {
// write_config(&self.config, &self.db_path)
// }
pub fn get_config(&self) -> &SimpleCasConfig {
&self.config
}
}
@@ -152,7 +154,7 @@ impl Cas for SimpleCas {
fn open_object(&self, oid: &ObjectId) -> Result<(ObjectMetadata, Box<dyn Reader>)> {
let opath = obj_path(&self.db_path, oid);
if !opath.is_file() {
return err!("object not found: {}", oid);
return Err(Error::ObjectDoesNotExists(oid.clone()));
}
let file = std::fs::File::open(opath).or_else(|err|
@@ -184,56 +186,59 @@ impl Cas for SimpleCas {
}
impl RefStore for SimpleCas {
fn get_ref<P: AsRef<Utf8Path>>(&self, key: P) -> Result<ObjectId> {
let path = ref_dir(&self.db_path).join(key.as_ref());
fn get_ref(&self, key: &Utf8Path) -> Result<ObjectId> {
let path = ref_dir(&self.db_path).join(key);
if !path.exists() {
err!("reference {} does not exists", key.as_ref())
err!("reference {} does not exists", key)
}
else if !path.is_file() {
err!("reference {} is not a file", key.as_ref())
err!("reference {} is not a file", key)
}
else {
let file = std::fs::read(path).or_else(|err|
err!("failed to read reference file for {}: {}", key.as_ref(), err)
err!("failed to read reference file for {}: {}", key, err)
)?;
Ok(
ObjectId::from_str(
std::str::from_utf8(&file).or_else(|err|
err!("invalid reference file at {}: {}", key.as_ref(), err)
err!("invalid reference file at {}: {}", key, err)
)?
)?
)
}
}
fn set_ref<P: AsRef<Utf8Path>>(&mut self, key: P, value: &ObjectId) -> Result<()> {
let path = ref_dir(&self.db_path).join(key.as_ref());
fn set_ref(&mut self, key: &Utf8Path, value: &ObjectId) -> Result<()> {
let path = ref_dir(&self.db_path).join(key);
std::fs::create_dir_all(path.parent().ok_or_else(||
Error::unknown(format!("reference file {} has no parent dir?", key.as_ref()))
Error::unknown(format!("reference file {} has no parent dir?", key))
)?).or_else(|err|
err!("failed to create reference dir for {}: {}", key.as_ref(), err)
err!("failed to create reference dir for {}: {}", key, err)
)?;
std::fs::write(path, value.to_string()).or_else(|err|
err!("failed to write reference {}: {}", key.as_ref(), err)
err!("failed to write reference {}: {}", key, err)
)
}
fn remove_ref<P: AsRef<Utf8Path>>(&mut self, key: P) -> Result<()> {
let path = ref_dir(&self.db_path).join(key.as_ref());
fn remove_ref(&mut self, key: &Utf8Path) -> Result<()> {
let path = ref_dir(&self.db_path).join(key);
if !path.exists() {
err!("reference {} does not exists", key.as_ref())
err!("reference {} does not exists", key)
}
else if !path.is_file() {
err!("reference {} is not a file", key.as_ref())
err!("reference {} is not a file", key)
}
else {
std::fs::remove_file(path).or_else(|err|
err!("failed to remove reference file {}: {}", key.as_ref(), err)
err!("failed to remove reference file {}: {}", key, err)
)
}
}
}
impl CasWithRef for SimpleCas {
}
pub struct ObjectIdIterator {
root_dirs: Vec<Utf8PathBuf>,
@@ -387,13 +392,6 @@ mod tests {
use super::*;
fn get_config() -> Value {
toml::toml!(
[cas]
digest = "sha256"
)
}
fn get_cas_path(dir: &Utf8Path) -> Utf8PathBuf {
let mut cas_path = dir.to_path_buf();
cas_path.push(".bsv");
@@ -404,7 +402,7 @@ mod tests {
fn test_create_simple_cas() {
let dir = tempfile::tempdir().expect("failed to create temp test dir");
let cas_path = get_cas_path(Utf8Path::from_path(dir.path()).unwrap());
let config = get_config();
let config = SimpleCasConfig::new("sha256".into());
let cas = SimpleCas::create(cas_path, config)
.expect("failed to create SimpleCas object");
@@ -418,7 +416,7 @@ mod tests {
fn test_write_object() {
let dir = tempfile::tempdir().expect("failed to create temp test dir");
let cas_path = get_cas_path(dir.path().try_into().unwrap());
let config = get_config();
let config = SimpleCasConfig::new("sha256".into());
let otype = ObjectType::new(b"blob").expect("failed to create object type");
let payload = b"Hello World!";
@@ -448,7 +446,7 @@ mod tests {
let dir = tempfile::tempdir().expect("failed to create temp test dir");
let cas_path = get_cas_path(dir.path().try_into().unwrap());
let config = get_config();
let config = SimpleCasConfig::new("sha256".into());
let otype = ObjectType::new(b"blob").expect("failed to create object type");
let payload = b"Hello World!";
@@ -487,7 +485,7 @@ mod tests {
fn test_read_write_object() {
let dir = tempfile::tempdir().expect("failed to create temp test dir");
let cas_path = get_cas_path(dir.path().try_into().unwrap());
let config = get_config();
let config = SimpleCasConfig::new("sha256".into());
let otype = ObjectType::new(b"blob").expect("failed to create object type");
let payload = b"This is a test.";
@@ -507,7 +505,7 @@ mod tests {
fn test_remove_object() {
let dir = tempfile::tempdir().expect("failed to create temp test dir");
let cas_path = get_cas_path(dir.path().try_into().unwrap());
let config = get_config();
let config = SimpleCasConfig::new("sha256".into());
let otype = ObjectType::new(b"blob").expect("failed to create object type");
let payload = b"This is a test.";
@@ -528,7 +526,7 @@ mod tests {
fn test_object_id_iterator() {
let dir = tempfile::tempdir().expect("failed to create temp test dir");
let cas_path = get_cas_path(dir.path().try_into().unwrap());
let config = get_config();
let config = SimpleCasConfig::new("sha256".into());
let mut cas = SimpleCas::create(cas_path.clone(), config)
.expect("failed to create SimpleCas object");
@@ -642,21 +640,21 @@ mod tests {
fn test_reference() {
let dir = tempfile::tempdir().expect("failed to create temp test dir");
let cas_path = get_cas_path(dir.path().try_into().unwrap());
let config = get_config();
let config = SimpleCasConfig::new("sha256".into());
let mut cas = SimpleCas::create(cas_path.clone(), config)
.expect("failed to create SimpleCas object");
let oid_0 = ObjectId::from_str("f731f6bc6a6a73bad170e56452473ef6930b7a0ab33cc54be44221a89b49d786").unwrap();
assert!(cas.get_ref("foo/bar").is_err());
assert!(cas.remove_ref("foo/bar").is_err());
assert!(cas.get_ref("foo/bar".into()).is_err());
assert!(cas.remove_ref("foo/bar".into()).is_err());
cas.set_ref("foo/bar", &oid_0).unwrap();
assert_eq!(cas.get_ref("foo/bar").unwrap(), oid_0);
cas.set_ref("foo/bar".into(), &oid_0).unwrap();
assert_eq!(cas.get_ref("foo/bar".into()).unwrap(), oid_0);
cas.remove_ref("foo/bar").unwrap();
cas.remove_ref("foo/bar".into()).unwrap();
assert!(cas.get_ref("foo/bar").is_err());
assert!(cas.get_ref("foo/bar".into()).is_err());
}
}

2
cas-simple/src/lib.rs
View File

@@ -32,4 +32,4 @@ mod wfile;
mod cas;
pub use cas::SimpleCas;
pub use cas::{SimpleCas, SimpleCasConfig};

96
cas-simple/src/utils.rs
View File

@@ -16,7 +16,7 @@
use digest::DynDigest;
use toml::Value;
// use toml::Value;
use camino::{Utf8Path, Utf8PathBuf};
use cas_core::{
@@ -49,39 +49,39 @@ pub fn config_path(cas_path: &Utf8Path) -> Utf8PathBuf {
}
pub fn read_config(db_path: &Utf8Path) -> Result<Value> {
use std::io::Read;
// pub fn read_config(db_path: &Utf8Path) -> Result<Value> {
// use std::io::Read;
let mut file = std::fs::File::open(config_path(db_path)).or_else(|err|
err!("invalid repository: no config file: {}", err)
)?;
let mut config_str = String::new();
file.read_to_string(&mut config_str).or_else(|err|
err!("cannot read config file: {}", err)
)?;
let config = toml::from_str(&config_str).or_else(|err|
err!("error while reading config file: {}", err)
)?;
Ok(config)
}
// let mut file = std::fs::File::open(config_path(db_path)).or_else(|err|
// err!("invalid repository: no config file: {}", err)
// )?;
// let mut config_str = String::new();
// file.read_to_string(&mut config_str).or_else(|err|
// err!("cannot read config file: {}", err)
// )?;
// let config = toml::from_str(&config_str).or_else(|err|
// err!("error while reading config file: {}", err)
// )?;
// Ok(config)
// }
pub fn write_config(config: &Value, db_path: &Utf8Path) -> Result<()> {
use std::io::Write;
// pub fn write_config(config: &Value, db_path: &Utf8Path) -> Result<()> {
// use std::io::Write;
let config_str = toml::to_string_pretty(config).or_else(|err|
err!("failed to serialize config: {}", err)
)?;
let mut file = tempfile::NamedTempFile::new_in(tmp_dir(db_path)).or_else(|err|
err!("cannot create temp config file: {}", err)
)?;
file.write_all(config_str.as_bytes()).or_else(|err|
err!("failed to write to temp config: {}", err)
)?;
file.persist(config_path(db_path)).or_else(|err|
err!("failed to (over)write config: {}", err)
)?;
Ok(())
}
// let config_str = toml::to_string_pretty(config).or_else(|err|
// err!("failed to serialize config: {}", err)
// )?;
// let mut file = tempfile::NamedTempFile::new_in(tmp_dir(db_path)).or_else(|err|
// err!("cannot create temp config file: {}", err)
// )?;
// file.write_all(config_str.as_bytes()).or_else(|err|
// err!("failed to write to temp config: {}", err)
// )?;
// file.persist(config_path(db_path)).or_else(|err|
// err!("failed to (over)write config: {}", err)
// )?;
// Ok(())
// }
pub fn read_metadata(read: &mut dyn std::io::Read) -> Result<ObjectMetadata> {
@@ -151,28 +151,28 @@ mod tests {
);
}
#[test]
fn test_read_write_config() {
let config = toml::toml!{
[cas]
path = "/foo/bar"
digest = "sha1"
// #[test]
// fn test_read_write_config() {
// let config = toml::toml!{
// [cas]
// path = "/foo/bar"
// digest = "sha1"
[extra]
test = 42
};
// [extra]
// test = 42
// };
let dir = tempfile::TempDir::new()
.expect("failed to create tmp dir");
std::fs::create_dir(tmp_dir(Utf8Path::from_path(dir.path()).unwrap())).expect("failed to create db/tmp dir");
// let dir = tempfile::TempDir::new()
// .expect("failed to create tmp dir");
// std::fs::create_dir(tmp_dir(Utf8Path::from_path(dir.path()).unwrap())).expect("failed to create db/tmp dir");
write_config(&config, Utf8Path::from_path(dir.path()).unwrap()).expect("failed to write config");
let config2 = read_config(Utf8Path::from_path(dir.path()).unwrap()).expect("failed to read config");
// write_config(&config, Utf8Path::from_path(dir.path()).unwrap()).expect("failed to write config");
// let config2 = read_config(Utf8Path::from_path(dir.path()).unwrap()).expect("failed to read config");
assert_eq!(config2, config);
// assert_eq!(config2, config);
dir.close().expect("failed to close tmp dir")
}
// dir.close().expect("failed to close tmp dir")
// }
#[test]
fn test_read_write_metadata() {