|
|
@ -17,60 +17,75 @@ |
|
|
use std::str::FromStr; |
|
|
use std::str::FromStr; |
|
|
|
|
|
|
|
|
use digest::DynDigest; |
|
|
use digest::DynDigest; |
|
|
use toml::Value; |
|
|
use toml::{Value, Table}; |
|
|
use camino::{Utf8Path, Utf8PathBuf}; |
|
|
use camino::{Utf8Path, Utf8PathBuf}; |
|
|
use cas_core::{ |
|
|
use cas_core::{ |
|
|
Cas, DefaultPipeline, err, Error, ObjectId, ObjectMetadata, ObjectType, |
|
|
Cas, DefaultPipeline, err, Error, ObjectId, ObjectMetadata, ObjectType, |
|
|
Pipeline, Reader, ReadWrapper, RefStore, Result, Writer, |
|
|
Pipeline, Reader, ReadWrapper, RefStore, Result, Writer, CasWithRef, |
|
|
}; |
|
|
}; |
|
|
|
|
|
|
|
|
use crate::utils::{ |
|
|
use crate::utils::{ |
|
|
obj_dir, obj_path, ref_dir, tmp_dir, |
|
|
obj_dir, obj_path, ref_dir, tmp_dir, |
|
|
read_config, write_config, |
|
|
|
|
|
read_metadata, write_metadata, |
|
|
read_metadata, write_metadata, |
|
|
new_digest, |
|
|
new_digest, |
|
|
}; |
|
|
}; |
|
|
use crate::wfile::WFile; |
|
|
use crate::wfile::WFile; |
|
|
|
|
|
|
|
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)] |
|
|
|
|
|
pub struct SimpleCasConfig { |
|
|
|
|
|
pub digest_id: String, |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
impl SimpleCasConfig { |
|
|
|
|
|
pub fn new(digest_id: String) -> Self { |
|
|
|
|
|
Self { |
|
|
|
|
|
digest_id, |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
pub fn from_toml_value(value: &Value) -> Result<Self> { |
|
|
|
|
|
if let Value::Table(ref table) = value { |
|
|
|
|
|
let engine = |
|
|
|
|
|
table.get("engine") |
|
|
|
|
|
.ok_or_else(|| Error::unknown("missing engine field in cas config"))? |
|
|
|
|
|
.as_str() |
|
|
|
|
|
.ok_or_else(|| Error::unknown("cas engine must be a string"))?; |
|
|
|
|
|
if engine != "simple" { |
|
|
|
|
|
return err!("expected simple cas engine"); |
|
|
|
|
|
} |
|
|
|
|
|
let digest_id = |
|
|
|
|
|
table.get("digest") |
|
|
|
|
|
.ok_or_else(|| Error::unknown("missing digest field in cas config"))? |
|
|
|
|
|
.as_str() |
|
|
|
|
|
.ok_or_else(|| Error::unknown("cas digest must be a string"))?; |
|
|
|
|
|
Ok(Self { |
|
|
|
|
|
digest_id: digest_id.to_string(), |
|
|
|
|
|
}) |
|
|
|
|
|
} |
|
|
|
|
|
else { |
|
|
|
|
|
err!("cas config must be a table") |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
pub fn to_toml_value(&self) -> Result<Value> { |
|
|
|
|
|
let mut table = Table::new(); |
|
|
|
|
|
table["engine"] = "simple".into(); |
|
|
|
|
|
table["digest"] = "sha256".into(); |
|
|
|
|
|
return Ok(Value::Table(table)) |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
pub struct SimpleCas { |
|
|
pub struct SimpleCas { |
|
|
|
|
|
config: SimpleCasConfig, |
|
|
db_path: Utf8PathBuf, |
|
|
db_path: Utf8PathBuf, |
|
|
digest: Box<dyn DynDigest>, |
|
|
digest: Box<dyn DynDigest>, |
|
|
pipeline: DefaultPipeline, |
|
|
pipeline: DefaultPipeline, |
|
|
// config: Value,
|
|
|
|
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
impl SimpleCas { |
|
|
impl SimpleCas { |
|
|
pub fn create(db_path: Utf8PathBuf, mut config: Value) -> Result<Self> { |
|
|
pub fn create(db_path: Utf8PathBuf, config: SimpleCasConfig) -> Result<Self> { |
|
|
if !config.is_table() { |
|
|
|
|
|
return Error::err("invalid config object: must be table"); |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
let maybe_engine = config.as_table_mut().unwrap() |
|
|
|
|
|
.entry("cas") |
|
|
|
|
|
.or_insert_with(|| toml::value::Table::new().into()) |
|
|
|
|
|
.as_table_mut().unwrap() |
|
|
|
|
|
.entry("engine") |
|
|
|
|
|
.or_insert("simple".into()) |
|
|
|
|
|
.as_str(); |
|
|
|
|
|
match maybe_engine { |
|
|
|
|
|
Some(engine) if engine != "simple" => { |
|
|
|
|
|
return err!("invalid cas.engine in config: got {}, expected simple", engine); |
|
|
|
|
|
}, |
|
|
|
|
|
None => { |
|
|
|
|
|
return Error::err("invalid casengine in config: expected String"); |
|
|
|
|
|
}, |
|
|
|
|
|
_ => {} |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
let digest_id = config["cas"]["digest"].as_str() |
|
|
|
|
|
.ok_or_else(|| Error::unknown( |
|
|
|
|
|
"mandatory cas.digest value is invalid or missing from config" |
|
|
|
|
|
))?; |
|
|
|
|
|
let digest = new_digest(digest_id)?; |
|
|
|
|
|
let pipeline = DefaultPipeline::new(digest.box_clone()); |
|
|
|
|
|
|
|
|
|
|
|
if db_path.exists() { |
|
|
if db_path.exists() { |
|
|
return err!( |
|
|
return err!( |
|
|
"failed to create SimpleCas: target directory already exists ({})", |
|
|
"failed to create SimpleCas: target directory already exists ({})", |
|
|
@ -83,8 +98,8 @@ impl SimpleCas { |
|
|
&obj_dir(&db_path), |
|
|
&obj_dir(&db_path), |
|
|
&ref_dir(&db_path), |
|
|
&ref_dir(&db_path), |
|
|
&tmp_dir(&db_path), |
|
|
&tmp_dir(&db_path), |
|
|
] { |
|
|
] { |
|
|
std::fs::create_dir(path).or_else(|e| |
|
|
std::fs::create_dir(path).or_else(|e| |
|
|
err!( |
|
|
err!( |
|
|
"failed to create directory ({}): {}", |
|
|
"failed to create directory ({}): {}", |
|
|
path, e, |
|
|
path, e, |
|
|
@ -92,37 +107,24 @@ impl SimpleCas { |
|
|
)?; |
|
|
)?; |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
write_config(&config, &db_path)?; |
|
|
Self::open(db_path, config) |
|
|
|
|
|
|
|
|
Ok(SimpleCas { |
|
|
|
|
|
db_path, |
|
|
|
|
|
digest, |
|
|
|
|
|
pipeline, |
|
|
|
|
|
// config,
|
|
|
|
|
|
}) |
|
|
|
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
pub fn open(db_path: Utf8PathBuf) -> Result<Self> { |
|
|
pub fn open(db_path: Utf8PathBuf, config: SimpleCasConfig) -> Result<Self> { |
|
|
let config = read_config(&db_path)?; |
|
|
let digest = new_digest(&config.digest_id)?; |
|
|
|
|
|
|
|
|
let digest_id = config["cas"]["digest"].as_str() |
|
|
|
|
|
.ok_or_else(|| Error::unknown( |
|
|
|
|
|
"mandatory cas.digest value is invalid or missing from config" |
|
|
|
|
|
))?; |
|
|
|
|
|
let digest = new_digest(digest_id)?; |
|
|
|
|
|
let pipeline = DefaultPipeline::new(digest.box_clone()); |
|
|
let pipeline = DefaultPipeline::new(digest.box_clone()); |
|
|
|
|
|
|
|
|
Ok(SimpleCas { |
|
|
Ok(SimpleCas { |
|
|
|
|
|
config, |
|
|
db_path, |
|
|
db_path, |
|
|
digest, |
|
|
digest, |
|
|
pipeline, |
|
|
pipeline, |
|
|
// config,
|
|
|
|
|
|
}) |
|
|
}) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
// pub fn save_config(&self) -> Result<()> {
|
|
|
pub fn get_config(&self) -> &SimpleCasConfig { |
|
|
// write_config(&self.config, &self.db_path)
|
|
|
&self.config |
|
|
// }
|
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -152,7 +154,7 @@ impl Cas for SimpleCas { |
|
|
fn open_object(&self, oid: &ObjectId) -> Result<(ObjectMetadata, Box<dyn Reader>)> { |
|
|
fn open_object(&self, oid: &ObjectId) -> Result<(ObjectMetadata, Box<dyn Reader>)> { |
|
|
let opath = obj_path(&self.db_path, oid); |
|
|
let opath = obj_path(&self.db_path, oid); |
|
|
if !opath.is_file() { |
|
|
if !opath.is_file() { |
|
|
return err!("object not found: {}", oid); |
|
|
return Err(Error::ObjectDoesNotExists(oid.clone())); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
let file = std::fs::File::open(opath).or_else(|err| |
|
|
let file = std::fs::File::open(opath).or_else(|err| |
|
|
@ -184,56 +186,59 @@ impl Cas for SimpleCas { |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
impl RefStore for SimpleCas { |
|
|
impl RefStore for SimpleCas { |
|
|
fn get_ref<P: AsRef<Utf8Path>>(&self, key: P) -> Result<ObjectId> { |
|
|
fn get_ref(&self, key: &Utf8Path) -> Result<ObjectId> { |
|
|
let path = ref_dir(&self.db_path).join(key.as_ref()); |
|
|
let path = ref_dir(&self.db_path).join(key); |
|
|
if !path.exists() { |
|
|
if !path.exists() { |
|
|
err!("reference {} does not exists", key.as_ref()) |
|
|
err!("reference {} does not exists", key) |
|
|
} |
|
|
} |
|
|
else if !path.is_file() { |
|
|
else if !path.is_file() { |
|
|
err!("reference {} is not a file", key.as_ref()) |
|
|
err!("reference {} is not a file", key) |
|
|
} |
|
|
} |
|
|
else { |
|
|
else { |
|
|
let file = std::fs::read(path).or_else(|err| |
|
|
let file = std::fs::read(path).or_else(|err| |
|
|
err!("failed to read reference file for {}: {}", key.as_ref(), err) |
|
|
err!("failed to read reference file for {}: {}", key, err) |
|
|
)?; |
|
|
)?; |
|
|
Ok( |
|
|
Ok( |
|
|
ObjectId::from_str( |
|
|
ObjectId::from_str( |
|
|
std::str::from_utf8(&file).or_else(|err| |
|
|
std::str::from_utf8(&file).or_else(|err| |
|
|
err!("invalid reference file at {}: {}", key.as_ref(), err) |
|
|
err!("invalid reference file at {}: {}", key, err) |
|
|
)? |
|
|
)? |
|
|
)? |
|
|
)? |
|
|
) |
|
|
) |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
fn set_ref<P: AsRef<Utf8Path>>(&mut self, key: P, value: &ObjectId) -> Result<()> { |
|
|
fn set_ref(&mut self, key: &Utf8Path, value: &ObjectId) -> Result<()> { |
|
|
let path = ref_dir(&self.db_path).join(key.as_ref()); |
|
|
let path = ref_dir(&self.db_path).join(key); |
|
|
std::fs::create_dir_all(path.parent().ok_or_else(|| |
|
|
std::fs::create_dir_all(path.parent().ok_or_else(|| |
|
|
Error::unknown(format!("reference file {} has no parent dir?", key.as_ref())) |
|
|
Error::unknown(format!("reference file {} has no parent dir?", key)) |
|
|
)?).or_else(|err| |
|
|
)?).or_else(|err| |
|
|
err!("failed to create reference dir for {}: {}", key.as_ref(), err) |
|
|
err!("failed to create reference dir for {}: {}", key, err) |
|
|
)?; |
|
|
)?; |
|
|
std::fs::write(path, value.to_string()).or_else(|err| |
|
|
std::fs::write(path, value.to_string()).or_else(|err| |
|
|
err!("failed to write reference {}: {}", key.as_ref(), err) |
|
|
err!("failed to write reference {}: {}", key, err) |
|
|
) |
|
|
) |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
fn remove_ref<P: AsRef<Utf8Path>>(&mut self, key: P) -> Result<()> { |
|
|
fn remove_ref(&mut self, key: &Utf8Path) -> Result<()> { |
|
|
let path = ref_dir(&self.db_path).join(key.as_ref()); |
|
|
let path = ref_dir(&self.db_path).join(key); |
|
|
if !path.exists() { |
|
|
if !path.exists() { |
|
|
err!("reference {} does not exists", key.as_ref()) |
|
|
err!("reference {} does not exists", key) |
|
|
} |
|
|
} |
|
|
else if !path.is_file() { |
|
|
else if !path.is_file() { |
|
|
err!("reference {} is not a file", key.as_ref()) |
|
|
err!("reference {} is not a file", key) |
|
|
} |
|
|
} |
|
|
else { |
|
|
else { |
|
|
std::fs::remove_file(path).or_else(|err| |
|
|
std::fs::remove_file(path).or_else(|err| |
|
|
err!("failed to remove reference file {}: {}", key.as_ref(), err) |
|
|
err!("failed to remove reference file {}: {}", key, err) |
|
|
) |
|
|
) |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
impl CasWithRef for SimpleCas { |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
pub struct ObjectIdIterator { |
|
|
pub struct ObjectIdIterator { |
|
|
root_dirs: Vec<Utf8PathBuf>, |
|
|
root_dirs: Vec<Utf8PathBuf>, |
|
|
@ -387,13 +392,6 @@ mod tests { |
|
|
|
|
|
|
|
|
use super::*; |
|
|
use super::*; |
|
|
|
|
|
|
|
|
fn get_config() -> Value { |
|
|
|
|
|
toml::toml!( |
|
|
|
|
|
[cas] |
|
|
|
|
|
digest = "sha256" |
|
|
|
|
|
) |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
fn get_cas_path(dir: &Utf8Path) -> Utf8PathBuf { |
|
|
fn get_cas_path(dir: &Utf8Path) -> Utf8PathBuf { |
|
|
let mut cas_path = dir.to_path_buf(); |
|
|
let mut cas_path = dir.to_path_buf(); |
|
|
cas_path.push(".bsv"); |
|
|
cas_path.push(".bsv"); |
|
|
@ -404,7 +402,7 @@ mod tests { |
|
|
fn test_create_simple_cas() { |
|
|
fn test_create_simple_cas() { |
|
|
let dir = tempfile::tempdir().expect("failed to create temp test dir"); |
|
|
let dir = tempfile::tempdir().expect("failed to create temp test dir"); |
|
|
let cas_path = get_cas_path(Utf8Path::from_path(dir.path()).unwrap()); |
|
|
let cas_path = get_cas_path(Utf8Path::from_path(dir.path()).unwrap()); |
|
|
let config = get_config(); |
|
|
let config = SimpleCasConfig::new("sha256".into()); |
|
|
|
|
|
|
|
|
let cas = SimpleCas::create(cas_path, config) |
|
|
let cas = SimpleCas::create(cas_path, config) |
|
|
.expect("failed to create SimpleCas object"); |
|
|
.expect("failed to create SimpleCas object"); |
|
|
@ -418,7 +416,7 @@ mod tests { |
|
|
fn test_write_object() { |
|
|
fn test_write_object() { |
|
|
let dir = tempfile::tempdir().expect("failed to create temp test dir"); |
|
|
let dir = tempfile::tempdir().expect("failed to create temp test dir"); |
|
|
let cas_path = get_cas_path(dir.path().try_into().unwrap()); |
|
|
let cas_path = get_cas_path(dir.path().try_into().unwrap()); |
|
|
let config = get_config(); |
|
|
let config = SimpleCasConfig::new("sha256".into()); |
|
|
|
|
|
|
|
|
let otype = ObjectType::new(b"blob").expect("failed to create object type"); |
|
|
let otype = ObjectType::new(b"blob").expect("failed to create object type"); |
|
|
let payload = b"Hello World!"; |
|
|
let payload = b"Hello World!"; |
|
|
@ -448,7 +446,7 @@ mod tests { |
|
|
|
|
|
|
|
|
let dir = tempfile::tempdir().expect("failed to create temp test dir"); |
|
|
let dir = tempfile::tempdir().expect("failed to create temp test dir"); |
|
|
let cas_path = get_cas_path(dir.path().try_into().unwrap()); |
|
|
let cas_path = get_cas_path(dir.path().try_into().unwrap()); |
|
|
let config = get_config(); |
|
|
let config = SimpleCasConfig::new("sha256".into()); |
|
|
|
|
|
|
|
|
let otype = ObjectType::new(b"blob").expect("failed to create object type"); |
|
|
let otype = ObjectType::new(b"blob").expect("failed to create object type"); |
|
|
let payload = b"Hello World!"; |
|
|
let payload = b"Hello World!"; |
|
|
@ -487,7 +485,7 @@ mod tests { |
|
|
fn test_read_write_object() { |
|
|
fn test_read_write_object() { |
|
|
let dir = tempfile::tempdir().expect("failed to create temp test dir"); |
|
|
let dir = tempfile::tempdir().expect("failed to create temp test dir"); |
|
|
let cas_path = get_cas_path(dir.path().try_into().unwrap()); |
|
|
let cas_path = get_cas_path(dir.path().try_into().unwrap()); |
|
|
let config = get_config(); |
|
|
let config = SimpleCasConfig::new("sha256".into()); |
|
|
|
|
|
|
|
|
let otype = ObjectType::new(b"blob").expect("failed to create object type"); |
|
|
let otype = ObjectType::new(b"blob").expect("failed to create object type"); |
|
|
let payload = b"This is a test."; |
|
|
let payload = b"This is a test."; |
|
|
@ -507,7 +505,7 @@ mod tests { |
|
|
fn test_remove_object() { |
|
|
fn test_remove_object() { |
|
|
let dir = tempfile::tempdir().expect("failed to create temp test dir"); |
|
|
let dir = tempfile::tempdir().expect("failed to create temp test dir"); |
|
|
let cas_path = get_cas_path(dir.path().try_into().unwrap()); |
|
|
let cas_path = get_cas_path(dir.path().try_into().unwrap()); |
|
|
let config = get_config(); |
|
|
let config = SimpleCasConfig::new("sha256".into()); |
|
|
|
|
|
|
|
|
let otype = ObjectType::new(b"blob").expect("failed to create object type"); |
|
|
let otype = ObjectType::new(b"blob").expect("failed to create object type"); |
|
|
let payload = b"This is a test."; |
|
|
let payload = b"This is a test."; |
|
|
@ -528,7 +526,7 @@ mod tests { |
|
|
fn test_object_id_iterator() { |
|
|
fn test_object_id_iterator() { |
|
|
let dir = tempfile::tempdir().expect("failed to create temp test dir"); |
|
|
let dir = tempfile::tempdir().expect("failed to create temp test dir"); |
|
|
let cas_path = get_cas_path(dir.path().try_into().unwrap()); |
|
|
let cas_path = get_cas_path(dir.path().try_into().unwrap()); |
|
|
let config = get_config(); |
|
|
let config = SimpleCasConfig::new("sha256".into()); |
|
|
|
|
|
|
|
|
let mut cas = SimpleCas::create(cas_path.clone(), config) |
|
|
let mut cas = SimpleCas::create(cas_path.clone(), config) |
|
|
.expect("failed to create SimpleCas object"); |
|
|
.expect("failed to create SimpleCas object"); |
|
|
@ -642,21 +640,21 @@ mod tests { |
|
|
fn test_reference() { |
|
|
fn test_reference() { |
|
|
let dir = tempfile::tempdir().expect("failed to create temp test dir"); |
|
|
let dir = tempfile::tempdir().expect("failed to create temp test dir"); |
|
|
let cas_path = get_cas_path(dir.path().try_into().unwrap()); |
|
|
let cas_path = get_cas_path(dir.path().try_into().unwrap()); |
|
|
let config = get_config(); |
|
|
let config = SimpleCasConfig::new("sha256".into()); |
|
|
|
|
|
|
|
|
let mut cas = SimpleCas::create(cas_path.clone(), config) |
|
|
let mut cas = SimpleCas::create(cas_path.clone(), config) |
|
|
.expect("failed to create SimpleCas object"); |
|
|
.expect("failed to create SimpleCas object"); |
|
|
|
|
|
|
|
|
let oid_0 = ObjectId::from_str("f731f6bc6a6a73bad170e56452473ef6930b7a0ab33cc54be44221a89b49d786").unwrap(); |
|
|
let oid_0 = ObjectId::from_str("f731f6bc6a6a73bad170e56452473ef6930b7a0ab33cc54be44221a89b49d786").unwrap(); |
|
|
|
|
|
|
|
|
assert!(cas.get_ref("foo/bar").is_err()); |
|
|
assert!(cas.get_ref("foo/bar".into()).is_err()); |
|
|
assert!(cas.remove_ref("foo/bar").is_err()); |
|
|
assert!(cas.remove_ref("foo/bar".into()).is_err()); |
|
|
|
|
|
|
|
|
cas.set_ref("foo/bar", &oid_0).unwrap(); |
|
|
cas.set_ref("foo/bar".into(), &oid_0).unwrap(); |
|
|
assert_eq!(cas.get_ref("foo/bar").unwrap(), oid_0); |
|
|
assert_eq!(cas.get_ref("foo/bar".into()).unwrap(), oid_0); |
|
|
|
|
|
|
|
|
cas.remove_ref("foo/bar").unwrap(); |
|
|
cas.remove_ref("foo/bar".into()).unwrap(); |
|
|
|
|
|
|
|
|
assert!(cas.get_ref("foo/bar").is_err()); |
|
|
assert!(cas.get_ref("foo/bar".into()).is_err()); |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
|
