draklaw/bsv
1
0
Fork 0
Code Issues 6 Pull Requests Projects 1 Releases Wiki Activity

Better error management, Utf8Paths & other stuff.

Browse Source
This commit is contained in:
Draklaw
2022-08-08 11:40:45 +02:00
parent b1ceaaf636
commit 5d08b1ea57
33 changed files with 1483 additions and 1216 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
cas-simple/Cargo.toml
View File

@@ -1,13 +1,16 @@
[package]
name = "cas-simple"
version = "0.1.0"
edition = "2018"
authors = ["Simon Boyé <sim.boye@gmail.com>"]
edition = "2021"
license = "AGPL-3.0-or-later"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
digest = { version = "0.9.0", features = ["alloc"] }
sha2 = "0.9.5"
camino = { version = "1.0.7" }
toml = "0.5.8"
cas-core = { path = "../cas-core" }
tempfile = "3.2.0"

146
cas-simple/src/cas.rs
View File

@@ -15,12 +15,12 @@
use std::str::FromStr;
use std::path::{Path, PathBuf};
use digest::DynDigest;
use toml::Value;
use camino::{Utf8Path, Utf8PathBuf};
use cas_core::{
Cas, DefaultPipeline, Error, ObjectId, ObjectMetadata, ObjectType,
Cas, DefaultPipeline, err, Error, ObjectId, ObjectMetadata, ObjectType,
Pipeline, Reader, ReadWrapper, RefStore, Result, Writer,
};
@@ -34,27 +34,48 @@ use crate::wfile::WFile;
pub struct SimpleCas {
db_path: PathBuf,
db_path: Utf8PathBuf,
digest: Box<dyn DynDigest>,
pipeline: DefaultPipeline,
config: Value,
// config: Value,
}
impl SimpleCas {
pub fn create(db_path: PathBuf, config: Value) -> Result<Self> {
pub fn create(db_path: Utf8PathBuf, mut config: Value) -> Result<Self> {
if !config.is_table() {
return Error::err("invalid config object: must be table");
}
let maybe_engine = config.as_table_mut().unwrap()
.entry("cas")
.or_insert_with(|| toml::value::Table::new().into())
.as_table_mut().unwrap()
.entry("engine")
.or_insert("simple".into())
.as_str();
match maybe_engine {
Some(engine) if engine != "simple" => {
return err!("invalid cas.engine in config: got {}, expected simple", engine);
},
None => {
return Error::err("invalid casengine in config: expected String");
},
_ => {}
}
let digest_id = config["cas"]["digest"].as_str()
.ok_or_else(|| Error::error(
.ok_or_else(|| Error::unknown(
"mandatory cas.digest value is invalid or missing from config"
))?;
let digest = new_digest(digest_id)?;
let pipeline = DefaultPipeline::new(digest.box_clone());
if db_path.exists() {
return Err(Error::error(format!(
return err!(
"failed to create SimpleCas: target directory already exists ({})",
db_path.to_string_lossy(),
)));
db_path,
);
}
for path in [
@@ -64,10 +85,10 @@ impl SimpleCas {
&tmp_dir(&db_path),
] {
std::fs::create_dir(path).or_else(|e|
Err(Error::error(format!(
err!(
"failed to create directory ({}): {}",
path.to_string_lossy(), e,
)))
path, e,
)
)?;
}
@@ -77,15 +98,15 @@ impl SimpleCas {
db_path,
digest,
pipeline,
config,
// config,
})
}
pub fn open(db_path: PathBuf) -> Result<Self> {
pub fn open(db_path: Utf8PathBuf) -> Result<Self> {
let config = read_config(&db_path)?;
let digest_id = config["cas"]["digest"].as_str()
.ok_or_else(|| Error::error(
.ok_or_else(|| Error::unknown(
"mandatory cas.digest value is invalid or missing from config"
))?;
let digest = new_digest(digest_id)?;
@@ -95,13 +116,13 @@ impl SimpleCas {
db_path,
digest,
pipeline,
config,
// config,
})
}
pub fn save_config(&self) -> Result<()> {
write_config(&self.config, &self.db_path)
}
// pub fn save_config(&self) -> Result<()> {
// write_config(&self.config, &self.db_path)
// }
}
@@ -111,10 +132,10 @@ impl Cas for SimpleCas {
ObjectId::from_str(hex)
}
else {
Err(Error::error(format!(
err!(
"invalid object id size: got {}, expected {}",
hex.len(), self.digest.output_size() * 2,
)))
)
}
}
@@ -131,11 +152,11 @@ impl Cas for SimpleCas {
fn open_object(&self, oid: &ObjectId) -> Result<(ObjectMetadata, Box<dyn Reader>)> {
let opath = obj_path(&self.db_path, oid);
if !opath.is_file() {
return Err(Error::error(format!("object not found: {}", oid)));
return err!("object not found: {}", oid);
}
let file = std::fs::File::open(opath).or_else(|err|
Err(Error::error(format!("failed to open object {}: {}", oid, err))))?;
err!("failed to open object {}: {}", oid, err))?;
let wrapper = Box::new(ReadWrapper::new(file));
let mut reader = self.pipeline.new_reader(wrapper);
let metadata = read_metadata(&mut reader)?;
@@ -154,60 +175,60 @@ impl Cas for SimpleCas {
fn remove_object(&mut self, oid: &ObjectId) -> Result<()> {
let opath = obj_path(&self.db_path, oid);
if !opath.is_file() {
return Err(Error::error(format!("object not found: {}", oid)));
return err!("object not found: {}", oid);
}
std::fs::remove_file(opath).or_else(|err|
Err(Error::error(format!("failed to remove object {}: {}", oid, err))))?;
err!("failed to remove object {}: {}", oid, err))?;
Ok(())
}
}
impl RefStore for SimpleCas {
fn get_ref<P: AsRef<Path>>(&self, key: P) -> Result<ObjectId> {
fn get_ref<P: AsRef<Utf8Path>>(&self, key: P) -> Result<ObjectId> {
let path = ref_dir(&self.db_path).join(key.as_ref());
if !path.exists() {
Error::err(format!("reference {} does not exists", key.as_ref().to_string_lossy()))
err!("reference {} does not exists", key.as_ref())
}
else if !path.is_file() {
Error::err(format!("reference {} is not a file", key.as_ref().to_string_lossy()))
err!("reference {} is not a file", key.as_ref())
}
else {
let file = std::fs::read(path).or_else(|err|
Error::err(format!("failed to read reference file for {}: {}", key.as_ref().to_string_lossy(), err))
err!("failed to read reference file for {}: {}", key.as_ref(), err)
)?;
Ok(
ObjectId::from_str(
std::str::from_utf8(&file).or_else(|err|
Error::err(format!("invalid reference file at {}: {}", key.as_ref().to_string_lossy(), err))
err!("invalid reference file at {}: {}", key.as_ref(), err)
)?
)?
)
}
}
fn set_ref<P: AsRef<Path>>(&mut self, key: P, value: &ObjectId) -> Result<()> {
fn set_ref<P: AsRef<Utf8Path>>(&mut self, key: P, value: &ObjectId) -> Result<()> {
let path = ref_dir(&self.db_path).join(key.as_ref());
std::fs::create_dir_all(path.parent().ok_or_else(||
Error::error(format!("reference file {} has no parent dir?", key.as_ref().to_string_lossy()))
Error::unknown(format!("reference file {} has no parent dir?", key.as_ref()))
)?).or_else(|err|
Error::err(format!("failed to create reference dir for {}: {}", key.as_ref().to_string_lossy(), err))
err!("failed to create reference dir for {}: {}", key.as_ref(), err)
)?;
std::fs::write(path, value.to_string()).or_else(|err|
Error::err(format!("failed to write reference {}: {}", key.as_ref().to_string_lossy(), err))
err!("failed to write reference {}: {}", key.as_ref(), err)
)
}
fn remove_ref<P: AsRef<Path>>(&mut self, key: P) -> Result<()> {
fn remove_ref<P: AsRef<Utf8Path>>(&mut self, key: P) -> Result<()> {
let path = ref_dir(&self.db_path).join(key.as_ref());
if !path.exists() {
Error::err(format!("reference {} does not exists", key.as_ref().to_string_lossy()))
err!("reference {} does not exists", key.as_ref())
}
else if !path.is_file() {
Error::err(format!("reference {} is not a file", key.as_ref().to_string_lossy()))
err!("reference {} is not a file", key.as_ref())
}
else {
std::fs::remove_file(path).or_else(|err|
Error::err(format!("failed to remove reference file {}: {}", key.as_ref().to_string_lossy(), err))
err!("failed to remove reference file {}: {}", key.as_ref(), err)
)
}
}
@@ -215,9 +236,9 @@ impl RefStore for SimpleCas {
pub struct ObjectIdIterator {
root_dirs: Vec<PathBuf>,
inner_dirs: Vec<PathBuf>,
objects: Vec<PathBuf>,
root_dirs: Vec<Utf8PathBuf>,
inner_dirs: Vec<Utf8PathBuf>,
objects: Vec<Utf8PathBuf>,
root_index: usize,
inner_index: usize,
object_index: usize,
@@ -247,7 +268,7 @@ impl ObjectIdIterator {
let path = &self.objects[self.object_index];
if !path.is_file() {
return Error::err(format!("item in object database is not a file: {:?}", path));
return err!("item in object database is not a file: {:?}", path);
}
let id = path.ancestors()
@@ -255,9 +276,9 @@ impl ObjectIdIterator {
.collect::<Vec<_>>()
.iter()
.rev()
.map(|p| p.file_name()?.to_str())
.map(|p| p.file_name())
.collect::<Option<String>>()
.ok_or_else(|| Error::error(format!("invalid object in object database: {:?}", path)))?;
.ok_or_else(|| Error::unknown(format!("invalid object in object database: {:?}", path)))?;
ObjectId::from_str(&id)
}
@@ -342,14 +363,19 @@ impl Iterator for ObjectIdIterator {
}
}
fn read_dir(path: &Path) -> Result<Vec<PathBuf>> {
fn read_dir(path: &Utf8Path) -> Result<Vec<Utf8PathBuf>> {
let mut paths = std::fs::read_dir(path)
.or_else(|err| Err(Error::error(format!(
"failed to read directory {:?}: {}", path, err))))?
.map(|dir| dir.map(|dir| dir.path()))
.collect::<std::io::Result<Vec<_>>>()
.or_else(|err| Err(Error::error(format!(
"error while reading directory {:?}: {}", path, err))))?;
.or_else(|err| err!("failed to read directory {:?}: {}", path, err))?
.map(|res| match res {
Ok(dir) => Ok(
Utf8PathBuf::from_path_buf(dir.path())
.or_else(|e| err!("non-unicode character in path: {}: {:?}", path, e))?
),
Err(err) => err!("error while reading directory {:?}: {}", path, err),
})
.collect::<Result<Vec<_>>>()
.or_else(|err| err!(
"error while reading directory {:?}: {}", path, err))?;
paths.sort();
Ok(paths)
}
@@ -357,7 +383,7 @@ fn read_dir(path: &Path) -> Result<Vec<PathBuf>> {
#[cfg(test)]
mod tests {
use std::path::Path;
use camino::Utf8Path;
use super::*;
@@ -368,7 +394,7 @@ mod tests {
)
}
fn get_cas_path(dir: &Path) -> PathBuf {
fn get_cas_path(dir: &Utf8Path) -> Utf8PathBuf {
let mut cas_path = dir.to_path_buf();
cas_path.push(".bsv");
cas_path
@@ -377,7 +403,7 @@ mod tests {
#[test]
fn test_create_simple_cas() {
let dir = tempfile::tempdir().expect("failed to create temp test dir");
let cas_path = get_cas_path(dir.path());
let cas_path = get_cas_path(Utf8Path::from_path(dir.path()).unwrap());
let config = get_config();
let cas = SimpleCas::create(cas_path, config)
@@ -391,7 +417,7 @@ mod tests {
#[test]
fn test_write_object() {
let dir = tempfile::tempdir().expect("failed to create temp test dir");
let cas_path = get_cas_path(dir.path());
let cas_path = get_cas_path(dir.path().try_into().unwrap());
let config = get_config();
let otype = ObjectType::new(b"blob").expect("failed to create object type");
@@ -421,7 +447,7 @@ mod tests {
use std::io::Write;
let dir = tempfile::tempdir().expect("failed to create temp test dir");
let cas_path = get_cas_path(dir.path());
let cas_path = get_cas_path(dir.path().try_into().unwrap());
let config = get_config();
let otype = ObjectType::new(b"blob").expect("failed to create object type");
@@ -460,7 +486,7 @@ mod tests {
#[test]
fn test_read_write_object() {
let dir = tempfile::tempdir().expect("failed to create temp test dir");
let cas_path = get_cas_path(dir.path());
let cas_path = get_cas_path(dir.path().try_into().unwrap());
let config = get_config();
let otype = ObjectType::new(b"blob").expect("failed to create object type");
@@ -480,7 +506,7 @@ mod tests {
#[test]
fn test_remove_object() {
let dir = tempfile::tempdir().expect("failed to create temp test dir");
let cas_path = get_cas_path(dir.path());
let cas_path = get_cas_path(dir.path().try_into().unwrap());
let config = get_config();
let otype = ObjectType::new(b"blob").expect("failed to create object type");
@@ -501,7 +527,7 @@ mod tests {
#[test]
fn test_object_id_iterator() {
let dir = tempfile::tempdir().expect("failed to create temp test dir");
let cas_path = get_cas_path(dir.path());
let cas_path = get_cas_path(dir.path().try_into().unwrap());
let config = get_config();
let mut cas = SimpleCas::create(cas_path.clone(), config)
@@ -615,7 +641,7 @@ mod tests {
#[test]
fn test_reference() {
let dir = tempfile::tempdir().expect("failed to create temp test dir");
let cas_path = get_cas_path(dir.path());
let cas_path = get_cas_path(dir.path().try_into().unwrap());
let config = get_config();
let mut cas = SimpleCas::create(cas_path.clone(), config)

3
cas-simple/src/lib.rs
View File

@@ -21,12 +21,13 @@
extern crate digest;
extern crate sha2;
extern crate camino;
extern crate toml;
extern crate cas_core;
extern crate tempfile;
mod utils;
pub mod utils;
mod wfile;
mod cas;

61
cas-simple/src/utils.rs
View File

@@ -15,21 +15,20 @@
// along with cdb. If not, see <https://www.gnu.org/licenses/>.
use std::path::{Path, PathBuf};
use digest::DynDigest;
use toml::Value;
use camino::{Utf8Path, Utf8PathBuf};
use cas_core::{
Error, hex, ObjectId, ObjectMetadata, ObjectType, Result,
err, Error, hex, ObjectId, ObjectMetadata, ObjectType, Result,
};
pub fn obj_dir(cas_path: &Path) -> PathBuf {
pub fn obj_dir(cas_path: &Utf8Path) -> Utf8PathBuf {
cas_path.join("obj")
}
pub fn obj_path(cas_path: &Path, oid: &ObjectId) -> PathBuf {
pub fn obj_path(cas_path: &Utf8Path, oid: &ObjectId) -> Utf8PathBuf {
let mut path = obj_dir(cas_path);
path.push(hex(&oid.id()[0..1]));
path.push(hex(&oid.id()[1..2]));
@@ -37,49 +36,49 @@ pub fn obj_path(cas_path: &Path, oid: &ObjectId) -> PathBuf {
path
}
pub fn ref_dir(cas_path: &Path) -> PathBuf {
pub fn ref_dir(cas_path: &Utf8Path) -> Utf8PathBuf {
cas_path.join("ref")
}
pub fn tmp_dir(cas_path: &Path) -> PathBuf {
pub fn tmp_dir(cas_path: &Utf8Path) -> Utf8PathBuf {
cas_path.join("tmp")
}
pub fn config_path(cas_path: &Path) -> PathBuf {
pub fn config_path(cas_path: &Utf8Path) -> Utf8PathBuf {
cas_path.join("config")
}
pub fn read_config(db_path: &Path) -> Result<Value> {
pub fn read_config(db_path: &Utf8Path) -> Result<Value> {
use std::io::Read;
let mut file = std::fs::File::open(config_path(db_path)).or_else(|err|
Err(Error::error(format!("invalid repository: no config file: {}", err)))
err!("invalid repository: no config file: {}", err)
)?;
let mut config_str = String::new();
file.read_to_string(&mut config_str).or_else(|err|
Err(Error::error(format!("cannot read config file: {}", err)))
err!("cannot read config file: {}", err)
)?;
let config = toml::from_str(&config_str).or_else(|err|
Err(Error::error(format!("error while reading config file: {}", err)))
err!("error while reading config file: {}", err)
)?;
Ok(config)
}
pub fn write_config(config: &Value, db_path: &Path) -> Result<()> {
pub fn write_config(config: &Value, db_path: &Utf8Path) -> Result<()> {
use std::io::Write;
let config_str = toml::to_string_pretty(config).or_else(|err|
Err(Error::error(format!("failed to serialize config: {}", err)))
err!("failed to serialize config: {}", err)
)?;
let mut file = tempfile::NamedTempFile::new_in(tmp_dir(db_path)).or_else(|err|
Err(Error::error(format!("cannot create temp config file: {}", err)))
err!("cannot create temp config file: {}", err)
)?;
file.write_all(config_str.as_bytes()).or_else(|err|
Err(Error::error(format!("failed to write to temp config: {}", err)))
err!("failed to write to temp config: {}", err)
)?;
file.persist(config_path(db_path)).or_else(|err|
Err(Error::error(format!("failed to (over)write config: {}", err)))
err!("failed to (over)write config: {}", err)
)?;
Ok(())
}
@@ -89,13 +88,13 @@ pub fn read_metadata(read: &mut dyn std::io::Read) -> Result<ObjectMetadata> {
let otype = ObjectType::new(&{
let mut buf = [0; 4];
read.read_exact(&mut buf).or_else(|err|
Err(Error::error(format!("failed to read object type: {}", err))))?;
err!("failed to read object type: {}", err))?;
buf
})?;
let size = u64::from_be_bytes({
let mut buf = [0; 8];
read.read_exact(&mut buf).or_else(|err|
Err(Error::error(format!("failed to read object size: {}", err))))?;
err!("failed to read object size: {}", err))?;
buf
});
Ok(ObjectMetadata::new(otype, size))
@@ -103,9 +102,9 @@ pub fn read_metadata(read: &mut dyn std::io::Read) -> Result<ObjectMetadata> {
pub fn write_metadata(write: &mut dyn std::io::Write, otype: &ObjectType, size: u64) -> Result<()> {
write.write_all(otype.id()).or_else(|err|
Err(Error::error(format!("failed to write object type: {}", err))))?;
err!("failed to write object type: {}", err))?;
write.write_all(&size.to_be_bytes()).or_else(|err|
Err(Error::error(format!("failed to write object size: {}", err))))?;
err!("failed to write object size: {}", err))?;
Ok(())
}
@@ -113,7 +112,7 @@ pub fn write_metadata(write: &mut dyn std::io::Write, otype: &ObjectType, size:
pub fn new_digest(id: &str) -> Result<Box<dyn DynDigest>> {
match id {
"sha256" => { Ok(Box::new(sha2::Sha256::default())) },
_ => { Err(Error::error(format!("unknown digest '{}'", id))) }
_ => { err!("unknown digest '{}'", id) }
}
}
@@ -126,29 +125,29 @@ mod tests {
#[test]
fn test_dirs() {
let base_path = PathBuf::from("/foo/bar");
let base_path = Utf8PathBuf::from("/foo/bar");
let oid = ObjectId::from_str("0123456789abcdef")
.expect("failed to create object id");
assert_eq!(
config_path(&base_path),
PathBuf::from("/foo/bar/config")
Utf8PathBuf::from("/foo/bar/config")
);
assert_eq!(
obj_dir(&base_path),
PathBuf::from("/foo/bar/obj")
Utf8PathBuf::from("/foo/bar/obj")
);
assert_eq!(
obj_path(&base_path, &oid),
PathBuf::from("/foo/bar/obj/01/23/456789abcdef")
Utf8PathBuf::from("/foo/bar/obj/01/23/456789abcdef")
);
assert_eq!(
ref_dir(&base_path),
PathBuf::from("/foo/bar/ref")
Utf8PathBuf::from("/foo/bar/ref")
);
assert_eq!(
tmp_dir(&base_path),
PathBuf::from("/foo/bar/tmp")
Utf8PathBuf::from("/foo/bar/tmp")
);
}
@@ -165,10 +164,10 @@ mod tests {
let dir = tempfile::TempDir::new()
.expect("failed to create tmp dir");
std::fs::create_dir(tmp_dir(dir.path())).expect("failed to create db/tmp dir");
std::fs::create_dir(tmp_dir(Utf8Path::from_path(dir.path()).unwrap())).expect("failed to create db/tmp dir");
write_config(&config, dir.path()).expect("failed to write config");
let config2 = read_config(dir.path()).expect("failed to read config");
write_config(&config, Utf8Path::from_path(dir.path()).unwrap()).expect("failed to write config");
let config2 = read_config(Utf8Path::from_path(dir.path()).unwrap()).expect("failed to read config");
assert_eq!(config2, config);

24
cas-simple/src/wfile.rs
View File

@@ -14,10 +14,10 @@
// along with cdb. If not, see <https://www.gnu.org/licenses/>.
use std::path::{PathBuf};
use camino::{Utf8PathBuf};
use cas_core::{
Error, ObjectId, Result, Writer,
err, Error, ObjectId, Result, Writer,
};
use crate::utils::{obj_path, tmp_dir};
@@ -25,13 +25,13 @@ use crate::utils::{obj_path, tmp_dir};
pub struct WFile {
file: tempfile::NamedTempFile,
db_path: PathBuf,
db_path: Utf8PathBuf,
}
impl WFile {
pub fn new(db_path: PathBuf) -> Result<Self> {
pub fn new(db_path: Utf8PathBuf) -> Result<Self> {
let file = tempfile::NamedTempFile::new_in(tmp_dir(&db_path)).or_else(|err|
Err(Error::error(format!("failed to create_file: {}", err))))?;
err!("failed to create_file: {}", err))?;
Ok(WFile {
file,
db_path,
@@ -51,7 +51,7 @@ impl std::io::Write for WFile {
impl Writer for WFile {
fn finalize(self: Box<Self>) -> Result<ObjectId> {
Err(Error::error("reader pipline has no digest step"))
err!("reader pipline has no digest step")
}
fn _finalize(self: Box<Self>, oid: ObjectId) -> Result<ObjectId> {
@@ -59,9 +59,9 @@ impl Writer for WFile {
std::fs::create_dir_all(
path.parent().expect("cannot access to object parent directory")
).or_else(|err|
Err(Error::error(format!("failed to create object dir: {}", err))))?;
err!("failed to create object dir: {}", err))?;
self.file.persist(path).or_else(|err|
Err(Error::error(format!("failed to persist file: {}", err))))?;
err!("failed to persist file: {}", err))?;
Ok(oid)
}
}
@@ -86,18 +86,18 @@ mod tests {
let dir = tempfile::TempDir::new()
.expect("failed to create tmp dir");
std::fs::create_dir(tmp_dir(dir.path())).expect("failed to create db/obj dir");
std::fs::create_dir(obj_dir(dir.path())).expect("failed to create db/tmp dir");
std::fs::create_dir(tmp_dir(dir.path().try_into().unwrap())).expect("failed to create db/obj dir");
std::fs::create_dir(obj_dir(dir.path().try_into().unwrap())).expect("failed to create db/tmp dir");
{
let mut writer = Box::new(WFile::new(dir.path().to_path_buf()).expect("failed to create WFile"));
let mut writer = Box::new(WFile::new(dir.path().to_path_buf().try_into().unwrap()).expect("failed to create WFile"));
writer.write(data).expect("failed to write to WFile");
let oid2 = writer._finalize(oid.clone()).expect("failed to finalize WFile");
assert_eq!(oid2, oid);
}
let mut file = std::fs::File::open(obj_path(dir.path(), &oid)).expect("failed to open object file");
let mut file = std::fs::File::open(obj_path(dir.path().try_into().unwrap(), &oid)).expect("failed to open object file");
let mut buf = Vec::new();
file.read_to_end(&mut buf).expect("failed to read object file");