9 changed files with 1002 additions and 271 deletions
@ -0,0 +1,563 @@ |
|||
// This file is part of bsv.
|
|||
//
|
|||
// bsv is free software: you can redistribute it and/or modify it under the
|
|||
// terms of the GNU Affero General Public License as published by the Free
|
|||
// Software Foundation, either version 3 of the License, or (at your option)
|
|||
// any later version.
|
|||
//
|
|||
// cdb is distributed in the hope that it will be useful, but WITHOUT ANY
|
|||
// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
|||
// FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for
|
|||
// more details.
|
|||
//
|
|||
// You should have received a copy of the Affero GNU General Public License
|
|||
// along with cdb. If not, see <https://www.gnu.org/licenses/>.
|
|||
|
|||
|
|||
use std::str::FromStr; |
|||
use std::path::{Path, PathBuf}; |
|||
|
|||
use digest::DynDigest; |
|||
use toml::Value; |
|||
use cas_core::{ |
|||
Cas, DefaultPipeline, Error, ObjectId, ObjectMetadata, ObjectType, |
|||
Pipeline, Reader, ReadWrapper, Result, Writer, |
|||
}; |
|||
|
|||
use crate::utils::{ |
|||
obj_dir, obj_path, ref_dir, tmp_dir, |
|||
read_config, write_config, |
|||
read_metadata, write_metadata, |
|||
new_digest, |
|||
}; |
|||
use crate::wfile::WFile; |
|||
|
|||
|
|||
pub struct SimpleCas { |
|||
db_path: PathBuf, |
|||
digest: Box<dyn DynDigest>, |
|||
pipeline: DefaultPipeline, |
|||
config: Value, |
|||
} |
|||
|
|||
|
|||
impl SimpleCas { |
|||
pub fn create(db_path: PathBuf, config: Value) -> Result<Self> { |
|||
let digest_id = config["cas"]["digest"].as_str() |
|||
.ok_or_else(|| Error::error( |
|||
"mandatory cas.digest value is invalid or missing from config" |
|||
))?; |
|||
let digest = new_digest(digest_id)?; |
|||
let pipeline = DefaultPipeline::new(digest.box_clone()); |
|||
|
|||
if db_path.exists() { |
|||
return Err(Error::error(format!( |
|||
"failed to create SimpleCas: target directory already exists ({})", |
|||
db_path.to_string_lossy(), |
|||
))); |
|||
} |
|||
|
|||
for path in [ |
|||
&db_path, |
|||
&obj_dir(&db_path), |
|||
&ref_dir(&db_path), |
|||
&tmp_dir(&db_path), |
|||
] { |
|||
std::fs::create_dir(path).or_else(|e| |
|||
Err(Error::error(format!( |
|||
"failed to create directory ({}): {}", |
|||
path.to_string_lossy(), e, |
|||
))) |
|||
)?; |
|||
} |
|||
|
|||
write_config(&config, &db_path)?; |
|||
|
|||
Ok(SimpleCas { |
|||
db_path, |
|||
digest, |
|||
pipeline, |
|||
config, |
|||
}) |
|||
} |
|||
|
|||
pub fn open(db_path: PathBuf) -> Result<Self> { |
|||
let config = read_config(&db_path)?; |
|||
|
|||
let digest_id = config["cas"]["digest"].as_str() |
|||
.ok_or_else(|| Error::error( |
|||
"mandatory cas.digest value is invalid or missing from config" |
|||
))?; |
|||
let digest = new_digest(digest_id)?; |
|||
let pipeline = DefaultPipeline::new(digest.box_clone()); |
|||
|
|||
Ok(SimpleCas { |
|||
db_path, |
|||
digest, |
|||
pipeline, |
|||
config, |
|||
}) |
|||
} |
|||
|
|||
pub fn save_config(&self) -> Result<()> { |
|||
write_config(&self.config, &self.db_path) |
|||
} |
|||
} |
|||
|
|||
|
|||
impl Cas for SimpleCas { |
|||
fn object_id_from_string(&self, hex: &str) -> Result<ObjectId> { |
|||
if hex.len() == self.digest.output_size() * 2 { |
|||
ObjectId::from_str(hex) |
|||
} |
|||
else { |
|||
Err(Error::error(format!( |
|||
"invalid object id size: got {}, expected {}", |
|||
hex.len(), self.digest.output_size() * 2, |
|||
))) |
|||
} |
|||
} |
|||
|
|||
fn has_object_id(&self, oid: &ObjectId) -> Result<bool> { |
|||
let opath = obj_path(&self.db_path, oid); |
|||
Ok(opath.is_file()) |
|||
} |
|||
|
|||
fn iter_object_id<'s>(&'s self) -> Result<Box<dyn 's + Iterator<Item = Result<ObjectId>>>> { |
|||
Ok(Box::new(ObjectIdIterator::new(self)?)) |
|||
} |
|||
|
|||
|
|||
fn open_object(&self, oid: &ObjectId) -> Result<(ObjectMetadata, Box<dyn Reader>)> { |
|||
let opath = obj_path(&self.db_path, oid); |
|||
if !opath.is_file() { |
|||
return Err(Error::error(format!("object not found: {}", oid))); |
|||
} |
|||
|
|||
let file = std::fs::File::open(opath).or_else(|err| |
|||
Err(Error::error(format!("failed to open object {}: {}", oid, err))))?; |
|||
let wrapper = Box::new(ReadWrapper::new(file)); |
|||
let mut reader = self.pipeline.new_reader(wrapper); |
|||
let metadata = read_metadata(&mut reader)?; |
|||
|
|||
Ok((metadata, reader)) |
|||
} |
|||
|
|||
fn new_writer(&mut self, otype: &ObjectType, size: u64) -> Result<Box<dyn Writer>> { |
|||
let file = Box::new(WFile::new(self.db_path.clone())?); |
|||
let mut writer = self.pipeline.new_writer(file); |
|||
write_metadata(&mut writer, otype, size)?; |
|||
|
|||
Ok(writer) |
|||
} |
|||
|
|||
fn remove_object(&mut self, oid: &ObjectId) -> Result<()> { |
|||
let opath = obj_path(&self.db_path, oid); |
|||
if !opath.is_file() { |
|||
return Err(Error::error(format!("object not found: {}", oid))); |
|||
} |
|||
std::fs::remove_file(opath).or_else(|err| |
|||
Err(Error::error(format!("failed to remove object {}: {}", oid, err))))?; |
|||
Ok(()) |
|||
} |
|||
} |
|||
|
|||
|
|||
pub struct ObjectIdIterator { |
|||
root_dirs: Vec<PathBuf>, |
|||
inner_dirs: Vec<PathBuf>, |
|||
objects: Vec<PathBuf>, |
|||
root_index: usize, |
|||
inner_index: usize, |
|||
object_index: usize, |
|||
} |
|||
|
|||
impl ObjectIdIterator { |
|||
fn new(cas: &SimpleCas) -> Result<Self> { |
|||
let root_dirs = read_dir(&obj_dir(&cas.db_path))?; |
|||
let mut it = Self { |
|||
root_dirs, |
|||
inner_dirs: Vec::new(), |
|||
objects: Vec::new(), |
|||
root_index: 0, |
|||
inner_index: 0, |
|||
object_index: 0, |
|||
}; |
|||
|
|||
it.fetch_inner_dirs()?; |
|||
if !it.is_item_valid() { |
|||
it.next_valid_item()?; |
|||
} |
|||
|
|||
Ok(it) |
|||
} |
|||
|
|||
fn get_object_id(&mut self) -> Result<ObjectId> { |
|||
let path = &self.objects[self.object_index]; |
|||
|
|||
if !path.is_file() { |
|||
return Error::err(format!("item in object database is not a file: {:?}", path)); |
|||
} |
|||
|
|||
let id = path.ancestors() |
|||
.take(3) |
|||
.collect::<Vec<_>>() |
|||
.iter() |
|||
.rev() |
|||
.map(|p| p.file_name()?.to_str()) |
|||
.collect::<Option<String>>() |
|||
.ok_or_else(|| Error::error(format!("invalid object in object database: {:?}", path)))?; |
|||
ObjectId::from_str(&id) |
|||
} |
|||
|
|||
fn is_done(&self) -> bool { |
|||
self.root_index == self.root_dirs.len() |
|||
} |
|||
|
|||
fn is_item_valid(&self) -> bool { |
|||
self.root_index < self.root_dirs.len() && |
|||
self.inner_index < self.inner_dirs.len() && |
|||
self.object_index < self.objects.len() |
|||
} |
|||
|
|||
fn next_item(&mut self) -> Result<()> { |
|||
if self.object_index < self.objects.len() { |
|||
self.object_index += 1; |
|||
} |
|||
else if self.inner_index < self.inner_dirs.len() { |
|||
self.inner_index += 1; |
|||
self.fetch_objects()?; |
|||
} |
|||
else if self.root_index < self.root_dirs.len() { |
|||
self.root_index += 1; |
|||
self.fetch_inner_dirs()?; |
|||
} |
|||
Ok(()) |
|||
} |
|||
|
|||
fn next_valid_item(&mut self) -> Result<bool> { |
|||
if !self.is_done() { |
|||
self.next_item()?; |
|||
} |
|||
while !self.is_done() && !self.is_item_valid() { |
|||
self.next_item()?; |
|||
} |
|||
Ok(self.is_done()) |
|||
} |
|||
|
|||
fn fetch_objects(&mut self) -> Result<()> { |
|||
self.object_index = 0; |
|||
if self.inner_index < self.inner_dirs.len() { |
|||
let path = &self.inner_dirs[self.inner_index]; |
|||
self.objects = read_dir(path)?; |
|||
} |
|||
else { |
|||
self.objects.clear(); |
|||
} |
|||
Ok(()) |
|||
} |
|||
|
|||
fn fetch_inner_dirs(&mut self) -> Result<()> { |
|||
self.inner_index = 0; |
|||
if self.root_index < self.root_dirs.len() { |
|||
let path = &self.root_dirs[self.root_index]; |
|||
self.inner_dirs = read_dir(path)?; |
|||
self.fetch_objects()?; |
|||
} |
|||
else { |
|||
self.inner_dirs.clear(); |
|||
} |
|||
Ok(()) |
|||
} |
|||
} |
|||
|
|||
impl Iterator for ObjectIdIterator { |
|||
type Item = Result<ObjectId>; |
|||
|
|||
fn next(&mut self) -> Option<Self::Item> { |
|||
if !self.is_done() { |
|||
debug_assert!(self.is_item_valid()); |
|||
let item = self.get_object_id(); |
|||
if item.is_ok() { |
|||
if let Err(err) = self.next_valid_item() { |
|||
return Some(Err(err)); |
|||
} |
|||
} |
|||
Some(item) |
|||
} |
|||
else { |
|||
None |
|||
} |
|||
} |
|||
} |
|||
|
|||
fn read_dir(path: &Path) -> Result<Vec<PathBuf>> { |
|||
let mut paths = std::fs::read_dir(path) |
|||
.or_else(|err| Err(Error::error(format!( |
|||
"failed to read directory {:?}: {}", path, err))))? |
|||
.map(|dir| dir.map(|dir| dir.path())) |
|||
.collect::<std::io::Result<Vec<_>>>() |
|||
.or_else(|err| Err(Error::error(format!( |
|||
"error while reading directory {:?}: {}", path, err))))?; |
|||
paths.sort(); |
|||
Ok(paths) |
|||
} |
|||
|
|||
|
|||
#[cfg(test)] |
|||
mod tests { |
|||
use std::path::Path; |
|||
|
|||
use super::*; |
|||
|
|||
fn get_config() -> Value { |
|||
toml::toml!( |
|||
[cas] |
|||
digest = "sha256" |
|||
) |
|||
} |
|||
|
|||
fn get_cas_path(dir: &Path) -> PathBuf { |
|||
let mut cas_path = dir.to_path_buf(); |
|||
cas_path.push(".bsv"); |
|||
cas_path |
|||
} |
|||
|
|||
#[test] |
|||
fn test_create_simple_cas() { |
|||
let dir = tempfile::tempdir().expect("failed to create temp test dir"); |
|||
let cas_path = get_cas_path(dir.path()); |
|||
let config = get_config(); |
|||
|
|||
let cas = SimpleCas::create(cas_path, config) |
|||
.expect("failed to create SimpleCas object"); |
|||
let oid = cas.object_id_from_string("f731f6bc6a6a73bad170e56452473ef6930b7a0ab33cc54be44221a89b49d786") |
|||
.expect("failed to create object id"); |
|||
|
|||
assert!(!cas.has_object_id(&oid).expect("has_object_id failed")); |
|||
} |
|||
|
|||
#[test] |
|||
fn test_write_object() { |
|||
let dir = tempfile::tempdir().expect("failed to create temp test dir"); |
|||
let cas_path = get_cas_path(dir.path()); |
|||
let config = get_config(); |
|||
|
|||
let otype = ObjectType::new(b"blob").expect("failed to create object type"); |
|||
let payload = b"Hello World!"; |
|||
|
|||
let mut cas = SimpleCas::create(cas_path.clone(), config) |
|||
.expect("failed to create SimpleCas object"); |
|||
let oid = cas.object_id_from_string("f731f6bc6a6a73bad170e56452473ef6930b7a0ab33cc54be44221a89b49d786") |
|||
.expect("failed to create object id"); |
|||
|
|||
assert!(!cas.has_object_id(&oid).expect("has_object_id failed")); |
|||
|
|||
{ |
|||
let mut writer = cas.new_writer(&otype, payload.len() as u64) |
|||
.expect("failed to create object writer"); |
|||
writer.write_all(payload).expect("failed to write object"); |
|||
let oid2 = writer.finalize().expect("failed to finalize writer"); |
|||
|
|||
assert_eq!(oid2, oid); |
|||
} |
|||
|
|||
assert!(cas.has_object_id(&oid).expect("has_object_id failed")); |
|||
} |
|||
|
|||
#[test] |
|||
fn test_read_object() { |
|||
use std::io::Write; |
|||
|
|||
let dir = tempfile::tempdir().expect("failed to create temp test dir"); |
|||
let cas_path = get_cas_path(dir.path()); |
|||
let config = get_config(); |
|||
|
|||
let otype = ObjectType::new(b"blob").expect("failed to create object type"); |
|||
let payload = b"Hello World!"; |
|||
|
|||
let cas = SimpleCas::create(cas_path.clone(), config) |
|||
.expect("failed to create SimpleCas object"); |
|||
let oid = cas.object_id_from_string("f731f6bc6a6a73bad170e56452473ef6930b7a0ab33cc54be44221a89b49d786") |
|||
.expect("failed to create object id"); |
|||
|
|||
assert!(!cas.has_object_id(&oid).expect("has_object_id failed")); |
|||
|
|||
{ |
|||
let opath = obj_path(&cas_path, &oid); |
|||
std::fs::create_dir_all(opath.parent().unwrap()).expect("failed to create object dir"); |
|||
let mut file = std::fs::File::create(opath) |
|||
.expect("failed to open object file for write"); |
|||
file.write_all(b"blob\x00\x00\x00\x00\x00\x00\x00\x0c") |
|||
.expect("failed to write object header"); |
|||
file.write_all(payload).expect("failed to write object payload"); |
|||
} |
|||
|
|||
assert!(cas.has_object_id(&oid).expect("has_object_id failed")); |
|||
|
|||
let (metadata, mut reader) = cas.open_object(&oid).expect("failed to open object"); |
|||
let mut buf = Vec::new(); |
|||
reader.read_to_end(&mut buf).expect("failed to read object"); |
|||
let oid2 = reader.finalize().expect("failed to finalize object reader"); |
|||
|
|||
assert_eq!(oid2, oid); |
|||
assert_eq!(metadata.otype(), &otype); |
|||
assert_eq!(metadata.size(), payload.len() as u64); |
|||
assert_eq!(buf, payload); |
|||
} |
|||
|
|||
#[test] |
|||
fn test_read_write_object() { |
|||
let dir = tempfile::tempdir().expect("failed to create temp test dir"); |
|||
let cas_path = get_cas_path(dir.path()); |
|||
let config = get_config(); |
|||
|
|||
let otype = ObjectType::new(b"blob").expect("failed to create object type"); |
|||
let payload = b"This is a test."; |
|||
|
|||
let mut cas = SimpleCas::create(cas_path.clone(), config) |
|||
.expect("failed to create SimpleCas object"); |
|||
|
|||
let oid = cas.write_object(&otype, payload).expect("failed to write object"); |
|||
let (metadata, data) = cas.read_object(&oid).expect("failed to read object"); |
|||
|
|||
assert_eq!(metadata.otype(), &otype); |
|||
assert_eq!(metadata.size(), payload.len() as u64); |
|||
assert_eq!(data, payload); |
|||
} |
|||
|
|||
#[test] |
|||
fn test_remove_object() { |
|||
let dir = tempfile::tempdir().expect("failed to create temp test dir"); |
|||
let cas_path = get_cas_path(dir.path()); |
|||
let config = get_config(); |
|||
|
|||
let otype = ObjectType::new(b"blob").expect("failed to create object type"); |
|||
let payload = b"This is a test."; |
|||
|
|||
let mut cas = SimpleCas::create(cas_path.clone(), config) |
|||
.expect("failed to create SimpleCas object"); |
|||
|
|||
let oid = cas.write_object(&otype, payload).expect("failed to write object"); |
|||
|
|||
assert!(cas.has_object_id(&oid).unwrap()); |
|||
|
|||
cas.remove_object(&oid).expect("failed to remove object"); |
|||
|
|||
assert!(!cas.has_object_id(&oid).unwrap()); |
|||
} |
|||
|
|||
#[test] |
|||
fn test_object_id_iterator() { |
|||
let dir = tempfile::tempdir().expect("failed to create temp test dir"); |
|||
let cas_path = get_cas_path(dir.path()); |
|||
let config = get_config(); |
|||
|
|||
let mut cas = SimpleCas::create(cas_path.clone(), config) |
|||
.expect("failed to create SimpleCas object"); |
|||
|
|||
let add_fake_object = |oid: &ObjectId| { |
|||
use std::io::Write; |
|||
|
|||
let path = obj_path(&cas.db_path, oid); |
|||
std::fs::create_dir_all(path.parent().unwrap()) |
|||
.expect("failed to create object directory"); |
|||
let mut file = std::fs::File::create(path).unwrap(); |
|||
file.write(b"Test").unwrap(); |
|||
}; |
|||
|
|||
let oid_0_0_0 = ObjectId::from_str("0000000000000000000000000000000000000000000000000000000000000000").unwrap(); |
|||
let oid_0_0_1 = ObjectId::from_str("0000000000000000000000000000000000000000000000000000000000000001").unwrap(); |
|||
let oid_0_0_2 = ObjectId::from_str("0000000000000000000000000000000000000000000000000000000000000002").unwrap(); |
|||
|
|||
let oid_0_1_0 = ObjectId::from_str("0001000000000000000000000000000000000000000000000000000000000000").unwrap(); |
|||
let oid_0_2_0 = ObjectId::from_str("0002000000000000000000000000000000000000000000000000000000000000").unwrap(); |
|||
|
|||
let oid_1_0_0 = ObjectId::from_str("0100000000000000000000000000000000000000000000000000000000000000").unwrap(); |
|||
let oid_2_0_0 = ObjectId::from_str("0200000000000000000000000000000000000000000000000000000000000000").unwrap(); |
|||
|
|||
let objects = cas.iter_object_id().unwrap().collect::<Result<Vec<_>>>().unwrap(); |
|||
assert_eq!(objects, []); |
|||
|
|||
let all_oids = [ |
|||
oid_0_0_0.clone(), oid_0_0_1.clone(), oid_0_0_2.clone(), |
|||
oid_0_1_0.clone(), oid_0_2_0.clone(), |
|||
oid_1_0_0.clone(), oid_2_0_0.clone(), |
|||
]; |
|||
for oid in all_oids.iter() { |
|||
add_fake_object(&oid); |
|||
} |
|||
|
|||
let objects = cas.iter_object_id().unwrap() |
|||
.collect::<Result<Vec<_>>>().unwrap(); |
|||
assert_eq!(objects, all_oids); |
|||
|
|||
cas.remove_object(&oid_0_0_0).unwrap(); |
|||
|
|||
let oids = [ |
|||
oid_0_0_1.clone(), oid_0_0_2.clone(), |
|||
oid_0_1_0.clone(), oid_0_2_0.clone(), |
|||
oid_1_0_0.clone(), oid_2_0_0.clone(), |
|||
]; |
|||
let objects = cas.iter_object_id().unwrap() |
|||
.collect::<Result<Vec<_>>>().unwrap(); |
|||
assert_eq!(objects, oids); |
|||
|
|||
cas.remove_object(&oid_0_0_2).unwrap(); |
|||
|
|||
let oids = [ |
|||
oid_0_0_1.clone(), |
|||
oid_0_1_0.clone(), oid_0_2_0.clone(), |
|||
oid_1_0_0.clone(), oid_2_0_0.clone(), |
|||
]; |
|||
let objects = cas.iter_object_id().unwrap() |
|||
.collect::<Result<Vec<_>>>().unwrap(); |
|||
assert_eq!(objects, oids); |
|||
|
|||
cas.remove_object(&oid_0_0_1).unwrap(); |
|||
|
|||
let oids = [ |
|||
oid_0_1_0.clone(), oid_0_2_0.clone(), |
|||
oid_1_0_0.clone(), oid_2_0_0.clone(), |
|||
]; |
|||
let objects = cas.iter_object_id().unwrap() |
|||
.collect::<Result<Vec<_>>>().unwrap(); |
|||
assert_eq!(objects, oids); |
|||
|
|||
cas.remove_object(&oid_2_0_0).unwrap(); |
|||
|
|||
let oids = [ |
|||
oid_0_1_0.clone(), oid_0_2_0.clone(), |
|||
oid_1_0_0.clone(), |
|||
]; |
|||
let objects = cas.iter_object_id().unwrap() |
|||
.collect::<Result<Vec<_>>>().unwrap(); |
|||
assert_eq!(objects, oids); |
|||
|
|||
cas.remove_object(&oid_1_0_0).unwrap(); |
|||
|
|||
let oids = [ |
|||
oid_0_1_0.clone(), oid_0_2_0.clone(), |
|||
]; |
|||
let objects = cas.iter_object_id().unwrap() |
|||
.collect::<Result<Vec<_>>>().unwrap(); |
|||
assert_eq!(objects, oids); |
|||
|
|||
cas.remove_object(&oid_0_1_0).unwrap(); |
|||
|
|||
let oids = [ |
|||
oid_0_2_0.clone(), |
|||
]; |
|||
let objects = cas.iter_object_id().unwrap() |
|||
.collect::<Result<Vec<_>>>().unwrap(); |
|||
assert_eq!(objects, oids); |
|||
|
|||
cas.remove_object(&oid_0_2_0).unwrap(); |
|||
|
|||
let oids = [ |
|||
]; |
|||
let objects = cas.iter_object_id().unwrap() |
|||
.collect::<Result<Vec<_>>>().unwrap(); |
|||
assert_eq!(objects, oids); |
|||
} |
|||
} |
|||
@ -0,0 +1,210 @@ |
|||
|
|||
// This file is part of bsv.
|
|||
//
|
|||
// bsv is free software: you can redistribute it and/or modify it under the
|
|||
// terms of the GNU Affero General Public License as published by the Free
|
|||
// Software Foundation, either version 3 of the License, or (at your option)
|
|||
// any later version.
|
|||
//
|
|||
// cdb is distributed in the hope that it will be useful, but WITHOUT ANY
|
|||
// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
|||
// FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for
|
|||
// more details.
|
|||
//
|
|||
// You should have received a copy of the Affero GNU General Public License
|
|||
// along with cdb. If not, see <https://www.gnu.org/licenses/>.
|
|||
|
|||
|
|||
use std::path::{Path, PathBuf}; |
|||
|
|||
use digest::DynDigest; |
|||
use toml::Value; |
|||
|
|||
use cas_core::{ |
|||
Error, hex, ObjectId, ObjectMetadata, ObjectType, Result, |
|||
}; |
|||
|
|||
|
|||
pub fn obj_dir(cas_path: &Path) -> PathBuf { |
|||
cas_path.join("obj") |
|||
} |
|||
|
|||
pub fn obj_path(cas_path: &Path, oid: &ObjectId) -> PathBuf { |
|||
let mut path = obj_dir(cas_path); |
|||
path.push(hex(&oid.id()[0..1])); |
|||
path.push(hex(&oid.id()[1..2])); |
|||
path.push(hex(&oid.id()[2..])); |
|||
path |
|||
} |
|||
|
|||
pub fn ref_dir(cas_path: &Path) -> PathBuf { |
|||
cas_path.join("ref") |
|||
} |
|||
|
|||
pub fn tmp_dir(cas_path: &Path) -> PathBuf { |
|||
cas_path.join("tmp") |
|||
} |
|||
|
|||
pub fn config_path(cas_path: &Path) -> PathBuf { |
|||
cas_path.join("config") |
|||
} |
|||
|
|||
|
|||
pub fn read_config(db_path: &Path) -> Result<Value> { |
|||
use std::io::Read; |
|||
|
|||
let mut file = std::fs::File::open(config_path(db_path)).or_else(|err| |
|||
Err(Error::error(format!("invalid repository: no config file: {}", err))) |
|||
)?; |
|||
let mut config_str = String::new(); |
|||
file.read_to_string(&mut config_str).or_else(|err| |
|||
Err(Error::error(format!("cannot read config file: {}", err))) |
|||
)?; |
|||
let config = toml::from_str(&config_str).or_else(|err| |
|||
Err(Error::error(format!("error while reading config file: {}", err))) |
|||
)?; |
|||
Ok(config) |
|||
} |
|||
|
|||
pub fn write_config(config: &Value, db_path: &Path) -> Result<()> { |
|||
use std::io::Write; |
|||
|
|||
let config_str = toml::to_string_pretty(config).or_else(|err| |
|||
Err(Error::error(format!("failed to serialize config: {}", err))) |
|||
)?; |
|||
let mut file = tempfile::NamedTempFile::new_in(tmp_dir(db_path)).or_else(|err| |
|||
Err(Error::error(format!("cannot create temp config file: {}", err))) |
|||
)?; |
|||
file.write_all(config_str.as_bytes()).or_else(|err| |
|||
Err(Error::error(format!("failed to write to temp config: {}", err))) |
|||
)?; |
|||
file.persist(config_path(db_path)).or_else(|err| |
|||
Err(Error::error(format!("failed to (over)write config: {}", err))) |
|||
)?; |
|||
Ok(()) |
|||
} |
|||
|
|||
|
|||
pub fn read_metadata(read: &mut dyn std::io::Read) -> Result<ObjectMetadata> { |
|||
let otype = ObjectType::new(&{ |
|||
let mut buf = [0; 4]; |
|||
read.read_exact(&mut buf).or_else(|err| |
|||
Err(Error::error(format!("failed to read object type: {}", err))))?; |
|||
buf |
|||
})?; |
|||
let size = u64::from_be_bytes({ |
|||
let mut buf = [0; 8]; |
|||
read.read_exact(&mut buf).or_else(|err| |
|||
Err(Error::error(format!("failed to read object size: {}", err))))?; |
|||
buf |
|||
}); |
|||
Ok(ObjectMetadata::new(otype, size)) |
|||
} |
|||
|
|||
pub fn write_metadata(write: &mut dyn std::io::Write, otype: &ObjectType, size: u64) -> Result<()> { |
|||
write.write_all(otype.id()).or_else(|err| |
|||
Err(Error::error(format!("failed to write object type: {}", err))))?; |
|||
write.write_all(&size.to_be_bytes()).or_else(|err| |
|||
Err(Error::error(format!("failed to write object size: {}", err))))?; |
|||
Ok(()) |
|||
} |
|||
|
|||
|
|||
pub fn new_digest(id: &str) -> Result<Box<dyn DynDigest>> { |
|||
match id { |
|||
"sha256" => { Ok(Box::new(sha2::Sha256::default())) }, |
|||
_ => { Err(Error::error(format!("unknown digest '{}'", id))) } |
|||
} |
|||
} |
|||
|
|||
|
|||
#[cfg(test)] |
|||
mod tests { |
|||
use std::str::FromStr; |
|||
|
|||
use super::*; |
|||
|
|||
#[test] |
|||
fn test_dirs() { |
|||
let base_path = PathBuf::from("/foo/bar"); |
|||
let oid = ObjectId::from_str("0123456789abcdef") |
|||
.expect("failed to create object id"); |
|||
|
|||
assert_eq!( |
|||
config_path(&base_path), |
|||
PathBuf::from("/foo/bar/config") |
|||
); |
|||
assert_eq!( |
|||
obj_dir(&base_path), |
|||
PathBuf::from("/foo/bar/obj") |
|||
); |
|||
assert_eq!( |
|||
obj_path(&base_path, &oid), |
|||
PathBuf::from("/foo/bar/obj/01/23/456789abcdef") |
|||
); |
|||
assert_eq!( |
|||
ref_dir(&base_path), |
|||
PathBuf::from("/foo/bar/ref") |
|||
); |
|||
assert_eq!( |
|||
tmp_dir(&base_path), |
|||
PathBuf::from("/foo/bar/tmp") |
|||
); |
|||
} |
|||
|
|||
#[test] |
|||
fn test_read_write_config() { |
|||
let config = toml::toml!{ |
|||
[cas] |
|||
path = "/foo/bar" |
|||
digest = "sha1" |
|||
|
|||
[extra] |
|||
test = 42 |
|||
}; |
|||
|
|||
let dir = tempfile::TempDir::new() |
|||
.expect("failed to create tmp dir"); |
|||
std::fs::create_dir(tmp_dir(dir.path())).expect("failed to create db/tmp dir"); |
|||
|
|||
write_config(&config, dir.path()).expect("failed to write config"); |
|||
let config2 = read_config(dir.path()).expect("failed to read config"); |
|||
|
|||
assert_eq!(config2, config); |
|||
|
|||
dir.close().expect("failed to close tmp dir") |
|||
} |
|||
|
|||
#[test] |
|||
fn test_read_write_metadata() { |
|||
use std::io::Write; |
|||
|
|||
let data = b"test\x00\x00\x00\x00\x00\x00\x00\x0cHello World!"; |
|||
let reader: &mut dyn std::io::Read = &mut &data[..]; |
|||
let mut payload = Vec::new(); |
|||
|
|||
let metadata = read_metadata(reader).expect("failed to read metadata"); |
|||
reader.read_to_end(&mut payload).expect("failed to read payload"); |
|||
|
|||
let otype = ObjectType::new(b"test").expect("failed to create ObjectType"); |
|||
assert_eq!(metadata, ObjectMetadata::new(otype.clone(), 12)); |
|||
assert_eq!(payload, b"Hello World!"); |
|||
|
|||
let mut data2 = Vec::new(); |
|||
|
|||
write_metadata(&mut data2, &otype, payload.len() as u64).expect("failed to write metadata"); |
|||
data2.write_all(&payload).expect("failed to write payload"); |
|||
|
|||
assert_eq!(data2, data); |
|||
} |
|||
|
|||
#[test] |
|||
fn test_new_digest() { |
|||
let mut sha256 = new_digest("sha256").expect("failed to create sha256 digest"); |
|||
|
|||
sha256.update(b"Hello World!"); |
|||
assert_eq!(hex(&sha256.finalize()), "7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069"); |
|||
|
|||
assert!(new_digest("do_not_exists").is_err()); |
|||
} |
|||
} |
|||
@ -0,0 +1,106 @@ |
|||
// This file is part of bsv.
|
|||
//
|
|||
// bsv is free software: you can redistribute it and/or modify it under the
|
|||
// terms of the GNU Affero General Public License as published by the Free
|
|||
// Software Foundation, either version 3 of the License, or (at your option)
|
|||
// any later version.
|
|||
//
|
|||
// cdb is distributed in the hope that it will be useful, but WITHOUT ANY
|
|||
// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
|||
// FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for
|
|||
// more details.
|
|||
//
|
|||
// You should have received a copy of the Affero GNU General Public License
|
|||
// along with cdb. If not, see <https://www.gnu.org/licenses/>.
|
|||
|
|||
|
|||
use std::path::{PathBuf}; |
|||
|
|||
use cas_core::{ |
|||
Error, ObjectId, Result, Writer, |
|||
}; |
|||
|
|||
use crate::utils::{obj_path, tmp_dir}; |
|||
|
|||
|
|||
pub struct WFile { |
|||
file: tempfile::NamedTempFile, |
|||
db_path: PathBuf, |
|||
} |
|||
|
|||
impl WFile { |
|||
pub fn new(db_path: PathBuf) -> Result<Self> { |
|||
let file = tempfile::NamedTempFile::new_in(tmp_dir(&db_path)).or_else(|err| |
|||
Err(Error::error(format!("failed to create_file: {}", err))))?; |
|||
Ok(WFile { |
|||
file, |
|||
db_path, |
|||
}) |
|||
} |
|||
} |
|||
|
|||
impl std::io::Write for WFile { |
|||
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> { |
|||
self.file.write(buf) |
|||
} |
|||
|
|||
fn flush(&mut self) -> std::io::Result<()> { |
|||
self.file.flush() |
|||
} |
|||
} |
|||
|
|||
impl Writer for WFile { |
|||
fn finalize(self: Box<Self>) -> Result<ObjectId> { |
|||
Err(Error::error("reader pipline has no digest step")) |
|||
} |
|||
|
|||
fn _finalize(self: Box<Self>, oid: ObjectId) -> Result<ObjectId> { |
|||
let path = obj_path(&self.db_path, &oid); |
|||
std::fs::create_dir_all( |
|||
path.parent().expect("cannot access to object parent directory") |
|||
).or_else(|err| |
|||
Err(Error::error(format!("failed to create object dir: {}", err))))?; |
|||
self.file.persist(path).or_else(|err| |
|||
Err(Error::error(format!("failed to persist file: {}", err))))?; |
|||
Ok(oid) |
|||
} |
|||
} |
|||
|
|||
|
|||
#[cfg(test)] |
|||
mod tests { |
|||
use std::str::FromStr; |
|||
|
|||
use crate::utils::{obj_dir, obj_path}; |
|||
use super::*; |
|||
|
|||
#[test] |
|||
fn test_wfile() { |
|||
use std::io::{Read, Write}; |
|||
|
|||
let data = b"Hello World!"; |
|||
let oid = ObjectId::from_str( |
|||
"7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069" |
|||
).expect("invalid object id"); |
|||
|
|||
let dir = tempfile::TempDir::new() |
|||
.expect("failed to create tmp dir"); |
|||
|
|||
std::fs::create_dir(tmp_dir(dir.path())).expect("failed to create db/obj dir"); |
|||
std::fs::create_dir(obj_dir(dir.path())).expect("failed to create db/tmp dir"); |
|||
|
|||
{ |
|||
let mut writer = Box::new(WFile::new(dir.path().to_path_buf()).expect("failed to create WFile")); |
|||
writer.write(data).expect("failed to write to WFile"); |
|||
let oid2 = writer._finalize(oid.clone()).expect("failed to finalize WFile"); |
|||
|
|||
assert_eq!(oid2, oid); |
|||
} |
|||
|
|||
let mut file = std::fs::File::open(obj_path(dir.path(), &oid)).expect("failed to open object file"); |
|||
let mut buf = Vec::new(); |
|||
file.read_to_end(&mut buf).expect("failed to read object file"); |
|||
|
|||
assert_eq!(buf, data); |
|||
} |
|||
} |
|||
Loading…
Reference in new issue