From d992fb3eb93322f6f5c16ae37245c399a5c8b635 Mon Sep 17 00:00:00 2001 From: Draklaw Date: Sun, 27 Sep 2020 01:54:39 +0200 Subject: [PATCH] SimpleDb supports reading & writing object. --- libbsv/src/core/error.rs | 9 +- libbsv/src/core/mod.rs | 2 +- libbsv/src/lib.rs | 4 +- libbsv/src/simple_db/mod.rs | 4 + libbsv/src/simple_db/object.rs | 151 ++++++++++++++++++++++-------- libbsv/src/simple_db/simple_db.rs | 71 +++++++++++++- libbsv/tests/simple_db.rs | 61 ++++++++++-- 7 files changed, 248 insertions(+), 54 deletions(-) diff --git a/libbsv/src/core/error.rs b/libbsv/src/core/error.rs index 9de5112..4284ebf 100644 --- a/libbsv/src/core/error.rs +++ b/libbsv/src/core/error.rs @@ -40,12 +40,17 @@ error_chain! { InvalidObjectIdCharacter { description("Object id contains invalid character") - display("Object id contains invalid character") + display("Object id contains invalid character.") } InvalidObjectType(otype: [u8; 4]) { description("Object has an invalid object type") - display("Object has an invalid object type {:?}", otype) + display("Object has an invalid object type {:?}.", otype) + } + + MismatchingObjectSize(actual: u64, expected: u64) { + description("Mismatching object size") + display("Mismatching object size: expected {}, got {}.", actual, expected) } } } diff --git a/libbsv/src/core/mod.rs b/libbsv/src/core/mod.rs index ddeb885..a8ba778 100644 --- a/libbsv/src/core/mod.rs +++ b/libbsv/src/core/mod.rs @@ -24,7 +24,7 @@ pub const NAME: &'static str = env!("CARGO_PKG_NAME"); pub const VERSION: &'static str = env!("CARGO_PKG_VERSION"); -pub use error::Error; +pub use error::{Error, Result, ErrorKind}; pub use config::{Config, RepositoryConfig}; pub use object_id::ObjectId; pub use repository::Repository; diff --git a/libbsv/src/lib.rs b/libbsv/src/lib.rs index 6f10920..1d49bd4 100644 --- a/libbsv/src/lib.rs +++ b/libbsv/src/lib.rs @@ -23,10 +23,8 @@ pub mod simple_db; pub use crate::core::{ - Error, + Error, Result, ErrorKind, Config, RepositoryConfig, ObjectId, Repository, }; - -pub use crate::simple_db::SimpleDb; diff --git a/libbsv/src/simple_db/mod.rs b/libbsv/src/simple_db/mod.rs index 040f716..1e0f8f0 100644 --- a/libbsv/src/simple_db/mod.rs +++ b/libbsv/src/simple_db/mod.rs @@ -19,3 +19,7 @@ pub mod object; pub use simple_db::SimpleDb; +pub use object::{ + OTYPE_BLOB, OTYPE_TREE, + ObjectReader, +}; diff --git a/libbsv/src/simple_db/object.rs b/libbsv/src/simple_db/object.rs index 82ea3fd..f1323e8 100644 --- a/libbsv/src/simple_db/object.rs +++ b/libbsv/src/simple_db/object.rs @@ -18,7 +18,7 @@ extern crate digest; extern crate sha2; extern crate flate2; -use std::io::{Read, Write, copy}; +use std::io::{Read, Seek, SeekFrom, Write, copy, sink}; use digest::Digest; @@ -32,15 +32,17 @@ use crate::core::error::*; use crate::core::ObjectId; -type ObjectType = [u8; 4]; +pub type ObjectType = [u8; 4]; -pub const TYPE_BLOB: &ObjectType = b"blob"; -pub const TYPE_TREE: &ObjectType = b"tree"; +pub const OTYPE_BLOB: &ObjectType = b"blob"; +pub const OTYPE_TREE: &ObjectType = b"tree"; pub struct ObjectWriter { writer: GzEncoder, digest: D, + size: u64, + written_size: u64, } @@ -61,15 +63,23 @@ impl ObjectWriter { Ok(ObjectWriter { writer: zwriter, digest, + size, + written_size: 0, }) } pub fn finish(mut self) -> Result<(ObjectId, W)> { self.writer.try_finish()?; - Ok(( - ObjectId::new(&self.digest.finalize()), - self.writer.finish()?, - )) + + if self.written_size == self.size { + Ok(( + ObjectId::new(&self.digest.finalize()), + self.writer.finish()?, + )) + } + else { + Err(ErrorKind::MismatchingObjectSize(self.written_size, self.size).into()) + } } } @@ -78,6 +88,7 @@ impl Write for ObjectWriter { fn write(&mut self, buf: &[u8]) -> std::io::Result { let size = self.writer.write(buf)?; self.digest.update(&buf[..size]); + self.written_size += size as u64; Ok(size) } @@ -87,28 +98,16 @@ impl Write for ObjectWriter { } -pub fn write_object(reader: &mut R, writer: W, - otype: &ObjectType, size: u64) - -> Result<(ObjectId, W)> where - R: Read, - W: Write { - let mut owriter: ObjectWriter - = ObjectWriter::new(writer, otype, size)?; - - copy(reader, &mut owriter)?; - - owriter.finish() -} - - pub struct ObjectReader { + otype: ObjectType, + size: u64, reader: GzDecoder, } impl ObjectReader { pub fn new(reader: R) - -> Result<(ObjectType, u64, ObjectReader)> { + -> Result> { let mut zreader = GzDecoder::new(reader); @@ -126,13 +125,19 @@ impl ObjectReader { u64::from_be_bytes(size_bytes) }; - Ok(( + Ok(ObjectReader { otype, size, - ObjectReader { - reader: zreader, - } - )) + reader: zreader, + }) + } + + pub fn object_type(&self) -> ObjectType { + self.otype + } + + pub fn size(&self) -> u64 { + self.size } pub fn close(self) -> Result { @@ -148,31 +153,97 @@ impl Read for ObjectReader { } +pub fn write_object(reader: &mut R, writer: W, + otype: &ObjectType, size: u64) + -> Result<(ObjectId, W)> where + R: Read, + W: Write { + let mut owriter: ObjectWriter + = ObjectWriter::new(writer, otype, size)?; + + copy(reader, &mut owriter)?; + + owriter.finish() +} + + +pub trait WriteAsObject { + fn write_as_object(&mut self, writer: W, otype: &ObjectType) -> Result; + + fn object_id(&mut self, otype: &ObjectType) -> Result { + self.write_as_object(sink(), otype) + } +} + + +impl WriteAsObject for T { + fn write_as_object(&mut self, writer: W, otype: &ObjectType) -> Result { + let start = self.seek(SeekFrom::Current(0))?; + let end = self.seek(SeekFrom::End(0))?; + self.seek(SeekFrom::Start(start))?; + + let (oid, _) = write_object(self, writer, otype, end - start)?; + Ok(oid) + } +} + + #[cfg(test)] mod tests { use super::*; + const PAYLOAD: &[u8; 12] = b"Hello World!"; + const PAYLOAD_OID: &str = "c3b4032160b015b2261530532a6c49f2bdadbe0687fb1f5a6a32e083"; + #[test] - fn object_read_write() { + fn object_read_write() -> Result<()> { use std::io::{Cursor, Seek, SeekFrom}; - let payload = b"Hello World!"; - let mut source = Cursor::new(payload); + let mut source = Cursor::new(PAYLOAD); let mut fake_file = Cursor::new(vec![]); let mut output = vec![]; let (oid, _) = write_object(&mut source, &mut fake_file, - TYPE_BLOB, payload.len() as u64).unwrap(); + OTYPE_BLOB, PAYLOAD.len() as u64)?; + + assert_eq!(oid, ObjectId::from_str(PAYLOAD_OID)?); + + fake_file.seek(SeekFrom::Start(0))?; + let mut reader = ObjectReader::new(fake_file)?; + let read_size = reader.read_to_end(&mut output)?; + + assert_eq!(reader.object_type(), *OTYPE_BLOB); + assert_eq!(reader.size(), PAYLOAD.len() as u64); + assert_eq!(read_size, PAYLOAD.len()); + assert_eq!(output, PAYLOAD); + + Ok(()) + } + + #[test] + fn object_write_invalid_size() { + let mut source = PAYLOAD.as_ref(); + let fake_file = vec![]; + + let result = write_object(&mut source, fake_file, OTYPE_BLOB, 13); + + match result { + Err(Error(ErrorKind::MismatchingObjectSize(actual, expected), _)) + if actual == 12 && expected == 13 => (), + Err(error) => panic!("Unexpected error: {:?}", error), + Ok(_) => panic!("Unexpected success"), + } + } + + #[test] + fn write_as_object() -> Result<()> { + use std::io::{Cursor, sink}; - assert_eq!(oid, ObjectId::from_str("c3b4032160b015b2261530532a6c49f2bdadbe0687fb1f5a6a32e083").unwrap()); + let mut source = Cursor::new(PAYLOAD); + let oid = source.write_as_object(sink(), OTYPE_BLOB)?; - fake_file.seek(SeekFrom::Start(0)).unwrap(); - let (otype, size, mut reader) = ObjectReader::new(fake_file).unwrap(); - let read_size = reader.read_to_end(&mut output).unwrap(); + assert_eq!(oid, ObjectId::from_str(PAYLOAD_OID)?); - assert_eq!(otype, *TYPE_BLOB); - assert_eq!(size, payload.len() as u64); - assert_eq!(read_size, payload.len()); - assert_eq!(output, payload); + Ok(()) } } diff --git a/libbsv/src/simple_db/simple_db.rs b/libbsv/src/simple_db/simple_db.rs index a394609..d973558 100644 --- a/libbsv/src/simple_db/simple_db.rs +++ b/libbsv/src/simple_db/simple_db.rs @@ -14,13 +14,25 @@ // along with cdb. If not, see . +extern crate tempfile; + use std::path::{Path, PathBuf}; +use std::io::{Read, Seek}; +use std::fs::{File, OpenOptions, create_dir, create_dir_all}; + +use tempfile::{NamedTempFile}; use crate::core::error::*; use crate::core::ObjectId; +use super::object::{ + OTYPE_BLOB, + ObjectReader, ObjectType, WriteAsObject, +}; -const OBJECTS_FOLDER: &str = "objects"; + +const OBJECTS_DIR: &str = "objects"; +const TMP_DIR: &str = "tmp"; #[derive(Debug)] @@ -37,6 +49,14 @@ impl SimpleDb { } + pub fn setup(&self) -> Result<()> { + create_dir(self.objects_dir())?; + create_dir(self.tmp_dir())?; + + Ok(()) + } + + pub fn has_object(&self, oid: &ObjectId) -> bool { let obj_path = self.path_from_id(oid); if let Ok(metadata) = std::fs::metadata(obj_path) { @@ -47,16 +67,63 @@ impl SimpleDb { } } + pub fn store_object(&self, otype: &ObjectType, mut reader: R) + -> Result { + let mut tmp_file = self.create_tmp_file()?; + let oid = reader.write_as_object(&mut tmp_file, otype)?; + let dst_file_path = self.path_from_id(&oid); + + // TODO: Check if dst_file_path exists + + create_dir_all(dst_file_path.parent().unwrap())?; + match tmp_file.persist(dst_file_path) { + Ok(file) => { + file.sync_data()?; + Ok(oid) + }, + Err(err) => Err(err.error.into()), + } + } + + pub fn store_file_as_blob(&self, file_path: &Path) -> Result { + let file = OpenOptions::new().read(true).open(file_path)?; + self.store_object(OTYPE_BLOB, file) + } + + pub fn read_object(&self, oid: &ObjectId) -> Result> { + let path = self.path_from_id(oid); + let file = OpenOptions::new().read(true).open(path)?; + ObjectReader::new(file) + } + + fn objects_dir(&self) -> PathBuf { + let mut path = self.path.clone(); + path.push(OBJECTS_DIR); + path + } + + fn tmp_dir(&self) -> PathBuf { + let mut path = self.path.clone(); + path.push(TMP_DIR); + path + } + fn path_from_id(&self, oid: &ObjectId) -> PathBuf { let oid_str = oid.to_string(); let mut path = self.path.clone(); - path.push(OBJECTS_FOLDER); + path.push(OBJECTS_DIR); path.push(oid_str.get(..4).unwrap()); // unwrap is ok here because we know oid_str is a path.push(oid_str.get(4..).unwrap()); // hexadecimal number (i.e. ascii only). path } + + fn create_tmp_file(&self) -> Result { + let tmp_dir = self.tmp_dir(); + + Ok(tempfile::Builder::new().tempfile_in(tmp_dir)?) + } } diff --git a/libbsv/tests/simple_db.rs b/libbsv/tests/simple_db.rs index c973c5a..03aa2c0 100644 --- a/libbsv/tests/simple_db.rs +++ b/libbsv/tests/simple_db.rs @@ -18,23 +18,72 @@ extern crate tempfile; extern crate libbsv; +use std::path::{PathBuf}; +use std::io::{Cursor, Read}; +use std::fs::{create_dir_all, write}; + +use tempfile::{TempDir}; + +use libbsv::{ + Result, + ObjectId, + simple_db::{ + OTYPE_BLOB, + SimpleDb, + } +}; + + +const PAYLOAD: &[u8; 12] = b"Hello World!"; +const PAYLOAD_OID: &str = "c3b4032160b015b2261530532a6c49f2bdadbe0687fb1f5a6a32e083"; + + #[test] fn simple_db_has_object() { - let temp_dir = tempfile::TempDir::new().unwrap(); - let db = libbsv::SimpleDb::new(temp_dir.path()).unwrap(); - let oid = libbsv::ObjectId::from_str("0001020304050607").unwrap(); + let temp_dir = TempDir::new().unwrap(); + let db = SimpleDb::new(temp_dir.path()).unwrap(); + db.setup().unwrap(); + + let oid = ObjectId::from_str("0001020304050607").unwrap(); assert!(!db.has_object(&oid)); - let mut object_path: std::path::PathBuf = temp_dir.path().into(); + let mut object_path: PathBuf = temp_dir.path().into(); object_path.push("objects"); object_path.push("0001"); - std::fs::create_dir_all(&object_path).unwrap(); + create_dir_all(&object_path).unwrap(); object_path.push("020304050607"); - std::fs::write(object_path, "test").unwrap(); + write(object_path, "test").unwrap(); assert!(db.has_object(&oid)); } + +#[test] +fn simple_db_store_read() -> Result<()> { + let expected_oid = ObjectId::from_str(PAYLOAD_OID)?; + let temp_dir = TempDir::new()?; + let db = SimpleDb::new(temp_dir.path())?; + db.setup()?; + + assert!(!db.has_object(&expected_oid)); + + let oid = db.store_object(OTYPE_BLOB, Cursor::new(PAYLOAD))?; + + assert!(db.has_object(&expected_oid)); + assert_eq!(oid, expected_oid); + + let mut reader = db.read_object(&oid)?; + + assert_eq!(reader.object_type(), *OTYPE_BLOB); + assert_eq!(reader.size(), PAYLOAD.len() as u64); + + let mut data = vec![]; + reader.read_to_end(&mut data)?; + + assert_eq!(data, *PAYLOAD); + + Ok(()) +}