draklaw/bsv
1
0
Fork 0
Code Issues 6 Pull Requests Projects 1 Releases Wiki Activity

SimpleDb supports reading & writing object.

Browse Source
This commit is contained in:
Draklaw
2020-09-27 01:54:39 +02:00
parent 86aa9a1eaa
commit d992fb3eb9
7 changed files with 266 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
libbsv/src/core/error.rs
View File

@@ -40,12 +40,17 @@ error_chain! {
InvalidObjectIdCharacter {
description("Object id contains invalid character")
display("Object id contains invalid character")
display("Object id contains invalid character.")
}
InvalidObjectType(otype: [u8; 4]) {
description("Object has an invalid object type")
display("Object has an invalid object type {:?}", otype)
display("Object has an invalid object type {:?}.", otype)
}
MismatchingObjectSize(actual: u64, expected: u64) {
description("Mismatching object size")
display("Mismatching object size: expected {}, got {}.", actual, expected)
}
}
}

2
libbsv/src/core/mod.rs
View File

@@ -24,7 +24,7 @@ pub const NAME: &'static str = env!("CARGO_PKG_NAME");
pub const VERSION: &'static str = env!("CARGO_PKG_VERSION");
pub use error::Error;
pub use error::{Error, Result, ErrorKind};
pub use config::{Config, RepositoryConfig};
pub use object_id::ObjectId;
pub use repository::Repository;

4
libbsv/src/lib.rs
View File

@@ -23,10 +23,8 @@ pub mod simple_db;
pub use crate::core::{
Error,
Error, Result, ErrorKind,
Config, RepositoryConfig,
ObjectId,
Repository,
};
pub use crate::simple_db::SimpleDb;

4
libbsv/src/simple_db/mod.rs
View File

@@ -19,3 +19,7 @@ pub mod object;
pub use simple_db::SimpleDb;
pub use object::{
OTYPE_BLOB, OTYPE_TREE,
ObjectReader,
};

187
libbsv/src/simple_db/object.rs
View File

@@ -18,7 +18,7 @@ extern crate digest;
extern crate sha2;
extern crate flate2;
use std::io::{Read, Write, copy};
use std::io::{Read, Seek, SeekFrom, Write, copy, sink};
use digest::Digest;
@@ -32,15 +32,17 @@ use crate::core::error::*;
use crate::core::ObjectId;
type ObjectType = [u8; 4];
pub type ObjectType = [u8; 4];
pub const TYPE_BLOB: &ObjectType = b"blob";
pub const TYPE_TREE: &ObjectType = b"tree";
pub const OTYPE_BLOB: &ObjectType = b"blob";
pub const OTYPE_TREE: &ObjectType = b"tree";
pub struct ObjectWriter<W: Write, D: Digest> {
writer: GzEncoder<W>,
digest: D,
size: u64,
written_size: u64,
}
@@ -61,15 +63,23 @@ impl<W: Write, D: Digest> ObjectWriter<W, D> {
Ok(ObjectWriter {
writer: zwriter,
digest,
size,
written_size: 0,
})
}
pub fn finish(mut self) -> Result<(ObjectId, W)> {
self.writer.try_finish()?;
Ok((
ObjectId::new(&self.digest.finalize()),
self.writer.finish()?,
))
if self.written_size == self.size {
Ok((
ObjectId::new(&self.digest.finalize()),
self.writer.finish()?,
))
}
else {
Err(ErrorKind::MismatchingObjectSize(self.written_size, self.size).into())
}
}
}
@@ -78,6 +88,7 @@ impl<W: Write, D: Digest> Write for ObjectWriter<W, D> {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
let size = self.writer.write(buf)?;
self.digest.update(&buf[..size]);
self.written_size += size as u64;
Ok(size)
}
@@ -87,6 +98,61 @@ impl<W: Write, D: Digest> Write for ObjectWriter<W, D> {
}
pub struct ObjectReader<R: Read> {
otype: ObjectType,
size: u64,
reader: GzDecoder<R>,
}
impl<R: Read> ObjectReader<R> {
pub fn new(reader: R)
-> Result<ObjectReader<R>> {
let mut zreader = GzDecoder::new(reader);
let mut buffer = [0u8; 12];
zreader.read_exact(&mut buffer)?;
let otype = {
let mut otype = [0; 4];
otype.copy_from_slice(&buffer[0..4]);
otype
};
let size = {
let mut size_bytes = [0; 8];
size_bytes.copy_from_slice(&buffer[4..12]);
u64::from_be_bytes(size_bytes)
};
Ok(ObjectReader {
otype,
size,
reader: zreader,
})
}
pub fn object_type(&self) -> ObjectType {
self.otype
}
pub fn size(&self) -> u64 {
self.size
}
pub fn close(self) -> Result<R> {
Ok(self.reader.into_inner())
}
}
impl<R: Read> Read for ObjectReader<R> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
self.reader.read(buf)
}
}
pub fn write_object<R, W>(reader: &mut R, writer: W,
otype: &ObjectType, size: u64)
-> Result<(ObjectId, W)> where
@@ -101,49 +167,23 @@ pub fn write_object<R, W>(reader: &mut R, writer: W,
}
pub struct ObjectReader<R: Read> {
reader: GzDecoder<R>,
}
pub trait WriteAsObject {
fn write_as_object<W: Write>(&mut self, writer: W, otype: &ObjectType) -> Result<ObjectId>;
impl<R: Read> ObjectReader<R> {
pub fn new(reader: R)
-> Result<(ObjectType, u64, ObjectReader<R>)> {
let mut zreader = GzDecoder::new(reader);
let mut buffer = [0u8; 12];
zreader.read_exact(&mut buffer)?;
let otype = {
let mut otype = [0; 4];
otype.copy_from_slice(&buffer[0..4]);
otype
};
let size = {
let mut size_bytes = [0; 8];
size_bytes.copy_from_slice(&buffer[4..12]);
u64::from_be_bytes(size_bytes)
};
Ok((
otype,
size,
ObjectReader {
reader: zreader,
}
))
}
pub fn close(self) -> Result<R> {
Ok(self.reader.into_inner())
fn object_id(&mut self, otype: &ObjectType) -> Result<ObjectId> {
self.write_as_object(sink(), otype)
}
}
impl<R: Read> Read for ObjectReader<R> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
self.reader.read(buf)
impl<T: Read + Seek> WriteAsObject for T {
fn write_as_object<W: Write>(&mut self, writer: W, otype: &ObjectType) -> Result<ObjectId> {
let start = self.seek(SeekFrom::Current(0))?;
let end = self.seek(SeekFrom::End(0))?;
self.seek(SeekFrom::Start(start))?;
let (oid, _) = write_object(self, writer, otype, end - start)?;
Ok(oid)
}
}
@@ -152,27 +192,58 @@ impl<R: Read> Read for ObjectReader<R> {
mod tests {
use super::*;
const PAYLOAD: &[u8; 12] = b"Hello World!";
const PAYLOAD_OID: &str = "c3b4032160b015b2261530532a6c49f2bdadbe0687fb1f5a6a32e083";
#[test]
fn object_read_write() {
fn object_read_write() -> Result<()> {
use std::io::{Cursor, Seek, SeekFrom};
let payload = b"Hello World!";
let mut source = Cursor::new(payload);
let mut source = Cursor::new(PAYLOAD);
let mut fake_file = Cursor::new(vec![]);
let mut output = vec![];
let (oid, _) = write_object(&mut source, &mut fake_file,
TYPE_BLOB, payload.len() as u64).unwrap();
OTYPE_BLOB, PAYLOAD.len() as u64)?;
assert_eq!(oid, ObjectId::from_str("c3b4032160b015b2261530532a6c49f2bdadbe0687fb1f5a6a32e083").unwrap());
assert_eq!(oid, ObjectId::from_str(PAYLOAD_OID)?);
fake_file.seek(SeekFrom::Start(0)).unwrap();
let (otype, size, mut reader) = ObjectReader::new(fake_file).unwrap();
let read_size = reader.read_to_end(&mut output).unwrap();
fake_file.seek(SeekFrom::Start(0))?;
let mut reader = ObjectReader::new(fake_file)?;
let read_size = reader.read_to_end(&mut output)?;
assert_eq!(otype, *TYPE_BLOB);
assert_eq!(size, payload.len() as u64);
assert_eq!(read_size, payload.len());
assert_eq!(output, payload);
assert_eq!(reader.object_type(), *OTYPE_BLOB);
assert_eq!(reader.size(), PAYLOAD.len() as u64);
assert_eq!(read_size, PAYLOAD.len());
assert_eq!(output, PAYLOAD);
Ok(())
}
#[test]
fn object_write_invalid_size() {
let mut source = PAYLOAD.as_ref();
let fake_file = vec![];
let result = write_object(&mut source, fake_file, OTYPE_BLOB, 13);
match result {
Err(Error(ErrorKind::MismatchingObjectSize(actual, expected), _))
if actual == 12 && expected == 13 => (),
Err(error) => panic!("Unexpected error: {:?}", error),
Ok(_) => panic!("Unexpected success"),
}
}
#[test]
fn write_as_object() -> Result<()> {
use std::io::{Cursor, sink};
let mut source = Cursor::new(PAYLOAD);
let oid = source.write_as_object(sink(), OTYPE_BLOB)?;
assert_eq!(oid, ObjectId::from_str(PAYLOAD_OID)?);
Ok(())
}
}

71
libbsv/src/simple_db/simple_db.rs
View File

@@ -14,13 +14,25 @@
// along with cdb. If not, see <https://www.gnu.org/licenses/>.
extern crate tempfile;
use std::path::{Path, PathBuf};
use std::io::{Read, Seek};
use std::fs::{File, OpenOptions, create_dir, create_dir_all};
use tempfile::{NamedTempFile};
use crate::core::error::*;
use crate::core::ObjectId;
use super::object::{
OTYPE_BLOB,
ObjectReader, ObjectType, WriteAsObject,
};
const OBJECTS_FOLDER: &str = "objects";
const OBJECTS_DIR: &str = "objects";
const TMP_DIR: &str = "tmp";
#[derive(Debug)]
@@ -37,6 +49,14 @@ impl SimpleDb {
}
pub fn setup(&self) -> Result<()> {
create_dir(self.objects_dir())?;
create_dir(self.tmp_dir())?;
Ok(())
}
pub fn has_object(&self, oid: &ObjectId) -> bool {
let obj_path = self.path_from_id(oid);
if let Ok(metadata) = std::fs::metadata(obj_path) {
@@ -47,16 +67,63 @@ impl SimpleDb {
}
}
pub fn store_object<R: Read + Seek>(&self, otype: &ObjectType, mut reader: R)
-> Result<ObjectId> {
let mut tmp_file = self.create_tmp_file()?;
let oid = reader.write_as_object(&mut tmp_file, otype)?;
let dst_file_path = self.path_from_id(&oid);
// TODO: Check if dst_file_path exists
create_dir_all(dst_file_path.parent().unwrap())?;
match tmp_file.persist(dst_file_path) {
Ok(file) => {
file.sync_data()?;
Ok(oid)
},
Err(err) => Err(err.error.into()),
}
}
pub fn store_file_as_blob(&self, file_path: &Path) -> Result<ObjectId> {
let file = OpenOptions::new().read(true).open(file_path)?;
self.store_object(OTYPE_BLOB, file)
}
pub fn read_object(&self, oid: &ObjectId) -> Result<ObjectReader<File>> {
let path = self.path_from_id(oid);
let file = OpenOptions::new().read(true).open(path)?;
ObjectReader::new(file)
}
fn objects_dir(&self) -> PathBuf {
let mut path = self.path.clone();
path.push(OBJECTS_DIR);
path
}
fn tmp_dir(&self) -> PathBuf {
let mut path = self.path.clone();
path.push(TMP_DIR);
path
}
fn path_from_id(&self, oid: &ObjectId) -> PathBuf {
let oid_str = oid.to_string();
let mut path = self.path.clone();
path.push(OBJECTS_FOLDER);
path.push(OBJECTS_DIR);
path.push(oid_str.get(..4).unwrap()); // unwrap is ok here because we know oid_str is a
path.push(oid_str.get(4..).unwrap()); // hexadecimal number (i.e. ascii only).
path
}
fn create_tmp_file(&self) -> Result<NamedTempFile> {
let tmp_dir = self.tmp_dir();
Ok(tempfile::Builder::new().tempfile_in(tmp_dir)?)
}
}

61
libbsv/tests/simple_db.rs
View File

@@ -18,23 +18,72 @@ extern crate tempfile;
extern crate libbsv;
use std::path::{PathBuf};
use std::io::{Cursor, Read};
use std::fs::{create_dir_all, write};
use tempfile::{TempDir};
use libbsv::{
Result,
ObjectId,
simple_db::{
OTYPE_BLOB,
SimpleDb,
}
};
const PAYLOAD: &[u8; 12] = b"Hello World!";
const PAYLOAD_OID: &str = "c3b4032160b015b2261530532a6c49f2bdadbe0687fb1f5a6a32e083";
#[test]
fn simple_db_has_object() {
let temp_dir = tempfile::TempDir::new().unwrap();
let db = libbsv::SimpleDb::new(temp_dir.path()).unwrap();
let oid = libbsv::ObjectId::from_str("0001020304050607").unwrap();
let temp_dir = TempDir::new().unwrap();
let db = SimpleDb::new(temp_dir.path()).unwrap();
db.setup().unwrap();
let oid = ObjectId::from_str("0001020304050607").unwrap();
assert!(!db.has_object(&oid));
let mut object_path: std::path::PathBuf = temp_dir.path().into();
let mut object_path: PathBuf = temp_dir.path().into();
object_path.push("objects");
object_path.push("0001");
std::fs::create_dir_all(&object_path).unwrap();
create_dir_all(&object_path).unwrap();
object_path.push("020304050607");
std::fs::write(object_path, "test").unwrap();
write(object_path, "test").unwrap();
assert!(db.has_object(&oid));
}
#[test]
fn simple_db_store_read() -> Result<()> {
let expected_oid = ObjectId::from_str(PAYLOAD_OID)?;
let temp_dir = TempDir::new()?;
let db = SimpleDb::new(temp_dir.path())?;
db.setup()?;
assert!(!db.has_object(&expected_oid));
let oid = db.store_object(OTYPE_BLOB, Cursor::new(PAYLOAD))?;
assert!(db.has_object(&expected_oid));
assert_eq!(oid, expected_oid);
let mut reader = db.read_object(&oid)?;
assert_eq!(reader.object_type(), *OTYPE_BLOB);
assert_eq!(reader.size(), PAYLOAD.len() as u64);
let mut data = vec![];
reader.read_to_end(&mut data)?;
assert_eq!(data, *PAYLOAD);
Ok(())
}