draklaw/bsv
1
0
Fork 0
Code Issues 6 Pull Requests Projects 1 Releases Wiki Activity

Mostly implement simple-cas.

Browse Source
This commit is contained in:
Draklaw
2021-08-22 17:19:32 +02:00
parent 5664beca57
commit 25b5f4e7ba
9 changed files with 1003 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
cas-core/src/cas.rs
View File

@@ -14,10 +14,13 @@
// along with cdb. If not, see <https://www.gnu.org/licenses/>.
use super::error::Result;
use std::path::Path;
use super::error::{Error, Result};
use super::object_id::ObjectId;
use super::object_type::ObjectType;
use super::object_metadata::ObjectMetadata;
use super::pipeline::{Reader, Writer};
// pub struct ObjectIdIterator;
@@ -25,16 +28,41 @@ use super::object_metadata::ObjectMetadata;
pub trait Cas {
fn object_id_from_string(&self, hex: &str) -> Result<ObjectId>;
fn object_id_from_partial(&self, hex: &str) -> Result<ObjectId>;
fn has_object_id(&self, oid: &ObjectId) -> Result<bool>;
// fn iter_object_id(&self) -> Result<ObjectIdIterator>;
fn iter_object_id<'s>(&'s self) -> Result<Box<dyn 's + Iterator<Item = Result<ObjectId>>>>;
fn open_object(&self, oid: &ObjectId) -> Result<(ObjectMetadata, Box<dyn Reader>)>;
fn new_writer(&mut self, otype: &ObjectType, size: u64) -> Result<Box<dyn Writer>>;
fn read_object(&self, oid: &ObjectId) -> Result<(ObjectMetadata, Vec<u8>)> {
let (metadata, mut reader) = self.open_object(oid)?;
let mut data = Vec::new();
reader.read_to_end(&mut data).or_else(|err|
Err(Error::error(format!("failed to read object {}: {}", oid, err))))?;
let result_oid = reader.finalize()?;
if &result_oid == oid {
Ok((metadata, data))
}
else {
Err(Error::error(format!("object id mismatch: requested {}, read {}", oid, result_oid)))
}
}
fn write_object(&mut self, otype: &ObjectType, data: &[u8]) -> Result<ObjectId> {
let mut writer = self.new_writer(otype, data.len() as u64)?;
writer.write_all(data).or_else(|err|
Err(Error::error(format!("failed to write object: {}", err))))?;
writer.finalize()
}
fn read_object(&self, oid: &ObjectId) -> Result<(ObjectMetadata, &dyn std::io::Read)>;
fn write_object(&mut self, otype: ObjectType, data: &mut dyn std::io::Read) -> Result<ObjectId>;
fn remove_object(&mut self, oid: &ObjectId) -> Result<()>;
}
fn read_ref(&self, key: &str) -> Result<ObjectId>;
fn write_ref(&mut self, key: &str, value: &ObjectId) -> Result<()>;
fn remove_ref(&mut self, key: &str) -> Result<()>;
pub trait RefStore {
fn get_ref<P: AsRef<Path>>(&self, key: P) -> Result<ObjectId>;
fn set_ref<P: AsRef<Path>>(&mut self, key: P, value: &ObjectId) -> Result<()>;
fn remove_ref<P: AsRef<Path>>(&mut self, key: P) -> Result<()>;
}

4
cas-core/src/error.rs
View File

@@ -111,6 +111,10 @@ impl Error {
pub fn error<M: Into<String>>(message: M) -> Box<Error> {
Box::new(Error::Error(message.into()))
}
pub fn err<M: Into<String>, T>(message: M) -> Result<T> {
Err(Self::error(message))
}
}

2
cas-core/src/lib.rs
View File

@@ -38,7 +38,7 @@ pub use crate::{
object_id::{ObjectId, hex, write_hex},
object_type::{ObjectType},
object_metadata::{ObjectMetadata},
pipeline::{Pipeline, DefaultPipeline},
pipeline::{Pipeline, DefaultPipeline, Reader, Writer, ReadWrapper, WriteWrapper},
cas::{Cas},
};

6
cas-core/src/object_metadata.rs
View File

@@ -20,11 +20,11 @@ use super::object_type::{ObjectType};
#[derive(Clone, Eq, PartialEq)]
pub struct ObjectMetadata {
otype: ObjectType,
size: usize,
size: u64,
}
impl ObjectMetadata {
pub fn new(otype: ObjectType, size: usize) -> Self {
pub fn new(otype: ObjectType, size: u64) -> Self {
Self {
otype,
size,
@@ -35,7 +35,7 @@ impl ObjectMetadata {
&self.otype
}
pub fn size(&self) -> usize {
pub fn size(&self) -> u64 {
self.size
}
}

119
cas-core/src/pipeline.rs
View File

@@ -19,29 +19,13 @@ use super::error::*;
use super::object_id::ObjectId;
pub trait Reader {
fn read(&mut self, buf: &mut [u8]) -> Result<usize>;
pub trait Reader: std::io::Read {
fn finalize(self: Box<Self>) -> Result<ObjectId>;
fn read_all(&mut self) -> Result<Vec<u8>> {
let mut buffer = [0u8; 1 << 13];
let mut data = Vec::new();
let mut count = usize::MAX;
while count != 0 {
count = self.read(&mut buffer)?;
if count != 0 {
data.extend_from_slice(&buffer[..count]);
}
}
Ok(data)
}
}
pub trait Writer {
fn write(&mut self, buf: &[u8]) -> Result<()>;
pub trait Writer: std::io::Write {
fn finalize(self: Box<Self>) -> Result<ObjectId>;
fn _finalize(self: Box<Self>, oid: ObjectId) -> Result<ObjectId>;
}
@@ -51,29 +35,57 @@ pub trait Pipeline {
}
impl<R: std::io::Read> Reader for R {
fn read(&mut self, buf: &mut [u8]) -> Result<usize> {
<Self as std::io::Read>::read(self, buf).or_else(|err|
Err(Error::error(format!("Read error: {}", err)))
)
}
pub struct ReadWrapper<R: std::io::Read> {
reader: R,
}
impl<R: std::io::Read> ReadWrapper<R> {
pub fn new(reader: R) -> Self {
Self { reader }
}
}
impl<R: std::io::Read> std::io::Read for ReadWrapper<R> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
self.reader.read(buf)
}
}
impl<R: std::io::Read> Reader for ReadWrapper<R> {
fn finalize(self: Box<Self>) -> Result<ObjectId> {
Err(Error::error("reader pipline has no digest step"))
}
}
impl<W: std::io::Write> Writer for W {
fn write(&mut self, buf: &[u8]) -> Result<()> {
self.write_all(buf).or_else(|err|
Err(Error::error(format!("Write error: {}", err)))
)
pub struct WriteWrapper<W: std::io::Write> {
writer: W,
}
impl<W: std::io::Write> WriteWrapper<W> {
pub fn new(writer: W) -> Self {
Self { writer }
}
}
impl<W: std::io::Write> std::io::Write for WriteWrapper<W> {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.writer.write(buf)
}
fn flush(&mut self) -> std::io::Result<()> {
self.writer.flush()
}
}
impl<W: std::io::Write> Writer for WriteWrapper<W> {
fn finalize(self: Box<Self>) -> Result<ObjectId> {
Err(Error::error("reader pipline has no digest step"))
}
fn _finalize(self: Box<Self>, oid: ObjectId) -> Result<ObjectId> {
Ok(oid)
}
}
@@ -93,8 +105,8 @@ impl<'a> DigestReader<'a> {
}
}
impl<'a> Reader for DigestReader<'a> {
fn read(&mut self, buf: &mut [u8]) -> Result<usize> {
impl<'a> std::io::Read for DigestReader<'a> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let count = if let Some(dig) = &mut self.digest {
let count = self.reader.read(buf)?;
dig.update(&buf[..count]);
@@ -112,7 +124,9 @@ impl<'a> Reader for DigestReader<'a> {
Ok(count)
}
}
impl<'a> Reader for DigestReader<'a> {
fn finalize(self: Box<Self>) -> Result<ObjectId> {
self.oid.ok_or_else(|| Error::error("Reader not finalized"))
}
@@ -133,15 +147,26 @@ impl<'a> DigestWriter<'a> {
}
}
impl<'a> Writer for DigestWriter<'a> {
fn write(&mut self, buf: &[u8]) -> Result<()> {
self.digest.update(buf);
self.writer.write(buf)?;
Ok(())
impl<'a> std::io::Write for DigestWriter<'a> {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
let count = self.writer.write(buf)?;
self.digest.update(&buf[..count]);
Ok(count)
}
fn flush(&mut self) -> std::io::Result<()> {
self.writer.flush()
}
}
impl<'a> Writer for DigestWriter<'a> {
fn finalize(self: Box<Self>) -> Result<ObjectId> {
Ok(ObjectId::new(&self.digest.finalize()))
let oid = ObjectId::new(&self.digest.finalize());
self.writer._finalize(oid)
}
fn _finalize(self: Box<Self>, _oid: ObjectId) -> Result<ObjectId> {
Err(Error::error("writer pipeline has several digest steps"))
}
}
@@ -182,9 +207,10 @@ mod tests {
#[test]
fn std_read_as_reader() {
let data = b"Hello World!".to_vec();
let mut reader: Box<dyn Reader> = Box::new(data.as_slice());
let mut reader: Box<dyn Reader> = Box::new(ReadWrapper::new(data.as_slice()));
let data2 = reader.read_all().expect("read failed");
let mut data2 = Vec::new();
reader.read_to_end(&mut data2).expect("read failed");
let maybe_oid = reader.finalize();
assert_eq!(data2, data);
@@ -197,7 +223,7 @@ mod tests {
let mut buffer = [0u8; 32];
let maybe_oid = {
let mut writer: Box<dyn Writer> = Box::new(&mut buffer[..]);
let mut writer: Box<dyn Writer> = Box::new(WriteWrapper::new(&mut buffer[..]));
writer.write(&data).expect("write failed");
writer.finalize()
};
@@ -210,17 +236,20 @@ mod tests {
#[test]
fn digest_reader() {
use std::io::Read;
let data = b"Hello World!".to_vec();
let oid = ObjectId::from_str(
"7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069"
).expect("invalid object id");
let mut reader = Box::new(DigestReader::new(
Box::new(data.as_slice()),
Box::new(ReadWrapper::new(data.as_slice())),
Box::new(Sha256::default()),
));
let data2 = reader.read_all().expect("read failed");
let mut data2 = Vec::new();
reader.read_to_end(&mut data2).expect("read failed");
let oid2 = reader.finalize().expect("finalize failed");
assert_eq!(data2, data);
@@ -229,6 +258,8 @@ mod tests {
#[test]
fn digest_writer() {
use std::io::Write;
let data = b"Hello World!".to_vec();
let oid = ObjectId::from_str(
"7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069"
@@ -237,7 +268,7 @@ mod tests {
let mut buffer = [0u8; 32];
let oid2 = {
let mut writer = Box::new(DigestWriter::new(
Box::new(&mut buffer[..]),
Box::new(WriteWrapper::new(&mut buffer[..])),
Box::new(Sha256::default()),
));
writer.write(&data).expect("write failed");