add content-addressable crate

Signed-off-by: Dave Grantham <dwh@linuxprogrammer.org>

Author: dhuseby

Cargo.toml

@@ -50,6 +50,7 @@ serde_json = { version = "1.0.104"}
 serde_test = { version = "1.0.104"}
 test-log = { version = "0.2.17", features = ["trace", "color"] }
 thiserror = "2.0.12"
+tokio = { version = "1.44.2", features = ["fs", "io-util", "macros", "rt", "test-util"] }
 tracing = "0.1.41"
 tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }
 unsigned-varint = { version = "0.8.0", features = ["std"] }

crates/content-addressable/Cargo.toml

@@ -0,0 +1,44 @@
+[package]
+name = "content-addressable"
+version.workspace = true
+edition.workspace = true
+authors = ["Dave Grantham <dwg@linuxprogrammer.org>"]
+description = "Content addressable storage traits and implementations"
+readme = "README.md"
+license = "Apache-2.0"
+
+[features]
+default = ["serde"]
+dag_cbor = ["serde_cbor", "serde_cbor/tags", "multicid/dag_cbor" ]
+
+[dependencies]
+async-trait = "0.1.88"
+multibase.workspace = true
+multicid.workspace = true
+multicodec.workspace = true
+multihash.workspace = true
+multikey.workspace = true
+multitrait.workspace = true
+multiutil.workspace = true
+rand.workspace = true
+rand_core_6.workspace = true
+rng.workspace = true
+serde = { workspace = true, optional = true }
+serde_cbor = { workspace = true, optional = true }
+tempfile = "3.10.1"
+test-log.workspace = true
+thiserror.workspace = true
+tokio.workspace = true
+tracing.workspace = true
+
+[dev-dependencies]
+hex.workspace = true
+rand.workspace = true
+rand_core_6.workspace = true
+rng.workspace = true
+serde_cbor.workspace = true
+serde_json.workspace = true
+serde_test.workspace = true
+
+[lints]
+workspace = true

crates/content-addressable/README.md

@@ -0,0 +1,21 @@
+[![](https://img.shields.io/badge/made%20by-Cryptid%20Technologies-gold.svg?style=flat-square)][CRYPTID]
+[![](https://img.shields.io/badge/project-provenance-purple.svg?style=flat-square)][PROVENANCE]
+[![](https://img.shields.io/badge/project-multiformats-blue.svg?style=flat-square)][MULTIFORMATS]
+![](https://github.com/cryptidtech/multicid/actions/workflows/rust.yml/badge.svg)
+
+# Content Addressable
+
+A Rust implementation of content addressable storage abstractions using
+[multiformats][MULTIFORMATS] [content identifiers (CID)][CID] as the content
+address container.
+
+## Current Status
+
+This crate provides a set of abstractions for resolving CIDs into data blocks,
+VLADs into CIDs, and Multikeys into CIDs. Currently the only implementation
+uses the local file system for storage.
+
+[CRYPTID]: https://cryptid.tech/
+[PROVENANCE]: https://github.com/cryptidtech/provenance-specifications/
+[MULTIFORMATS]: https://github.com/multiformats/multiformats/
+[CID]: https://docs.ipfs.tech/concepts/content-addressing/

crates/content-addressable/src/block.rs

@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: Apache-2.0
+use async_trait::async_trait;
+
+/// Abstract block that abstracts away the Cid calculation
+#[async_trait]
+pub trait Block<'a, K>: Send + Sync {
+    /// Return a reference to the data
+    async fn data(&'a self) -> &'a [u8];
+
+    /// Get the name of the block
+    async fn key(&self) -> K;
+}

crates/content-addressable/src/blocks.rs

@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: Apache-2.0
+use crate::Block;
+use async_trait::async_trait;
+
+/// Abstract block storage trait for getting and putting content addressed data
+#[async_trait]
+pub trait Blocks<'a, 'b, K> {
+    /// The error type returned
+    type Error;
+
+    /// Try to confirm a block exists
+    async fn exists(&self, key: &K) -> Result<bool, Self::Error>;
+
+    /// Try to get a block from its content address
+    async fn get(&self, key: &K) -> Result<impl Block<'b, K>, Self::Error>;
+
+    /// Try to put a block into storage
+    async fn put(&mut self, block: &'a impl Block<'a, K>) -> Result<K, Self::Error>;
+
+    /// Try to remove a block from storage
+    async fn rm(&self, key: &K) -> Result<impl Block<'b, K>, Self::Error>;
+}

crates/content-addressable/src/cids.rs

@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: Apache-2.0
+use async_trait::async_trait;
+use multicid::Cid;
+
+/// Abstract mapping from an arbitrary key to a Cid
+#[async_trait]
+pub trait Cids<K> {
+    /// The error type returned
+    type Error;
+
+    /// Try to confirm a key exists
+    async fn exists(&self, key: &K) -> Result<bool, Self::Error>;
+
+    /// Try to get a Cid from its key
+    async fn get(&self, key: &K) -> Result<Cid, Self::Error>;
+
+    /// Try to put a key and Cid into the map, returns the previous Cid value if it exists
+    async fn put(&mut self, key: &K, cid: &Cid) -> Result<Option<Cid>, Self::Error>;
+
+    /// Try to remove a key and Cid form the map
+    async fn rm(&self, key: &K) -> Result<Cid, Self::Error>;
+}

crates/content-addressable/src/error.rs

@@ -0,0 +1,46 @@
+// SPDX-License-Idnetifier: Apache-2.0
+use crate::fs;
+
+/// Errors created by this library
+#[derive(Debug, thiserror::Error)]
+#[non_exhaustive]
+pub enum Error {
+    /// formatting error
+    #[error(transparent)]
+    Fmt(#[from] std::fmt::Error),
+    /// I/O error
+    #[error(transparent)]
+    Io(#[from] std::io::Error),
+    /// Persist error
+    #[error(transparent)]
+    Persist(#[from] tempfile::PersistError),
+
+    /// A multicid error
+    #[error(transparent)]
+    Multicid(#[from] multicid::Error),
+    /// A multicodec error
+    #[error(transparent)]
+    Multicodec(#[from] multicodec::Error),
+    /// A multihash error
+    #[error(transparent)]
+    Multihash(#[from] multihash::Error),
+    /// A multikey error
+    #[error(transparent)]
+    Multikey(#[from] multikey::Error),
+    /// A multitrait error
+    #[error(transparent)]
+    Multitrait(#[from] multitrait::Error),
+    /// A multiutil error
+    #[error(transparent)]
+    Multiutil(#[from] multiutil::Error),
+    /// An Fs error
+    #[error(transparent)]
+    Fs(#[from] fs::Error),
+
+    /// A custom error for callback functions
+    #[error("Custom error: {0}")]
+    Custom(String),
+    /// A wraps any error
+    #[error(transparent)]
+    Wrapped(#[from] Box<dyn std::error::Error>),
+}

crates/content-addressable/src/fs.rs

@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: Apache-2.0
+
+/// Filesystem backed data block
+pub mod block;
+pub use block::{Block, Builder as BlockBuilder};
+
+/// Filesystem backed block storage
+pub mod blocks;
+pub use blocks::{Blocks, Builder as BlocksBuilder};
+
+/// Filesystem errors
+pub mod error;
+pub use error::Error;
+
+/// Abstract map to CIDs
+pub mod cidmap;
+pub use cidmap::CidMap;
+
+/// Filesystem backed multikey to cid mapping
+pub mod mkmap;
+pub use mkmap::{Builder as MkMapBuilder, MkMap};
+
+/// Generic content addressable storage
+pub mod storage;
+pub use storage::Storage;
+
+/// Filesystem backed vlad to cid mapping
+pub mod vladmap;
+pub use vladmap::{Builder as VladMapBuilder, VladMap};
+
+/*
+/// Filesystem backed multikey_map storage
+pub mod fsmultikey_map;
+pub use fsmultikey_map::FsMultikeyMap;
+
+*/
+
+/// Simple way to import all public symbols
+pub mod prelude {
+    pub use super::*;
+}

crates/content-addressable/src/fs/block.rs

@@ -0,0 +1,95 @@
+// SPDX-License-Identifier: Apache-2.0
+use crate::{Block as TBlock, Error};
+use async_trait::async_trait;
+use multibase::Base;
+use multicid::{cid, Cid, EncodedCid};
+use multicodec::Codec;
+use multihash::mh;
+use multiutil::{BaseEncoder, DetectedEncoder, EncodingInfo};
+use std::marker::Unpin;
+use tokio::io::{AsyncRead, AsyncReadExt};
+
+/// The hash function we used when hashing blocks
+pub const BLOCK_HASH: Codec = Codec::Blake2B256;
+
+/// Filesystem stored block
+#[derive(Clone, Debug, PartialEq)]
+pub struct Block {
+    /// The block cid
+    pub key: EncodedCid,
+    /// The block data
+    pub data: Vec<u8>,
+}
+
+#[async_trait]
+impl<'a> TBlock<'a, EncodedCid> for Block {
+    /// Return a reference to the data
+    async fn data(&'a self) -> &'a [u8] {
+        &self.data
+    }
+
+    /// Get the name of the block
+    async fn key(&self) -> EncodedCid {
+        self.key.clone()
+    }
+}
+
+/// Builder for creating blocks from readers
+#[derive(Clone, Debug)]
+pub struct Builder<R: AsyncRead> {
+    /// The base encoding
+    base: Option<Base>,
+    // The block cid
+    cid: Option<Cid>,
+    // The reader for the block data
+    reader: R,
+}
+
+impl<R: AsyncRead + Unpin> Builder<R> {
+    /// Create a new block builder
+    pub fn new(reader: R) -> Self {
+        Self {
+            base: None,
+            cid: None,
+            reader,
+        }
+    }
+
+    /// Set the base encoding
+    pub fn base(mut self, base: Base) -> Self {
+        self.base = Some(base);
+        self
+    }
+
+    /// Set the cid
+    pub fn cid(mut self, cid: Cid) -> Self {
+        self.cid = Some(cid);
+        self
+    }
+
+    /// Build the block
+    pub async fn try_build(mut self) -> Result<Block, Error> {
+        // get the base encoding
+        let base = self.base.unwrap_or(DetectedEncoder::preferred_encoding(
+            Cid::preferred_encoding(),
+        ));
+
+        // read the data from the reader
+        let mut data = Vec::new();
+        self.reader.read_to_end(&mut data).await?;
+
+        // calculate the cid from the data if not provided
+        let cid = match self.cid {
+            Some(cid) => cid,
+            None => cid::Builder::new(Codec::Cidv1)
+                .with_target_codec(Codec::DagCbor)
+                .with_hash(&mh::Builder::new_from_bytes(BLOCK_HASH, &data)?.try_build()?)
+                .try_build()?,
+        };
+
+        // encode the cid into the key
+        let key = EncodedCid::new(base, cid);
+
+        Ok(Block { key, data })
+    }
+}