Skip to content

Commit 2f82385

Browse files
committed
feat: WASM compatibility and registry refactor
WASM refactor: - Gate native-only code (spawn_blocking, filesystem, redb) behind #[cfg(not(target_arch = "wasm32"))] - Add cross-platform spawn utilities (spawn_task, spawn_delayed) in s5_fs - Split BlobStore imports into WASM-compatible (inline hash) and native-only (with outboard) variants - Add `server` feature to s5_blobs to gate native-only components Registry refactor: - Extract RedbRegistry to registries/redb crate (native-only) - Add registries/store for BlobStore-backed registry (WASM-compatible) - Make RegistryServer generic over any RegistryApi implementation - Add MemoryRegistry, TeeRegistry, MultiRegistry implementations - Add Delete RPC to registry protocol - All registry implementations now respect should_store semantics New features: - PinBlob RPC for pinning existing blobs without re-upload - Blinded queries for zero-knowledge availability checks - MultiFetcher for multi-source blob fetching with fallback - Store testutil module for testing Store implementations Bug fixes: - GC now aborts on root.fs5.cbor decryption failure instead of treating root as empty (which could mark all blobs as GC candidates)
1 parent 3d4515d commit 2f82385

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+2343
-183
lines changed

Cargo.lock

Lines changed: 38 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,25 @@
11
[workspace]
22
resolver = "3"
33
members = [
4-
"blob_stores/local",
5-
"blob_stores/memory",
6-
"blob_stores/s3",
7-
"blob_stores/sia",
8-
"importers/http",
9-
"importers/local",
4+
# Core crates
105
"s5_blobs",
116
"s5_cli",
127
"s5_core",
138
"s5_fs",
9+
"s5_fuse",
1410
"s5_node",
1511
"s5_registry",
16-
"s5_fuse",
12+
# Blob stores
13+
"blob_stores/local",
14+
"blob_stores/memory",
15+
"blob_stores/s3",
16+
"blob_stores/sia",
17+
# Registries
18+
"registries/redb",
19+
"registries/store",
20+
# Importers
21+
"importers/http",
22+
"importers/local",
1723
]
1824

1925
[workspace.dependencies]
@@ -33,21 +39,25 @@ log = "0.4.28"
3339
minicbor = { version = "2.1.1", features = ["alloc", "derive", "half"] }
3440
postcard = "1.1.3"
3541
redb = "3.1.0"
36-
s5_blobs = { path = "s5_blobs", version = "1.0.0-beta.1" }
42+
s5_blobs = { path = "s5_blobs", version = "1.0.0-beta.1", default-features = false }
3743
s5_core = { path = "s5_core", version = "1.0.0-beta.1" }
3844
s5_fs = { path = "s5_fs", version = "1.0.0-beta.1" }
3945
s5_fuse = { path = "s5_fuse", version = "1.0.0-beta.1" }
4046
s5_importer_http = { path = "importers/http", version = "1.0.0-beta.1" }
4147
s5_importer_local = { path = "importers/local", version = "1.0.0-beta.1" }
4248
s5_node = { path = "s5_node", version = "1.0.0-beta.1" }
4349
s5_registry = { path = "s5_registry", version = "1.0.0-beta.1" }
50+
s5_registry_redb = { path = "registries/redb", version = "1.0.0-beta.1" }
51+
s5_registry_store = { path = "registries/store", version = "1.0.0-beta.1" }
4452
s5_store_local = { path = "blob_stores/local", version = "1.0.0-beta.1" }
4553
s5_store_memory = { path = "blob_stores/memory", version = "1.0.0-beta.1" }
4654
s5_store_s3 = { path = "blob_stores/s3", version = "1.0.0-beta.1" }
4755
s5_store_sia = { path = "blob_stores/sia", version = "1.0.0-beta.1" }
4856
serde = { version = "1.0.228", features = ["derive"] }
4957
thiserror = "2.0.17"
50-
tokio = { version = "1.48.0", features = ["full"] }
58+
# WASM-compatible tokio features: sync, macros, io-util, rt, time
59+
# Native crates that need more features should add them explicitly
60+
tokio = { version = "1.48.0", features = ["sync", "macros", "io-util", "rt", "time"] }
5161
tokio-util = { version = "0.7.16", features = ["io"] }
5262
tracing = "0.1.41"
5363
criterion = { version = "0.5.1", features = ["async_tokio"], default-features = false }

blob_stores/local/Cargo.toml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,11 @@ bytes.workspace = true
1313
futures.workspace = true
1414
s5_core.workspace = true
1515
serde.workspace = true
16-
tokio.workspace = true
16+
# Local store needs tokio fs (native-only crate)
17+
tokio = { version = "1.48.0", features = ["sync", "macros", "io-util", "rt", "fs"] }
1718
tokio-util.workspace = true
1819
walkdir = "2.5.0"
20+
21+
[dev-dependencies]
22+
s5_core = { workspace = true, features = ["testutil"] }
23+
tempfile = "3"

blob_stores/local/src/lib.rs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,3 +204,16 @@ impl s5_core::store::Store for LocalStore {
204204
Ok(Box::new(stream))
205205
}
206206
}
207+
208+
#[cfg(test)]
209+
mod tests {
210+
use super::*;
211+
use s5_core::testutil::StoreTests;
212+
213+
#[tokio::test]
214+
async fn test_local_store() {
215+
let temp_dir = tempfile::tempdir().unwrap();
216+
let store = LocalStore::new(temp_dir.path());
217+
StoreTests::new(&store).run_all().await.unwrap();
218+
}
219+
}

blob_stores/memory/Cargo.toml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,7 @@ bytes.workspace = true
1212
dashmap = "6.1.0"
1313
futures.workspace = true
1414
s5_core.workspace = true
15+
16+
[dev-dependencies]
17+
s5_core = { workspace = true, features = ["testutil"] }
18+
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }

blob_stores/memory/src/lib.rs

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,3 +151,15 @@ impl s5_core::store::Store for MemoryStore {
151151
Ok(vec![])
152152
}
153153
}
154+
155+
#[cfg(test)]
156+
mod tests {
157+
use super::*;
158+
use s5_core::testutil::StoreTests;
159+
160+
#[tokio::test]
161+
async fn test_memory_store() {
162+
let store = MemoryStore::new();
163+
StoreTests::new(&store).run_all().await.unwrap();
164+
}
165+
}

blob_stores/s3/Cargo.toml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,5 +14,9 @@ futures.workspace = true
1414
rust-s3 = { version = "0.37.0" }
1515
s5_core.workspace = true
1616
serde.workspace = true
17-
tokio.workspace = true
17+
# S3 store uses async io
18+
tokio = { version = "1.48.0", features = ["sync", "macros", "io-util", "rt"] }
1819
tokio-util = { workspace = true, features = ["io"] }
20+
21+
[dev-dependencies]
22+
s5_core = { workspace = true, features = ["testutil"] }

blob_stores/s3/src/lib.rs

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,3 +154,27 @@ impl s5_core::store::Store for S3Store {
154154
Ok(Box::new(stream))
155155
}
156156
}
157+
158+
#[cfg(test)]
159+
mod tests {
160+
// S3 tests require a running S3-compatible server (e.g., MinIO)
161+
// They are ignored by default
162+
#[allow(unused_imports)]
163+
use super::*;
164+
#[allow(unused_imports)]
165+
use s5_core::testutil::StoreTests;
166+
167+
#[tokio::test]
168+
#[ignore = "requires S3-compatible server"]
169+
async fn test_s3_store() {
170+
let config = S3StoreConfig {
171+
endpoint: "http://localhost:9000".to_string(),
172+
region: "us-east-1".to_string(),
173+
bucket_name: "test-bucket".to_string(),
174+
access_key: "minioadmin".to_string(),
175+
secret_key: "minioadmin".to_string(),
176+
};
177+
let store = S3Store::create(config);
178+
StoreTests::new(&store).run_all().await.unwrap();
179+
}
180+
}

blob_stores/sia/Cargo.toml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ s5_core.workspace = true
2323
serde.workspace = true
2424
serde_json = "1.0.142"
2525
thiserror = "2.0.12"
26-
tokio.workspace = true
26+
# Sia store uses async io
27+
tokio = { version = "1.48.0", features = ["sync", "macros", "io-util", "rt"] }
2728
tokio-util.workspace = true
2829
anyhow.workspace = true
2930

importers/http/Cargo.toml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ reqwest = { version = "0.12.23", features = ["stream"] }
1515
s5_core.workspace = true
1616
s5_fs.workspace = true
1717
scraper = "0.23.1"
18-
tokio.workspace = true
18+
# HTTP importer uses tokio sync primitives
19+
tokio = { version = "1.48.0", features = ["sync", "macros", "rt"] }
1920
tracing = "0.1.41"
2021
tracing-subscriber = "0.3.19"
2122
url = "2.5.4"

0 commit comments

Comments
 (0)