libsql wal refactors (#1539)

* introduce IoBuf::map_slice

* rename db_size to size_after

* introduce Segment trait

* implement Segment for Arc<T>

* make SealedSegment clone

* implement Segment for SealedSegment

* rename bottomless to storage

- introduce `Storage` trait
- rename old `Storage` to `Backend`
- introduce `AsyncStorage` (former bottomless)

* add Storage to WalRegistry

remove namespace resolver from registry

* add async_get to registry

* decouple shared wal from registry

* move CompactedSegment headers to own module

* refactor SegmentList

* add durable frame_no to shared_wal

* make shared_wal checkpoint async

* expose TestEnv

create test environment for libsql-wal

* ignore dead_code

stuff still need to be wired

* let LibsqlWalManager resolve namespace name

* fix tests

* fix libsql-server

* fmt

* fix bench

* fix conflicts

* fmt

Author: MarinPostma

libsql-server/src/connection/connection_manager.rs

@@ -15,6 +15,7 @@ use libsql_sys::wal::either::Either3;
 use libsql_sys::wal::wrapper::{WrapWal, WrappedWal};
 use libsql_sys::wal::{CheckpointMode, Sqlite3Wal, Sqlite3WalManager, Wal};
 use libsql_wal::io::StdIO;
+use libsql_wal::storage::NoStorage;
 use libsql_wal::wal::{LibsqlWal, LibsqlWalManager};
 use metrics::atomics::AtomicU64;
 use parking_lot::{Mutex, MutexGuard};
@@ -26,12 +27,13 @@ use super::TXN_TIMEOUT;
 pub type ConnId = u64;
 #[cfg(feature = "durable-wal")]
 
-pub type InnerWalManager = Either3<Sqlite3WalManager, LibsqlWalManager<StdIO>, DurableWalManager>;
+pub type InnerWalManager =
+    Either3<Sqlite3WalManager, LibsqlWalManager<StdIO, NoStorage>, DurableWalManager>;
 #[cfg(feature = "durable-wal")]
 pub type InnerWal = Either3<Sqlite3Wal, LibsqlWal<StdIO>, DurableWal>;
 #[cfg(not(feature = "durable-wal"))]
 
-pub type InnerWalManager = Either<Sqlite3WalManager, LibsqlWalManager<StdIO>>;
+pub type InnerWalManager = Either<Sqlite3WalManager, LibsqlWalManager<StdIO, NoStorage>>;
 #[cfg(not(feature = "durable-wal"))]
 
 pub type InnerWal = Either<Sqlite3Wal, LibsqlWal<StdIO>>;

libsql-server/src/lib.rs

@@ -42,6 +42,7 @@ use libsql_sys::wal::either::Either as EitherWAL;
 use libsql_sys::wal::either::Either3 as EitherWAL;
 use libsql_sys::wal::Sqlite3WalManager;
 use libsql_wal::registry::WalRegistry;
+use libsql_wal::storage::NoStorage;
 use libsql_wal::wal::LibsqlWalManager;
 use namespace::meta_store::MetaStoreHandle;
 use namespace::{NamespaceConfig, NamespaceName};
@@ -702,9 +703,9 @@ where
 
         match self.use_custom_wal {
             Some(CustomWAL::LibsqlWal) => {
-                let registry = Arc::new(WalRegistry::new(wal_path, namespace_resolver, ())?);
+                let registry = Arc::new(WalRegistry::new(wal_path, NoStorage)?);
 
-                let wal = LibsqlWalManager::new(registry.clone());
+                let wal = LibsqlWalManager::new(registry.clone(), Arc::new(namespace_resolver));
                 let shutdown_notify = self.shutdown.clone();
                 let shutdown_fut = Box::pin(async move {
                     shutdown_notify.notified().await;

libsql-wal/benches/benchmarks.rs

@@ -7,6 +7,7 @@ use libsql_sys::rusqlite::{self, OpenFlags};
 use libsql_sys::wal::{Sqlite3Wal, Sqlite3WalManager, Wal};
 use libsql_sys::Connection;
 use libsql_wal::io::StdIO;
+use libsql_wal::storage::NoStorage;
 use libsql_wal::wal::LibsqlWal;
 use libsql_wal::{registry::WalRegistry, wal::LibsqlWalManager};
 use tempfile::tempdir;
@@ -58,8 +59,8 @@ fn with_libsql_conn(f: impl FnOnce(&mut Connection<LibsqlWal<StdIO>>)) {
     let tmp = tempdir().unwrap();
     let resolver = |_: &Path| NamespaceName::from_string("test".into());
 
-    let registry = Arc::new(WalRegistry::new(tmp.path().join("wals"), resolver, ()).unwrap());
-    let wal_manager = LibsqlWalManager::new(registry.clone());
+    let registry = Arc::new(WalRegistry::new(tmp.path().join("wals"), NoStorage).unwrap());
+    let wal_manager = LibsqlWalManager::new(registry.clone(), Arc::new(resolver));
 
     let mut conn = libsql_sys::Connection::open(
         tmp.path().join("data"),