Merge pull request #12 from monoscope-tech/memtable

Introduce a WAL, and write data in memory which gets saved to delta in 10mins interval, to reduce the small files problem.

Author: tonyalaribe

.env.minio

@@ -21,6 +21,9 @@ MAX_PG_CONNECTIONS=100
 # MinIO doesn't need DynamoDB locking, use local locking
 AWS_S3_LOCKING_PROVIDER=""
 
+# WAL storage directory for walrus-rust
+WALRUS_DATA_DIR=/tmp/walrus-wal
+
 # Foyer cache configuration for tests
 TIMEFUSION_FOYER_MEMORY_MB=256
 TIMEFUSION_FOYER_DISK_GB=10

.github/workflows/ci.yml

@@ -62,10 +62,13 @@ jobs:
       ENABLE_BATCH_QUEUE: "true"
       MAX_PG_CONNECTIONS: "100"
       AWS_S3_LOCKING_PROVIDER: ""
-      TIMEFUSION_FOYER_MEMORY_MB: "256"
-      TIMEFUSION_FOYER_DISK_GB: "10"
-      TIMEFUSION_FOYER_TTL_SECONDS: "300"
-      TIMEFUSION_FOYER_SHARDS: "8"
+      WALRUS_DATA_DIR: /tmp/walrus-wal
+      # Use small cache sizes for CI tests (similar to test_config in object_store_cache.rs)
+      TIMEFUSION_FOYER_MEMORY_MB: "10"
+      TIMEFUSION_FOYER_DISK_MB: "50"
+      TIMEFUSION_FOYER_METADATA_MEMORY_MB: "10"
+      TIMEFUSION_FOYER_METADATA_DISK_MB: "50"
+      TIMEFUSION_FOYER_SHARDS: "2"
     services:
       minio:
         image: public.ecr.aws/bitnami/minio:latest

.gitignore

@@ -9,3 +9,4 @@ minio
 dis-newstyle
 *.log
 .DS_Store
+wal_files/

Cargo.lock

@@ -6,6 +6,7 @@ edition = "2024"
 [dependencies]
 tokio = { version = "1.48", features = ["full"] }
 datafusion = "51.0.0"
+datafusion-datasource = "51.0.0"
 arrow = "57.1.0"
 arrow-json = "57.1.0"
 uuid = { version = "1.17", features = ["v4", "serde"] }
@@ -68,8 +69,12 @@ ahash = "0.8"
 lru = "0.16.1"
 serde_bytes = "0.11.19"
 dashmap = "6.1"
+envy = "0.4"
 tdigests = "1.0"
-bincode = "2.0"
+bincode = { version = "2.0", features = ["serde"] }
+walrus-rust = "0.2.0"
+thiserror = "2.0"
+strum = { version = "0.27", features = ["derive"] }
 
 [dev-dependencies]
 sqllogictest = { git = "https://github.com/risinglightdb/sqllogictest-rs.git" }
@@ -78,6 +83,7 @@ datafusion-common = "51.0.0"
 tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4"] }
 scopeguard = "1.2.0"
 rand = "0.9.2"
+tempfile = "3"
 
 [features]
 default = []