Add 'lsm' shell to work with lsm trees on the command-line

Author: lambdanature

Cargo.toml

@@ -16,29 +16,48 @@ categories = ["data-structures", "database-implementations", "algorithms"]
 name = "lsm_tree"
 path = "src/lib.rs"
 
+[[bin]]
+name = "lsm"
+path = "src/tool.rs"
+required-features = ["tool"]
+
 [features]
 default = []
 lz4 = ["dep:lz4_flex"]
 bytes_1 = ["dep:bytes"]
 metrics = []
+tool = ["dep:clap", "dep:shlex", "dep:rustyline", "dep:parse-size", "dep:humansize",
+        "dep:tracing", "dep:tracing-log", "dep:tracing-panic", "dep:tracing-subscriber"]
 
 [dependencies]
 bytes = { version = "1", optional = true }
 byteorder = { package = "byteorder-lite", version = "0.1.0" }
 byteview = "~0.10.0"
+clap = { version = "4", features = ["derive"], optional = true }
 crossbeam-skiplist = "0.1.3"
 enum_dispatch = "0.3.13"
+humansize = { version = "2.1", optional = true }
 interval-heap = "0.0.5"
 log = "0.4.27"
 lz4_flex = { version = "0.11.5", optional = true, default-features = false }
+parse-size = { version = "1.0", optional = true }
 quick_cache = { version = "0.6.16", default-features = false, features = [] }
 rustc-hash = "2.1.1"
+rustyline = { version = "15", optional = true }
 self_cell = "1.2.0"
 sfa = "~1.0.0"
+shlex = { version = "1", optional = true }
 tempfile = "3.20.0"
+tracing = { version = "0.1", optional = true }
+tracing-log = { version = "0.2", optional = true }
+tracing-panic = { version = "0.1.2", optional = true }
+tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt", "registry"], optional = true }
 varint-rs = "2.2.0"
 xxhash-rust = { version = "0.8.15", features = ["xxh3"] }
 
+[target.'cfg(unix)'.dev-dependencies]
+rexpect = "0.5"
+
 [dev-dependencies]
 criterion = { version = "0.8.0", features = ["html_reports"] }
 fs_extra = "1.3.0"

README.md

@@ -62,6 +62,149 @@ Uses [`bytes`](https://github.com/tokio-rs/bytes) as the underlying `Slice` type
 
 *Disabled by default.*
 
+### tool
+
+Enables the `lsm` CLI binary for interacting with LSM trees from the command line.
+
+*Disabled by default.*
+
+## CLI Tool
+
+The crate includes an optional CLI tool (`lsm`) for inspecting and manipulating LSM trees.
+
+### Installation
+
+```bash
+cargo install lsm-tree --features tool
+```
+
+Or build from source:
+
+```bash
+cargo build --release --features tool
+```
+
+### Usage
+
+The tool can be used either with direct commands or in interactive shell mode.
+
+#### Direct Commands
+
+```bash
+# Set a key-value pair
+lsm /path/to/db set mykey "my value"
+
+# Get a value
+lsm /path/to/db get mykey
+
+# Delete a key
+lsm /path/to/db del mykey
+
+# List all keys (aliases: list, ls)
+lsm /path/to/db scan
+
+# List keys with a prefix
+lsm /path/to/db scan "user:"
+
+# List keys in a range [start, end)
+lsm /path/to/db range a z
+
+# Count items
+lsm /path/to/db count
+
+# Show database info
+lsm /path/to/db info
+
+# Flush memtable to disk
+lsm /path/to/db flush
+
+# Run compaction
+lsm /path/to/db compact
+```
+
+#### Interactive Shell
+
+Start an interactive shell by running without a command:
+
+```bash
+lsm /path/to/db
+```
+
+The shell supports all the above commands plus:
+
+- `begin` - Start a batch/transaction
+- `commit` - Commit the current batch
+- `rollback` - Discard the current batch
+- `exit` / `quit` - Exit (flushes data first)
+- `abort` - Exit without flushing
+- `help` - Show available commands
+
+#### Batch Operations
+
+The shell supports batching multiple operations into an atomic unit:
+
+```
+lsm> begin
+OK (batch started)
+lsm> set key1 value1
+OK (batched, ready to commit)
+lsm> set key2 value2
+OK (batched, ready to commit)
+lsm> del key3
+OK (batched, ready to commit)
+lsm> commit
+OK (batch committed, ready to flush)
+```
+
+While a batch is active:
+- `get` reads from the batch first, then falls back to the tree
+- `scan` and `range` warn that they ignore uncommitted batch operations
+- `info` shows the pending batch operations
+- `rollback` discards all batched operations
+
+#### Long Scan
+
+Use `-l` / `--long` to show internal entry details including sequence numbers, value types, and tombstones:
+
+```
+lsm> scan -l
+=== Active Memtable ===
+key1 = value1 [seqno=0, type=Value]
+key2 [seqno=1, type=Tombstone]
+
+=== Persisted (on disk) ===
+key3 = value3 [seqno=2, type=Value]
+
+(3 total items, 2 in memtable, 1 persisted, 1 tombstones)
+```
+
+#### Blob Trees with Indirect Items
+
+A blob tree uses key-value separation, storing large values in separate blob files and keeping indirect references (indirections) in the main LSM-tree. This improves performance for large values by reducing write amplification and improving compaction efficiency.
+
+To create a blob tree, use the `--blob-tree` flag along with `--separation-threshold` (or `-t`) to specify the size threshold in bytes. Values larger than this threshold will be stored as indirect items:
+
+```bash
+# Create a blob tree with 1 KiB separation threshold
+lsm --blob-tree --separation-threshold 1024 /path/to/db set largekey "very large value..."
+
+# Or using the short form
+lsm -b -t 1KiB /path/to/db set largekey "very large value..."
+
+# In interactive mode
+lsm --blob-tree -t 1024 /path/to/db
+lsm> set largekey "very large value..."
+OK (set)
+lsm> flush
+OK (flushed)
+lsm> scan -l
+=== Active Memtable ===
+=== Persisted (on disk) ===
+largekey = very large value... [seqno=0, type=Indirection]
+```
+
+After flushing, values that exceed the separation threshold will appear as `type=Indirection` in verbose scan output, indicating they are stored in separate blob files rather than inline in the table.
+
 ## Run unit benchmarks
 
 ```bash