add sequential firehose + CLI support

Cargo.toml

@@ -88,6 +88,7 @@ thousands = "0"
 tokio-stream = { version = "0", default-features = false }
 rangemap = "1"
 rseek = ">= 0.2"
+ripget = "0.2"
 rayon = "1"
 xxhash-rust = { version = "0.8", features = ["xxh64"] }
 dashmap = "5"

README.md

@@ -73,6 +73,20 @@ If `JETSTREAMER_THREADS` is omitted, Jetstreamer auto-sizes the worker pool usin
 hardware-aware heuristic exposed by
 `jetstreamer_firehose::system::optimal_firehose_thread_count`.
 
+For sequential replay mode, enable `--sequential` (or `JETSTREAMER_SEQUENTIAL=1`). In this mode
+Jetstreamer uses a single firehose worker and reuses `JETSTREAMER_THREADS` as ripget parallel
+download concurrency:
+
+```bash
+# Sequential mode with CLI flag
+JETSTREAMER_THREADS=4 cargo run --release -- 800 --sequential
+
+# Sequential mode with explicit ripget window override
+JETSTREAMER_SEQUENTIAL=1 JETSTREAMER_BUFFER_WINDOW=4GiB cargo run --release -- 800
+```
+
+`JETSTREAMER_BUFFER_WINDOW` defaults to `min(4 GiB, 15% of available RAM)` when unset.
+
 The built-in program and instruction tracking plugins now record vote and non-vote activity
 separately. `program_invocations` includes an `is_vote` flag per row, while `slot_instructions`
 stores separate vote/non-vote instruction and transaction counts.

jetstreamer-firehose/Cargo.toml

@@ -41,7 +41,7 @@ solana-signature.workspace = true
 solana-sdk-ids.workspace = true
 prost_011.workspace = true
 solana-entry.workspace = true
-reqwest.workspace = true
+reqwest = { workspace = true, features = ["rustls-tls", "stream"] }
 tokio.workspace = true
 futures-util.workspace = true
 serde = { workspace = true, features = ["derive"] }
@@ -53,6 +53,7 @@ wincode.workspace = true
 solana-logger.workspace = true
 log.workspace = true
 rseek.workspace = true
+ripget.workspace = true
 crc.workspace = true
 serde_cbor.workspace = true
 fnv.workspace = true
@@ -64,6 +65,7 @@ xxhash-rust.workspace = true
 dashmap.workspace = true
 once_cell.workspace = true
 url.workspace = true
+libc.workspace = true
 
 aws-credential-types = { workspace = true, optional = true }
 aws-sdk-s3 = { workspace = true, optional = true }

jetstreamer-firehose/README.md

@@ -28,10 +28,12 @@ https://github.com/rpcpool/yellowstone-faithful/tree/main/geyser-plugin-runner
 - `JETSTREAMER_NETWORK_CAPACITY_MB` (default `1000`): assumed network throughput in megabytes
   per second when sizing the firehose thread pool. Increase or decrease to match your host's
   effective bandwidth.
-
 Notes:
 
 - `JETSTREAMER_HTTP_BASE_URL` and `JETSTREAMER_COMPACT_INDEX_BASE_URL` accept both full HTTP(S)
   URLs and `s3://bucket/...` URIs; the latter automatically activates the S3 transport layer.
 - Changing `JETSTREAMER_NETWORK` also alters the in-memory cache namespace, so you can switch
   networks without cross-contaminating cached offsets.
+- Sequential-mode ripget buffering is configured via the `buffer_window_bytes` parameter on
+  `firehose::firehose(...)`. If you run through the top-level `jetstreamer` binary crate, that
+  layer exposes `JETSTREAMER_BUFFER_WINDOW` and forwards it to firehose.