RRD footers 3: encoding/decoding manifests

Cargo.lock

@@ -9071,19 +9071,25 @@ dependencies = [
  "bytes",
  "criterion",
  "ehttp",
+ "insta",
+ "itertools 0.14.0",
  "js-sys",
  "lz4_flex 0.12.0",
  "mimalloc",
  "parking_lot",
+ "re_arrow_util",
  "re_build_info",
  "re_chunk",
  "re_log",
  "re_log_types",
  "re_protos",
  "re_smart_channel",
  "re_sorbet",
+ "re_span",
  "re_tracing",
  "re_types",
+ "re_types_core",
+ "sha2",
  "similar-asserts",
  "thiserror 2.0.17",
  "tokio",
@@ -9093,6 +9099,7 @@ dependencies = [
  "wasm-bindgen-futures",
  "web-sys",
  "web-time",
+ "xxhash-rust",
 ]
 
 [[package]]
@@ -9617,7 +9624,6 @@ dependencies = [
  "re_tuid",
  "re_types_core",
  "serde",
- "sha2",
  "tempfile",
  "thiserror 2.0.17",
  "tokio",

Cargo.toml

@@ -429,6 +429,7 @@ wgpu = { version = "27.0.1", default-features = false, features = [
   "fragile-send-sync-non-atomic-wasm",
 ] }
 xshell = "0.2.7"
+xxhash-rust = { version = "0.8", features = ["xxh32"] }
 
 # ---------------------------------------------------------------------------------
 [profile]

crates/store/re_log_encoding/Cargo.toml

@@ -42,21 +42,27 @@ stream_from_http = [
 [dependencies]
 
 # Rerun:
+re_arrow_util.workspace = true
 re_build_info.workspace = true
 re_chunk.workspace = true
 re_log_types.workspace = true
 re_log.workspace = true
 re_protos.workspace = true
 re_smart_channel.workspace = true
 re_sorbet.workspace = true
+re_span.workspace = true
 re_tracing.workspace = true
+re_types_core.workspace = true
 
 # External:
 arrow = { workspace = true, features = ["ipc"] }
+itertools.workspace = true
 lz4_flex = { workspace = true }
 parking_lot.workspace = true
+sha2.workspace = true
 thiserror.workspace = true
 tracing.workspace = true
+xxhash-rust.workspace = true
 
 # Optional external dependencies:
 bytes = { workspace = true, optional = true }
@@ -76,6 +82,7 @@ web-sys = { workspace = true, optional = true, features = ["Window", "MessageEve
 re_types.workspace = true
 
 criterion.workspace = true
+insta.workspace = true
 mimalloc.workspace = true
 similar-asserts.workspace = true
 

crates/store/re_log_encoding/src/rrd/decoder/iterator.rs

@@ -1,3 +1,4 @@
+use crate::RrdManifest;
 use crate::rrd::decoder::state_machine::DecoderState;
 use crate::rrd::{DecodeError, Decoder, DecoderEntrypoint};
 
@@ -109,6 +110,18 @@ impl<T, R: std::io::BufRead> DecoderIterator<T, R> {
     }
 }
 
+impl<T: DecoderEntrypoint, R: std::io::BufRead> DecoderIterator<T, R> {
+    /// Returns all the RRD manifests accumulated _so far_.
+    ///
+    /// RRD manifests are parsed from footers, of which there might be more than one e.g. in the
+    /// case of concatenated streams.
+    ///
+    /// This is not cheap: it automatically performs the transport to app level conversion.
+    pub fn rrd_manifests(&self) -> Result<Vec<RrdManifest>, DecodeError> {
+        self.decoder.rrd_manifests()
+    }
+}
+
 impl<T: DecoderEntrypoint, R: std::io::BufRead> std::iter::Iterator for DecoderIterator<T, R> {
     type Item = Result<T, DecodeError>;
 
@@ -139,7 +152,7 @@ impl<T: DecoderEntrypoint, R: std::io::BufRead> std::iter::Iterator for DecoderI
 
                         // …and the underlying decoder already considers that it's done (i.e. it's
                         // waiting for a whole new stream to begin): time to stop.
-                        Ok(None) if self.decoder.state == DecoderState::StreamHeader => {
+                        Ok(None) if self.decoder.state == DecoderState::WaitingForStreamHeader => {
                             return None;
                         }