Add - formatting option for redacted values (#11383)

grtlr · web-flow · commit f42d29230c28 · 2025-10-02T14:09:41.000+02:00
### Related * Follow-up of #11278. ### What This PR allows you to use the minus sign (`-`) formatting option in most `RecordBatch`-derived structs to get a string that has non-deterministic values redacted. Usage: ```rs let chunk = Chunk::new(/* ... */); println!("{chunk:-}"); ``` Using `#` instead of `-` would be an alternative, but I think it's better to reserve this for a more meaningful representation.
diff --git a/CODE_STYLE.md b/CODE_STYLE.md
@@ -131,6 +131,12 @@ Use debug-formatting (`{:?}`) when logging strings in logs and error messages. T
 
 Use `re_error::format(err)` when displaying an error.
 
+We make extensive use of snapshot testing. To work around non-deterministic values, such as TUIDs (time-prefixed unique IDs), many types (should) offer `std::fmt::Display` implementations with redactions that can be access via an overloaded `-` formatting option:
+
+```rs
+println!("{:-}, value"); // The `-` option stands for redaction.
+```
+
 
 ## C++
 We use `clang-format` to enforce most style choices (see [`.clang-format`](.clang-format)).
diff --git a/Cargo.lock b/Cargo.lock
@@ -8984,7 +8984,6 @@ dependencies = [
  "mcap",
  "prost-reflect",
  "re_chunk",
- "re_format_arrow",
  "re_log",
  "re_log_types",
  "re_tracing",
diff --git a/crates/store/re_chunk/src/chunk.rs b/crates/store/re_chunk/src/chunk.rs
@@ -1247,7 +1247,7 @@ impl std::fmt::Display for Chunk {
             re_log::error_once!("couldn't display Chunk: {err}");
             std::fmt::Error
         })?;
-        re_format_arrow::format_record_batch_with_width(&batch, f.width()).fmt(f)
+        re_format_arrow::format_record_batch_with_width(&batch, f.width(), f.sign_minus()).fmt(f)
     }
 }
 
diff --git a/crates/store/re_chunk/tests/formatting.rs b/crates/store/re_chunk/tests/formatting.rs
@@ -55,40 +55,11 @@ fn format_chunk() -> anyhow::Result<()> {
     Ok(())
 }
 
-/// Wrapper struct to help with `insta` snapshot tests.
-struct ChunkRedacted(Chunk);
-
-impl std::fmt::Display for ChunkRedacted {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let batch = self.0.to_record_batch().map_err(|err| {
-            re_log::error_once!("couldn't display Chunk: {err}");
-            std::fmt::Error
-        })?;
-        re_format_arrow::format_record_batch_opts(
-            &batch,
-            &re_format_arrow::RecordBatchFormatOpts {
-                transposed: false,
-                width: f.width(),
-                include_metadata: true,
-                include_column_metadata: true,
-                trim_field_names: false,
-                trim_metadata_keys: false,
-                trim_metadata_values: false,
-                redact_non_deterministic: true,
-            },
-        )
-        .fmt(f)
-    }
-}
-
 #[test]
 fn format_chunk_redacted() -> anyhow::Result<()> {
     let chunk = create_chunk()?;
 
-    insta::assert_snapshot!(
-        "format_chunk_redacted",
-        format!("{:240}", ChunkRedacted(chunk))
-    );
+    insta::assert_snapshot!("format_chunk_redacted", format!("{:-240}", chunk));
 
     Ok(())
 }
diff --git a/crates/store/re_chunk/tests/snapshots/formatting__format_chunk_redacted.snap b/crates/store/re_chunk/tests/snapshots/formatting__format_chunk_redacted.snap
@@ -1,24 +1,24 @@
 ---
 source: crates/store/re_chunk/tests/formatting.rs
-expression: "format!(\"{:240}\", ChunkRedacted(chunk))"
+expression: "format!(\"{:-240}\", chunk)"
 ---
-┌──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
-│ METADATA:                                                                                                                                                                                        │
-│ * rerun:entity_path: /this/that                                                                                                                                                                  │
-│ * rerun:heap_size_bytes: [**REDACTED**]                                                                                                                                                          │
-│ * rerun:id: [**REDACTED**]                                                                                                                                                                       │
-│ * sorbet:version: [**REDACTED**]                                                                                                                                                                 │
-├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
-│ ┌──────────────────────────────────────────────┬────────────────────────────┬───────────────────────────────┬───────────────────────────────────────┬──────────────────────────────────────────┐ │
-│ │ rerun.controls.RowId                         ┆ frame_nr                   ┆ log_time                      ┆ my_index                              ┆ example.MyPoints:colors                  │ │
-│ │ ---                                          ┆ ---                        ┆ ---                           ┆ ---                                   ┆ ---                                      │ │
-│ │ type: FixedSizeBinary[16]                    ┆ type: nullable i64         ┆ type: nullable Timestamp(ns)  ┆ type: nullable List[nullable u64]     ┆ type: nullable List[nullable u32]        │ │
-│ │ ARROW:extension:metadata:                    ┆ rerun:index_name: frame_nr ┆ rerun:index_name: log_time    ┆ rerun:component: my_index             ┆ rerun:archetype: example.MyPoints        │ │
-│ │ {"namespace":"row"}                          ┆ rerun:is_sorted: true      ┆ rerun:is_sorted: true         ┆ rerun:component_type: example.MyIndex ┆ rerun:component: example.MyPoints:colors │ │
-│ │ ARROW:extension:name: rerun.datatypes.TUID   ┆ rerun:kind: index          ┆ rerun:kind: index             ┆ rerun:kind: data                      ┆ rerun:component_type: example.MyColor    │ │
-│ │ rerun:is_sorted: true                        ┆                            ┆                               ┆                                       ┆ rerun:kind: data                         │ │
-│ │ rerun:kind: control                          ┆                            ┆                               ┆                                       ┆                                          │ │
-│ ╞══════════════════════════════════════════════╪════════════════════════════╪═══════════════════════════════╪═══════════════════════════════════════╪══════════════════════════════════════════╡ │
-│ │ row_[**REDACTED**]                           ┆ 1                          ┆ 2025-01-10T18:43:42.123456789 ┆ [0, 1, 2]                             ┆ [0, 1, 2]                                │ │
-│ └──────────────────────────────────────────────┴────────────────────────────┴───────────────────────────────┴───────────────────────────────────────┴──────────────────────────────────────────┘ │
-└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
+┌──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
+│ METADATA:                                                                                                                                                                        │
+│ * entity_path: /this/that                                                                                                                                                        │
+│ * heap_size_bytes: [**REDACTED**]                                                                                                                                                │
+│ * id: [**REDACTED**]                                                                                                                                                             │
+│ * version: [**REDACTED**]                                                                                                                                                        │
+├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
+│ ┌──────────────────────────────────────────────┬──────────────────────┬───────────────────────────────┬───────────────────────────────────┬────────────────────────────────────┐ │
+│ │ RowId                                        ┆ frame_nr             ┆ log_time                      ┆ my_index                          ┆ example.MyPoints:colors            │ │
+│ │ ---                                          ┆ ---                  ┆ ---                           ┆ ---                               ┆ ---                                │ │
+│ │ type: FixedSizeBinary[16]                    ┆ type: nullable i64   ┆ type: nullable Timestamp(ns)  ┆ type: nullable List[nullable u64] ┆ type: nullable List[nullable u32]  │ │
+│ │ ARROW:extension:metadata:                    ┆ index_name: frame_nr ┆ index_name: log_time          ┆ component: my_index               ┆ archetype: example.MyPoints        │ │
+│ │ {"namespace":"row"}                          ┆ is_sorted: true      ┆ is_sorted: true               ┆ component_type: example.MyIndex   ┆ component: example.MyPoints:colors │ │
+│ │ ARROW:extension:name: TUID                   ┆ kind: index          ┆ kind: index                   ┆ kind: data                        ┆ component_type: example.MyColor    │ │
+│ │ is_sorted: true                              ┆                      ┆                               ┆                                   ┆ kind: data                         │ │
+│ │ kind: control                                ┆                      ┆                               ┆                                   ┆                                    │ │
+│ ╞══════════════════════════════════════════════╪══════════════════════╪═══════════════════════════════╪═══════════════════════════════════╪════════════════════════════════════╡ │
+│ │ row_[**REDACTED**]                           ┆ 1                    ┆ 2025-01-10T18:43:42.123456789 ┆ [0, 1, 2]                         ┆ [0, 1, 2]                          │ │
+│ └──────────────────────────────────────────────┴──────────────────────┴───────────────────────────────┴───────────────────────────────────┴────────────────────────────────────┘ │
+└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
diff --git a/crates/store/re_dataframe/src/query.rs b/crates/store/re_dataframe/src/query.rs
@@ -1360,7 +1360,8 @@ mod tests {
         #[inline]
         fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
             let width = 200;
-            re_format_arrow::format_record_batch_with_width(&self.0, Some(width)).fmt(f)
+            re_format_arrow::format_record_batch_with_width(&self.0, Some(width), f.sign_minus())
+                .fmt(f)
         }
     }
 
diff --git a/crates/store/re_format_arrow/src/lib.rs b/crates/store/re_format_arrow/src/lib.rs
@@ -200,7 +200,7 @@ impl Default for RecordBatchFormatOpts {
 
 /// Nicely format this record batch in a way that fits the terminal.
 pub fn format_record_batch(batch: &arrow::array::RecordBatch) -> Table {
-    format_record_batch_with_width(batch, None)
+    format_record_batch_with_width(batch, None, false)
 }
 
 /// Nicely format this record batch using the specified options.
@@ -223,6 +223,7 @@ pub fn format_record_batch_opts(
 pub fn format_record_batch_with_width(
     batch: &arrow::array::RecordBatch,
     width: Option<usize>,
+    redact_non_deterministic: bool,
 ) -> Table {
     format_dataframe_with_metadata(
         &batch.schema_ref().metadata.clone().into_iter().collect(), // HashMap -> BTreeMap
@@ -236,7 +237,7 @@ pub fn format_record_batch_with_width(
             trim_field_names: true,
             trim_metadata_keys: true,
             trim_metadata_values: true,
-            redact_non_deterministic: false,
+            redact_non_deterministic,
         },
     )
 }
diff --git a/crates/store/re_sorbet/src/chunk_batch.rs b/crates/store/re_sorbet/src/chunk_batch.rs
@@ -114,7 +114,7 @@ impl ChunkBatch {
 impl std::fmt::Display for ChunkBatch {
     #[inline]
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        re_format_arrow::format_record_batch_with_width(self, f.width()).fmt(f)
+        re_format_arrow::format_record_batch_with_width(self, f.width(), f.sign_minus()).fmt(f)
     }
 }
 
diff --git a/crates/store/re_sorbet/src/sorbet_batch.rs b/crates/store/re_sorbet/src/sorbet_batch.rs
@@ -129,7 +129,7 @@ impl SorbetBatch {
 impl std::fmt::Display for SorbetBatch {
     #[inline]
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        re_format_arrow::format_record_batch_with_width(self, f.width()).fmt(f)
+        re_format_arrow::format_record_batch_with_width(self, f.width(), f.sign_minus()).fmt(f)
     }
 }
 
diff --git a/crates/utils/re_mcap/Cargo.toml b/crates/utils/re_mcap/Cargo.toml
@@ -33,7 +33,5 @@ serde_bytes.workspace = true
 thiserror.workspace = true
 
 [dev-dependencies]
-re_format_arrow.workspace = true
-
 insta = { workspace = true, features = ["filters", "redactions"] }
 prost-reflect = { workspace = true, features = ["text-format"] }
diff --git a/crates/utils/re_mcap/src/layers/protobuf.rs b/crates/utils/re_mcap/src/layers/protobuf.rs
@@ -547,32 +547,6 @@ mod test {
         chunks
     }
 
-    /// Wrapper to help with creating nicely formatted chunks to use with `insta`.
-    struct ChunkRedacted<'a>(&'a Chunk);
-
-    impl std::fmt::Display for ChunkRedacted<'_> {
-        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-            let batch = self.0.to_record_batch().map_err(|err| {
-                re_log::error_once!("couldn't display Chunk: {err}");
-                std::fmt::Error
-            })?;
-            re_format_arrow::format_record_batch_opts(
-                &batch,
-                &re_format_arrow::RecordBatchFormatOpts {
-                    transposed: false,
-                    width: f.width(),
-                    include_metadata: true,
-                    include_column_metadata: true,
-                    trim_field_names: true,
-                    trim_metadata_keys: true,
-                    trim_metadata_values: true,
-                    redact_non_deterministic: true,
-                },
-            )
-            .fmt(f)
-        }
-    }
-
     #[test]
     fn two_simple_rows() {
         // Writing to the MCAP buffer.
@@ -615,9 +589,6 @@ mod test {
         let chunks = run_layer(&summary, buffer.as_slice());
         assert_eq!(chunks.len(), 1);
 
-        insta::assert_snapshot!(
-            "two_simple_rows",
-            format!("{:240}", ChunkRedacted(&chunks[0]))
-        );
+        insta::assert_snapshot!("two_simple_rows", format!("{:-240}", &chunks[0]));
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -1247,7 +1247,7 @@ impl std::fmt::Display for Chunk {`
`1247`	`1247`	`re_log::error_once!("couldn't display Chunk: {err}");`
`1248`	`1248`	`std::fmt::Error`
`1249`	`1249`	`})?;`
`1250`		`- re_format_arrow::format_record_batch_with_width(&batch, f.width()).fmt(f)`
	`1250`	`+ re_format_arrow::format_record_batch_with_width(&batch, f.width(), f.sign_minus()).fmt(f)`
`1251`	`1251`	`}`
`1252`	`1252`	`}`
`1253`	`1253`
Original file line number	Diff line number	Diff line change
`@@ -1360,7 +1360,8 @@ mod tests {`
`1360`	`1360`	`#[inline]`
`1361`	`1361`	`fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {`
`1362`	`1362`	`let width = 200;`
`1363`		`- re_format_arrow::format_record_batch_with_width(&self.0, Some(width)).fmt(f)`
	`1363`	`+ re_format_arrow::format_record_batch_with_width(&self.0, Some(width), f.sign_minus())`
	`1364`	`+ .fmt(f)`
`1364`	`1365`	`}`
`1365`	`1366`	`}`
`1366`	`1367`
Original file line number	Diff line number	Diff line change
`@@ -114,7 +114,7 @@ impl ChunkBatch {`
`114`	`114`	`impl std::fmt::Display for ChunkBatch {`
`115`	`115`	`#[inline]`
`116`	`116`	`fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {`
`117`		`- re_format_arrow::format_record_batch_with_width(self, f.width()).fmt(f)`
	`117`	`+ re_format_arrow::format_record_batch_with_width(self, f.width(), f.sign_minus()).fmt(f)`
`118`	`118`	`}`
`119`	`119`	`}`
`120`	`120`
Original file line number	Diff line number	Diff line change
`@@ -129,7 +129,7 @@ impl SorbetBatch {`
`129`	`129`	`impl std::fmt::Display for SorbetBatch {`
`130`	`130`	`#[inline]`
`131`	`131`	`fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {`
`132`		`- re_format_arrow::format_record_batch_with_width(self, f.width()).fmt(f)`
	`132`	`+ re_format_arrow::format_record_batch_with_width(self, f.width(), f.sign_minus()).fmt(f)`
`133`	`133`	`}`
`134`	`134`	`}`
`135`	`135`