Skip to content

Introduce Lens as a concept for semantic mapping of arbitrary data #11394

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 12 commits into
base: main
Choose a base branch
from
+1,032 −89
Draft

Introduce Lens as a concept for semantic mapping of arbitrary data #11394

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
331502b
Initial commit for `TransformSink`
grtlr Sep 25, 2025
d0e0ffe
Minimal example
grtlr Sep 26, 2025
5aa672a
rename
grtlr Oct 1, 2025
fbe7a4d
Implement initial version of column-wise transformations
grtlr Sep 29, 2025
ef2ab01
Add nullability test case
grtlr Sep 30, 2025
e5a2222
better testing
grtlr Sep 30, 2025
7de07df
More tests and examples
grtlr Sep 30, 2025
39acd87
Add `-` formatting option for redacted values
grtlr Sep 30, 2025
f5bcb12
cleanup
grtlr Oct 1, 2025
c773761
add docs
grtlr Oct 1, 2025
b19428f
remove unwraps
grtlr Oct 1, 2025
b6bd58b
add test case for static chunk
grtlr Oct 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions Cargo.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1753,7 +1753,7 @@ checksum = "fe6d2e5af09e8c8ad56c969f2157a3d4238cebc7c55f0a517728c38f7b200f81"
dependencies = [
"serde",
"termcolor",
"unicode-width 0.1.14",
"unicode-width 0.2.1",
]

[[package]]
Expand Down Expand Up @@ -3026,7 +3026,7 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "330c60081dcc4c72131f8eb70510f1ac07223e5d4163db481a04a0befcffa412"
dependencies = [
"libloading 0.7.4",
"libloading 0.8.8",
]

[[package]]
Expand Down Expand Up @@ -4959,6 +4959,17 @@ version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67"

[[package]]
name = "lenses"
version = "0.26.0-alpha.1+dev"
dependencies = [
"anyhow",
"arrow",
"clap",
"insta",
"rerun",
]

[[package]]
name = "lexical-core"
version = "1.0.5"
Expand Down
2 changes: 1 addition & 1 deletion crates/store/re_chunk/src/chunk.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1247,7 +1247,7 @@ impl std::fmt::Display for Chunk {
re_log::error_once!("couldn't display Chunk: {err}");
std::fmt::Error
})?;
re_format_arrow::format_record_batch_with_width(&batch, f.width()).fmt(f)
re_format_arrow::format_record_batch_with_width(&batch, f.width(), f.sign_minus()).fmt(f)
}
}

Expand Down
31 changes: 1 addition & 30 deletions crates/store/re_chunk/tests/formatting.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,40 +55,11 @@ fn format_chunk() -> anyhow::Result<()> {
Ok(())
}

/// Wrapper struct to help with `insta` snapshot tests.
struct ChunkRedacted(Chunk);

impl std::fmt::Display for ChunkRedacted {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let batch = self.0.to_record_batch().map_err(|err| {
re_log::error_once!("couldn't display Chunk: {err}");
std::fmt::Error
})?;
re_format_arrow::format_record_batch_opts(
&batch,
&re_format_arrow::RecordBatchFormatOpts {
transposed: false,
width: f.width(),
include_metadata: true,
include_column_metadata: true,
trim_field_names: false,
trim_metadata_keys: false,
trim_metadata_values: false,
redact_non_deterministic: true,
},
)
.fmt(f)
}
}

#[test]
fn format_chunk_redacted() -> anyhow::Result<()> {
let chunk = create_chunk()?;

insta::assert_snapshot!(
"format_chunk_redacted",
format!("{:240}", ChunkRedacted(chunk))
);
insta::assert_snapshot!("format_chunk_redacted", format!("{:-240}", chunk));

Ok(())
}
42 changes: 21 additions & 21 deletions crates/store/re_chunk/tests/snapshots/formatting__format_chunk_redacted.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,24 +1,24 @@
---
source: crates/store/re_chunk/tests/formatting.rs
expression: "format!(\"{:240}\", ChunkRedacted(chunk))"
expression: "format!(\"{:-240}\", chunk)"
---
┌──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
│ METADATA: │
│ * rerun:entity_path: /this/that │
│ * rerun:heap_size_bytes: [**REDACTED**] │
│ * rerun:id: [**REDACTED**] │
│ * sorbet:version: [**REDACTED**] │
├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
│ ┌──────────────────────────────────────────────┬────────────────────────────┬───────────────────────────────┬───────────────────────────────────────┬──────────────────────────────────────────┐ │
│ │ rerun.controls.RowId ┆ frame_nr ┆ log_time ┆ my_index ┆ example.MyPoints:colors │ │
│ │ --- ┆ --- ┆ --- ┆ --- ┆ --- │ │
│ │ type: FixedSizeBinary[16] ┆ type: i64 ┆ type: Timestamp(ns) ┆ type: List[nullable u64] ┆ type: List[nullable u32] │ │
│ │ ARROW:extension:metadata: ┆ rerun:index_name: frame_nr ┆ rerun:index_name: log_time ┆ rerun:component: my_index ┆ rerun:archetype: example.MyPoints │ │
│ │ {"namespace":"row"} ┆ rerun:is_sorted: true ┆ rerun:is_sorted: true ┆ rerun:component_type: example.MyIndex ┆ rerun:component: example.MyPoints:colors │ │
│ │ ARROW:extension:name: rerun.datatypes.TUID ┆ rerun:kind: index ┆ rerun:kind: index ┆ rerun:kind: data ┆ rerun:component_type: example.MyColor │ │
│ │ rerun:is_sorted: true ┆ ┆ ┆ ┆ rerun:kind: data │ │
│ │ rerun:kind: control ┆ ┆ ┆ ┆ │ │
│ ╞══════════════════════════════════════════════╪════════════════════════════╪═══════════════════════════════╪═══════════════════════════════════════╪══════════════════════════════════════════╡ │
│ │ row_[**REDACTED**] ┆ 1 ┆ 2025-01-10T18:43:42.123456789 ┆ [0, 1, 2] ┆ [0, 1, 2] │ │
│ └──────────────────────────────────────────────┴────────────────────────────┴───────────────────────────────┴───────────────────────────────────────┴──────────────────────────────────────────┘ │
└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
┌────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
│ METADATA: │
│ * entity_path: /this/that │
│ * heap_size_bytes: [**REDACTED**] │
│ * id: [**REDACTED**] │
│ * version: [**REDACTED**] │
├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
│ ┌──────────────────────────────────────────────┬──────────────────────┬───────────────────────────────┬─────────────────────────────────┬────────────────────────────────────┐ │
│ │ RowId ┆ frame_nr ┆ log_time ┆ my_index ┆ example.MyPoints:colors │ │
│ │ --- ┆ --- ┆ --- ┆ --- ┆ --- │ │
│ │ type: FixedSizeBinary[16] ┆ type: i64 ┆ type: Timestamp(ns) ┆ type: List[nullable u64] ┆ type: List[nullable u32] │ │
│ │ ARROW:extension:metadata: ┆ index_name: frame_nr ┆ index_name: log_time ┆ component: my_index ┆ archetype: example.MyPoints │ │
│ │ {"namespace":"row"} ┆ is_sorted: true ┆ is_sorted: true ┆ component_type: example.MyIndex ┆ component: example.MyPoints:colors │ │
│ │ ARROW:extension:name: TUID ┆ kind: index ┆ kind: index ┆ kind: data ┆ component_type: example.MyColor │ │
│ │ is_sorted: true ┆ ┆ ┆ ┆ kind: data │ │
│ │ kind: control ┆ ┆ ┆ ┆ │ │
│ ╞══════════════════════════════════════════════╪══════════════════════╪═══════════════════════════════╪═════════════════════════════════╪════════════════════════════════════╡ │
│ │ row_[**REDACTED**] ┆ 1 ┆ 2025-01-10T18:43:42.123456789 ┆ [0, 1, 2] ┆ [0, 1, 2] │ │
│ └──────────────────────────────────────────────┴──────────────────────┴───────────────────────────────┴─────────────────────────────────┴────────────────────────────────────┘ │
└────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
3 changes: 2 additions & 1 deletion crates/store/re_dataframe/src/query.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1368,7 +1368,8 @@ mod tests {
#[inline]
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let width = 200;
re_format_arrow::format_record_batch_with_width(&self.0, Some(width)).fmt(f)
re_format_arrow::format_record_batch_with_width(&self.0, Some(width), f.sign_minus())
.fmt(f)
}
}

Expand Down
5 changes: 3 additions & 2 deletions crates/store/re_format_arrow/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ impl Default for RecordBatchFormatOpts {

/// Nicely format this record batch in a way that fits the terminal.
pub fn format_record_batch(batch: &arrow::array::RecordBatch) -> Table {
format_record_batch_with_width(batch, None)
format_record_batch_with_width(batch, None, false)
}

/// Nicely format this record batch using the specified options.
Expand All @@ -223,6 +223,7 @@ pub fn format_record_batch_opts(
pub fn format_record_batch_with_width(
batch: &arrow::array::RecordBatch,
width: Option<usize>,
redact_non_deterministic: bool,
) -> Table {
format_dataframe_with_metadata(
&batch.schema_ref().metadata.clone().into_iter().collect(), // HashMap -> BTreeMap
Expand All @@ -236,7 +237,7 @@ pub fn format_record_batch_with_width(
trim_field_names: true,
trim_metadata_keys: true,
trim_metadata_values: true,
redact_non_deterministic: false,
redact_non_deterministic,
},
)
}
Expand Down
2 changes: 1 addition & 1 deletion crates/store/re_sorbet/src/chunk_batch.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ impl ChunkBatch {
impl std::fmt::Display for ChunkBatch {
#[inline]
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
re_format_arrow::format_record_batch_with_width(self, f.width()).fmt(f)
re_format_arrow::format_record_batch_with_width(self, f.width(), f.sign_minus()).fmt(f)
}
}

Expand Down
2 changes: 1 addition & 1 deletion crates/store/re_sorbet/src/sorbet_batch.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ impl SorbetBatch {
impl std::fmt::Display for SorbetBatch {
#[inline]
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
re_format_arrow::format_record_batch_with_width(self, f.width()).fmt(f)
re_format_arrow::format_record_batch_with_width(self, f.width(), f.sign_minus()).fmt(f)
}
}

Expand Down
Loading
Loading