Treat BinaryArray and LargeBinaryArray the same in chunk comparison

Author: emilk

crates/store/re_chunk/src/chunk.rs

@@ -12,7 +12,7 @@ use arrow::{
 use itertools::{Either, Itertools as _, izip};
 use nohash_hasher::IntMap;
 
-use re_arrow_util::ArrowArrayDowncastRef as _;
+use re_arrow_util::{ArrowArrayDowncastRef as _, widen_binary_arrays};
 use re_byte_size::SizeBytes as _;
 use re_log_types::{
     AbsoluteTimeRange, EntityPath, NonMinI64, TimeInt, TimeType, Timeline, TimelineName,
@@ -101,6 +101,8 @@ impl ChunkComponents {
             let Some(right_array) = right.get(descr) else {
                 anyhow::bail!("rhs is missing {descr:?}");
             };
+            let left_array = widen_binary_arrays(left_array);
+            let right_array = widen_binary_arrays(right_array);
             re_arrow_util::ensure_similar(&left_array.to_data(), &right_array.to_data())
                 .with_context(|| format!("Component {descr:?}"))?;
         }

crates/utils/re_arrow_util/src/lib.rs

@@ -9,3 +9,75 @@ pub use self::arrays::*;
 pub use self::batches::*;
 pub use self::compare::*;
 pub use self::format_data_type::*;
+
+// ----------------------------------------------------------------
+
+use std::sync::Arc;
+
+use arrow::{
+    array::{Array as _, AsArray as _, ListArray},
+    datatypes::{DataType, Field},
+};
+
+/// Convert any `BinaryArray` to `LargeBinaryArray`, because we treat them logivally the same
+pub fn widen_binary_arrays(list_array: &ListArray) -> ListArray {
+    let list_data_type = list_array.data_type();
+    if let DataType::List(field) = list_data_type
+        && field.data_type() == &DataType::Binary
+    {
+        re_tracing::profile_function!();
+        let large_binary_field = Field::new("item", DataType::LargeBinary, true);
+        let target_type = DataType::List(Arc::new(large_binary_field));
+
+        #[expect(clippy::unwrap_used)]
+        arrow::compute::kernels::cast::cast(list_array, &target_type)
+            .unwrap()
+            .as_list()
+            .clone()
+    } else {
+        list_array.clone()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use arrow::array::{BinaryBuilder, ListBuilder};
+
+    #[test]
+    fn test_widen_list_binary() {
+        // Create test data
+        let mut list_builder = ListBuilder::new(BinaryBuilder::new());
+
+        // First list: [b"hello", b"world"]
+        list_builder.values().append_value(b"hello");
+        list_builder.values().append_value(b"world");
+        list_builder.append(true);
+
+        // Second list: [b"rust", b"arrow"]
+        list_builder.values().append_value(b"rust");
+        list_builder.values().append_value(b"arrow");
+        list_builder.append(true);
+
+        // Third list: null
+        list_builder.append_null();
+
+        let original_list = list_builder.finish();
+
+        // Widen to LargeBinaryArray
+        let widened_list = widen_binary_arrays(&original_list);
+
+        // Verify the result
+        assert_eq!(widened_list.len(), 3);
+        assert!(!widened_list.is_null(0));
+        assert!(!widened_list.is_null(1));
+        assert!(widened_list.is_null(2));
+
+        // Check data type
+        if let DataType::List(field) = widened_list.data_type() {
+            assert_eq!(field.data_type(), &DataType::LargeBinary);
+        } else {
+            panic!("Expected List data type");
+        }
+    }
+}