Use LargeBinary in MCAP loader

Author: emilk

crates/utils/re_mcap/src/layers/raw.rs

@@ -1,24 +1,24 @@
-use arrow::array::{ListBuilder, UInt8Builder};
+use arrow::array::LargeBinaryBuilder;
 use re_chunk::{ChunkId, external::arrow::array::FixedSizeListBuilder};
 use re_types::{
     Component as _, ComponentDescriptor, components, reflection::ComponentDescriptorExt as _,
 };
 
 use crate::{
     Error, LayerIdentifier, MessageLayer,
-    parsers::{MessageParser, ParserContext, util::blob_list_builder},
+    parsers::{MessageParser, ParserContext, util::fixed_size_list_builder},
 };
 
 struct RawMcapMessageParser {
-    data: FixedSizeListBuilder<ListBuilder<UInt8Builder>>,
+    data: FixedSizeListBuilder<LargeBinaryBuilder>,
 }
 
 impl RawMcapMessageParser {
     const ARCHETYPE_NAME: &str = "rerun.mcap.Message";
 
     fn new(num_rows: usize) -> Self {
         Self {
-            data: blob_list_builder(num_rows),
+            data: fixed_size_list_builder(1, num_rows),
         }
     }
 }
@@ -30,8 +30,7 @@ impl MessageParser for RawMcapMessageParser {
         msg: &::mcap::Message<'_>,
     ) -> anyhow::Result<()> {
         re_tracing::profile_function!();
-        self.data.values().values().append_slice(&msg.data);
-        self.data.values().append(true);
+        self.data.values().append_value(&msg.data);
         self.data.append(true);
         Ok(())
     }

crates/utils/re_mcap/src/parsers/mod.rs

@@ -7,32 +7,12 @@ pub use decode::{ChannelId, MessageParser, ParserContext};
 
 /// Defines utility functions shared across parsers.
 pub(crate) mod util {
-    use arrow::{
-        array::{FixedSizeListBuilder, ListBuilder, UInt8Builder},
-        datatypes::{DataType, Field},
-    };
-    use re_types::{Loggable as _, components};
-    use std::sync::Arc;
+    use arrow::array::{ArrayBuilder, FixedSizeListBuilder};
 
-    pub(crate) fn fixed_size_list_builder<T: arrow::array::ArrayBuilder + Default>(
+    pub(crate) fn fixed_size_list_builder<T: ArrayBuilder + Default>(
         value_length: i32,
         capacity: usize,
-    ) -> arrow::array::FixedSizeListBuilder<T> {
-        arrow::array::FixedSizeListBuilder::with_capacity(
-            Default::default(),
-            value_length,
-            capacity,
-        )
-    }
-
-    pub(crate) fn blob_list_builder(
-        capacity: usize,
-    ) -> FixedSizeListBuilder<ListBuilder<UInt8Builder>> {
-        let list_builder = ListBuilder::<UInt8Builder>::default()
-            .with_field(Arc::new(Field::new_list_field(DataType::UInt8, false)));
-
-        FixedSizeListBuilder::with_capacity(list_builder, 1, capacity).with_field(Arc::new(
-            Field::new_list_field(components::Blob::arrow_datatype(), false),
-        ))
+    ) -> FixedSizeListBuilder<T> {
+        FixedSizeListBuilder::with_capacity(Default::default(), value_length, capacity)
     }
 }

crates/utils/re_mcap/src/parsers/ros2msg/sensor_msgs/point_cloud_2.rs

@@ -3,8 +3,8 @@ use std::io::Cursor;
 use super::super::definitions::sensor_msgs::{self, PointField, PointFieldDatatype};
 use arrow::{
     array::{
-        BooleanBuilder, FixedSizeListBuilder, ListBuilder, StringBuilder, StructBuilder,
-        UInt8Builder, UInt32Builder,
+        BooleanBuilder, FixedSizeListBuilder, LargeBinaryBuilder, ListBuilder, StringBuilder,
+        StructBuilder, UInt8Builder, UInt32Builder,
     },
     datatypes::{DataType, Field, Fields},
 };
@@ -22,7 +22,7 @@ use crate::{
     parsers::{
         cdr,
         decode::{MessageParser, ParserContext},
-        util::{blob_list_builder, fixed_size_list_builder},
+        util::fixed_size_list_builder,
     },
 };
 
@@ -35,7 +35,7 @@ pub struct PointCloud2MessageParser {
     is_bigendian: FixedSizeListBuilder<BooleanBuilder>,
     point_step: FixedSizeListBuilder<UInt32Builder>,
     row_step: FixedSizeListBuilder<UInt32Builder>,
-    data: FixedSizeListBuilder<ListBuilder<UInt8Builder>>,
+    data: FixedSizeListBuilder<LargeBinaryBuilder>,
     is_dense: FixedSizeListBuilder<BooleanBuilder>,
 
     // We lazily create this, only if we can interpret the point cloud semantically.
@@ -75,7 +75,7 @@ impl PointCloud2MessageParser {
             is_bigendian: fixed_size_list_builder(1, num_rows),
             point_step: fixed_size_list_builder(1, num_rows),
             row_step: fixed_size_list_builder(1, num_rows),
-            data: blob_list_builder(num_rows),
+            data: fixed_size_list_builder(1, num_rows),
             is_dense: fixed_size_list_builder(1, num_rows),
 
             points_3ds: None,
@@ -257,7 +257,7 @@ impl MessageParser for PointCloud2MessageParser {
         point_step.values().append_slice(&[point_cloud.point_step]);
         row_step.values().append_slice(&[point_cloud.row_step]);
 
-        data.values().values().append_slice(&point_cloud.data);
+        data.values().append_value(&point_cloud.data);
         is_dense.values().append_slice(&[point_cloud.is_dense]);
 
         height.append(true);
@@ -267,7 +267,6 @@ impl MessageParser for PointCloud2MessageParser {
         row_step.append(true);
         is_dense.append(true);
 
-        data.values().append(true);
         data.append(true);
 
         Ok(())