Represent enums from protobuf messages as StructArray (#11458)

### What

In robotics, enums are often use to represent state. Currently the
viewer can't really handle categorical values in plots.

To make it easier to create step functions, we pass the enum's `number`
along with the actual value.

Author: grtlr

crates/utils/re_mcap/src/layers/protobuf.rs

@@ -245,7 +245,18 @@ fn append_value(
             let value = enum_descriptor
                 .get_value(*x)
                 .ok_or_else(|| ProtobufError::UnknownEnumNumber(*x))?;
-            downcast_err::<StringBuilder>(builder, val)?.append_value(value.name());
+
+            let struct_builder = downcast_err::<StructBuilder>(builder, val)?;
+            let field_builders = struct_builder.field_builders_mut();
+
+            // First field is "name" (String)
+            downcast_err::<StringBuilder>(field_builders[0].as_mut(), val)?
+                .append_value(value.name());
+
+            // Second field is "value" (Int32)
+            downcast_err::<Int32Builder>(field_builders[1].as_mut(), val)?.append_value(*x);
+
+            struct_builder.append(true);
         }
     }
 
@@ -287,11 +298,18 @@ fn arrow_builder_from_field(descr: &FieldDescriptor) -> Box<dyn ArrayBuilder> {
             Box::new(struct_builder_from_message(&message_descriptor)) as Box<dyn ArrayBuilder>
         }
         Kind::Enum(_) => {
-            // TODO(grtlr): It would be great to improve our `enum` support. Using `Utf8`
-            // means a lot of excess memory / storage usage. Ideally we would use something
-            // like `StringDictionary`, but it's not clear right now how this works with
-            // `dyn ArrayBuilder` and sharing entries across lists.
-            Box::new(StringBuilder::new())
+            // Create a struct with "name" (String) and "value" (Int32) fields.
+            // We can't use `DictionaryArray` because `concat` does not re-key, and there
+            // could be protobuf schema evolution with different enum values across chunks.
+            let fields = Fields::from(vec![
+                Field::new("name", DataType::Utf8, false),
+                Field::new("value", DataType::Int32, false),
+            ]);
+            let field_builders: Vec<Box<dyn ArrayBuilder>> = vec![
+                Box::new(StringBuilder::new()),
+                Box::new(Int32Builder::new()),
+            ];
+            Box::new(StructBuilder::new(fields, field_builders))
         }
     };
 
@@ -303,7 +321,20 @@ fn arrow_builder_from_field(descr: &FieldDescriptor) -> Box<dyn ArrayBuilder> {
 }
 
 fn arrow_field_from(descr: &FieldDescriptor) -> Field {
-    Field::new(descr.name(), datatype_from(descr), true)
+    let mut field = Field::new(descr.name(), datatype_from(descr), true);
+
+    // Add extension metadata for enum types
+    if matches!(descr.kind(), Kind::Enum(_)) {
+        field = field.with_metadata(
+            std::iter::once((
+                "ARROW:extension:name".to_owned(),
+                "rerun.datatypes.ProtobufEnum".to_owned(),
+            ))
+            .collect(),
+        );
+    }
+
+    field
 }
 
 fn datatype_from(descr: &FieldDescriptor) -> DataType {
@@ -325,8 +356,13 @@ fn datatype_from(descr: &FieldDescriptor) -> DataType {
             DataType::Struct(fields)
         }
         Kind::Enum(_) => {
-            // TODO(grtlr): Explanation see above.
-            DataType::Utf8
+            // Struct with "name" (String) and "value" (Int32) fields.
+            // See comment in arrow_builder_from_field for why we use a struct.
+            let fields = Fields::from(vec![
+                Field::new("name", DataType::Utf8, false),
+                Field::new("value", DataType::Int32, false),
+            ]);
+            DataType::Struct(fields)
         }
     };
 

crates/utils/re_mcap/src/layers/snapshots/re_mcap__layers__protobuf__test__two_simple_rows.snap

@@ -1,6 +1,6 @@
 ---
 source: crates/utils/re_mcap/src/layers/protobuf.rs
-expression: "format!(\"{:240}\", ChunkRedacted(&chunks[0]))"
+expression: "format!(\"{:-240}\", &chunks[0])"
 ---
 ┌────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
 │ METADATA:                                                                                                                                                                                                          │
@@ -12,15 +12,15 @@ expression: "format!(\"{:240}\", ChunkRedacted(&chunks[0]))"
 │ ┌─────────────────────────────────────┬─────────────────────────────┬─────────────────────────────┬───────────────────────────────────┬────────────────────────────────────┬─────────────────────────────────────┐ │
 │ │ RowId                               ┆ log_time                    ┆ publish_time                ┆ com.example.Person:id             ┆ com.example.Person:name            ┆ com.example.Person:status           │ │
 │ │ ---                                 ┆ ---                         ┆ ---                         ┆ ---                               ┆ ---                                ┆ ---                                 │ │
-│ │ type: FixedSizeBinary[16]           ┆ type: nullable Duration(ns) ┆ type: nullable Duration(ns) ┆ type: nullable List[nullable i32] ┆ type: nullable List[nullable Utf8] ┆ type: nullable List[nullable Utf8]  │ │
-│ │ ARROW:extension:metadata:           ┆ index_name: log_time        ┆ index_name: publish_time    ┆ archetype: com.example.Person     ┆ archetype: com.example.Person      ┆ archetype: com.example.Person       │ │
-│ │ {"namespace":"row"}                 ┆ is_sorted: true             ┆ is_sorted: true             ┆ component: com.example.Person:id  ┆ component: com.example.Person:name ┆ component:                          │ │
-│ │ ARROW:extension:name: TUID          ┆ kind: index                 ┆ kind: index                 ┆ kind: data                        ┆ kind: data                         ┆ com.example.Person:status           │ │
-│ │ is_sorted: true                     ┆                             ┆                             ┆                                   ┆                                    ┆ kind: data                          │ │
-│ │ kind: control                       ┆                             ┆                             ┆                                   ┆                                    ┆                                     │ │
+│ │ type: FixedSizeBinary[16]           ┆ type: nullable Duration(ns) ┆ type: nullable Duration(ns) ┆ type: nullable List[nullable i32] ┆ type: nullable List[nullable Utf8] ┆ type: nullable List[nullable        │ │
+│ │ ARROW:extension:metadata:           ┆ index_name: log_time        ┆ index_name: publish_time    ┆ archetype: com.example.Person     ┆ archetype: com.example.Person      ┆ Struct[2]]                          │ │
+│ │ {"namespace":"row"}                 ┆ is_sorted: true             ┆ is_sorted: true             ┆ component: com.example.Person:id  ┆ component: com.example.Person:name ┆ archetype: com.example.Person       │ │
+│ │ ARROW:extension:name: TUID          ┆ kind: index                 ┆ kind: index                 ┆ kind: data                        ┆ kind: data                         ┆ component:                          │ │
+│ │ is_sorted: true                     ┆                             ┆                             ┆                                   ┆                                    ┆ com.example.Person:status           │ │
+│ │ kind: control                       ┆                             ┆                             ┆                                   ┆                                    ┆ kind: data                          │ │
 │ ╞═════════════════════════════════════╪═════════════════════════════╪═════════════════════════════╪═══════════════════════════════════╪════════════════════════════════════╪═════════════════════════════════════╡ │
-│ │ row_[**REDACTED**]                  ┆ PT0.000000042S              ┆ PT0.000000042S              ┆ [0]                               ┆ [Bob]                              ┆ [INACTIVE]                          │ │
+│ │ row_[**REDACTED**]                  ┆ PT0.000000042S              ┆ PT0.000000042S              ┆ [0]                               ┆ [Bob]                              ┆ [{name: INACTIVE, value: 2}]        │ │
 │ ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ │
-│ │ row_[**REDACTED**]                  ┆ PT0.000000043S              ┆ PT0.000000043S              ┆ [123]                             ┆ [Alice]                            ┆ [UNKNOWN]                           │ │
+│ │ row_[**REDACTED**]                  ┆ PT0.000000043S              ┆ PT0.000000043S              ┆ [123]                             ┆ [Alice]                            ┆ [{name: UNKNOWN, value: 0}]         │ │
 │ └─────────────────────────────────────┴─────────────────────────────┴─────────────────────────────┴───────────────────────────────────┴────────────────────────────────────┴─────────────────────────────────────┘ │
 └────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘