Make camera tracker use multiple channels

Author: nvddr

examples/oxr/python/test_oak_camera.py

@@ -139,7 +139,7 @@ def run_test(duration: float = 10.0, metadata_track: bool = True):
                 with deviceio.DeviceIOSession.run(trackers, handles) as session:
                     print("  ✓ DeviceIO session initialized")
 
-                    # Create MCAP recorder with single CameraMetadataOak channel
+                    # Create MCAP recorder with per-stream FrameMetadataOak channels
                     mcap_entries = [(tracker, "oak_metadata")]
                     with mcap.McapRecorder.create(
                         mcap_filename,
@@ -168,9 +168,8 @@ def run_test(duration: float = 10.0, metadata_track: bool = True):
                             frame_count += 1
 
                             elapsed = time.time() - start_time
-                            oak_data = tracker.get_data(session)
-                            for md in oak_data.streams:
-                                name = md.stream.name
+                            for idx, name in enumerate(stream_names):
+                                md = tracker.get_stream_data(session, idx)
                                 if (
                                     md.timestamp
                                     and md.timestamp.device_time

examples/schemaio/frame_metadata_printer.cpp

@@ -5,8 +5,8 @@
  * @file frame_metadata_printer.cpp
  * @brief Standalone application that reads and prints camera frame metadata from the OpenXR runtime.
  *
- * This application demonstrates using FrameMetadataTrackerOak to read composed
- * CameraMetadataOak (containing per-stream FrameMetadataOak) pushed by a camera plugin.
+ * This application demonstrates using FrameMetadataTrackerOak to read per-stream
+ * FrameMetadataOak pushed by a camera plugin, with each stream on its own MCAP channel.
  *
  * Usage:
  *   ./frame_metadata_printer --collection-prefix=<prefix>
@@ -26,12 +26,14 @@
 
 static constexpr size_t MAX_FLATBUFFER_SIZE = 128;
 
-void print_oak_metadata(const core::CameraMetadataOakT& data, size_t sample_count)
+void print_oak_metadata(const core::FrameMetadataTrackerOak& tracker,
+                        const core::DeviceIOSession& session,
+                        size_t sample_count)
 {
     std::cout << "Sample " << sample_count << ": ";
-    for (size_t i = 0; i < data.streams.size(); ++i)
+    for (size_t i = 0; i < tracker.get_stream_count(); ++i)
     {
-        const auto& md = *data.streams[i];
+        const auto& md = tracker.get_stream_data(session, i);
         if (i > 0)
             std::cout << " | ";
         std::cout << core::EnumNameStreamType(md.stream) << " seq=" << md.sequence_number;
@@ -48,7 +50,7 @@ void print_usage(const char* program_name)
               << "  --collection-prefix=PREFIX  Tensor collection prefix (default: oak_camera)\n"
               << "  --help                      Show this help message\n"
               << "\nDescription:\n"
-              << "  Reads and prints CameraMetadataOak samples (all streams) pushed by a camera plugin.\n"
+              << "  Reads and prints per-stream FrameMetadataOak samples pushed by a camera plugin.\n"
               << "  The collection-prefix must match the value used by the camera plugin.\n";
 }
 
@@ -111,18 +113,17 @@ try
             break;
         }
 
-        const auto& data = tracker->get_data(*session);
-
         // Detect new data by checking if any stream has a newer device_time
         int64_t max_dt = -1;
-        for (const auto& md : data.streams)
+        for (size_t i = 0; i < tracker->get_stream_count(); ++i)
         {
-            if (md->timestamp)
-                max_dt = std::max(max_dt, md->timestamp->device_time());
+            const auto& md = tracker->get_stream_data(*session, i);
+            if (md.timestamp)
+                max_dt = std::max(max_dt, md.timestamp->device_time());
         }
         if (max_dt > last_device_time)
         {
-            print_oak_metadata(data, ++received_count);
+            print_oak_metadata(*tracker, *session, ++received_count);
             last_device_time = max_dt;
         }
 

src/core/deviceio/cpp/frame_metadata_tracker_oak.cpp

@@ -8,6 +8,7 @@
 #include <flatbuffers/flatbuffers.h>
 #include <schema/oak_bfbs_generated.h>
 
+#include <stdexcept>
 #include <vector>
 
 namespace core
@@ -48,36 +49,41 @@ class FrameMetadataTrackerOak::Impl : public ITrackerImpl
             }
         }
 
-        m_data.streams.clear();
-        for (const auto& s : m_streams)
-            m_data.streams.push_back(std::make_unique<FrameMetadataOakT>(s.data));
-
         return true;
     }
 
-    Timestamp serialize(flatbuffers::FlatBufferBuilder& builder, size_t /*channel_index*/) const override
+    Timestamp serialize(flatbuffers::FlatBufferBuilder& builder, size_t channel_index) const override
     {
-        auto offset = CameraMetadataOak::Pack(builder, &m_data);
-        builder.Finish(offset);
-
-        Timestamp latest{};
-        for (const auto& entry : m_data.streams)
+        if (channel_index >= m_streams.size())
         {
-            if (entry->timestamp && entry->timestamp->device_time() > latest.device_time())
-                latest = *entry->timestamp;
+            throw std::runtime_error("FrameMetadataTrackerOak::serialize: invalid channel_index " +
+                                     std::to_string(channel_index) + " (have " + std::to_string(m_streams.size()) +
+                                     " streams)");
         }
-        return latest;
+
+        const auto& data = m_streams[channel_index].data;
+        auto offset = FrameMetadataOak::Pack(builder, &data);
+        builder.Finish(offset);
+
+        if (data.timestamp)
+            return *data.timestamp;
+        return Timestamp{};
     }
 
-    const CameraMetadataOakT& get_data() const
+    const FrameMetadataOakT& get_stream_data(size_t stream_index) const
     {
-        return m_data;
+        if (stream_index >= m_streams.size())
+        {
+            throw std::runtime_error("FrameMetadataTrackerOak::get_stream_data: invalid stream_index " +
+                                     std::to_string(stream_index) + " (have " + std::to_string(m_streams.size()) +
+                                     " streams)");
+        }
+        return m_streams[stream_index].data;
     }
 
 private:
     std::vector<StreamState> m_streams;
     std::vector<uint8_t> m_buffer;
-    CameraMetadataOakT m_data;
 };
 
 // ============================================================================
@@ -88,12 +94,19 @@ FrameMetadataTrackerOak::FrameMetadataTrackerOak(const std::string& collection_p
                                                  const std::vector<StreamType>& streams,
                                                  size_t max_flatbuffer_size)
 {
+    if (streams.empty())
+    {
+        throw std::runtime_error("FrameMetadataTrackerOak: at least one stream is required");
+    }
+
     for (auto type : streams)
     {
-        m_configs.push_back({ .collection_id = collection_prefix + "/" + EnumNameStreamType(type),
+        const char* name = EnumNameStreamType(type);
+        m_configs.push_back({ .collection_id = collection_prefix + "/" + name,
                               .max_flatbuffer_size = max_flatbuffer_size,
                               .tensor_identifier = "frame_metadata",
-                              .localized_name = std::string("FrameMetadataTracker_") + EnumNameStreamType(type) });
+                              .localized_name = std::string("FrameMetadataTracker_") + name });
+        m_channel_names.emplace_back(name);
     }
 }
 
@@ -109,18 +122,18 @@ std::string_view FrameMetadataTrackerOak::get_name() const
 
 std::string_view FrameMetadataTrackerOak::get_schema_name() const
 {
-    return "core.CameraMetadataOak";
+    return "core.FrameMetadataOak";
 }
 
 std::string_view FrameMetadataTrackerOak::get_schema_text() const
 {
     return std::string_view(
-        reinterpret_cast<const char*>(CameraMetadataOakBinarySchema::data()), CameraMetadataOakBinarySchema::size());
+        reinterpret_cast<const char*>(FrameMetadataOakBinarySchema::data()), FrameMetadataOakBinarySchema::size());
 }
 
-const CameraMetadataOakT& FrameMetadataTrackerOak::get_data(const DeviceIOSession& session) const
+const FrameMetadataOakT& FrameMetadataTrackerOak::get_stream_data(const DeviceIOSession& session, size_t stream_index) const
 {
-    return static_cast<const Impl&>(session.get_tracker_impl(*this)).get_data();
+    return static_cast<const Impl&>(session.get_tracker_impl(*this)).get_stream_data(stream_index);
 }
 
 std::shared_ptr<ITrackerImpl> FrameMetadataTrackerOak::create_tracker(const OpenXRSessionHandles& handles) const

src/core/deviceio/cpp/inc/deviceio/frame_metadata_tracker_oak.hpp

@@ -16,20 +16,19 @@ namespace core
 {
 
 /*!
- * @brief Composite tracker for reading OAK FrameMetadataOak from multiple streams.
+ * @brief Multi-channel tracker for reading OAK FrameMetadataOak from multiple streams.
  *
- * Maintains one SchemaTracker per stream and composes them into a single
- * CameraMetadataOak message for serialization / MCAP recording.
+ * Maintains one SchemaTracker per stream and records each as a separate MCAP
+ * channel using FrameMetadataOak as the root type.
  *
  * Usage:
  * @code
  * auto tracker = std::make_shared<FrameMetadataTrackerOak>(
  *     "oak_camera", {StreamType_Color, StreamType_MonoLeft});
  * // ... create DeviceIOSession with tracker ...
  * session->update();
- * const auto& data = tracker->get_data(*session);
- * for (const auto& md : data.streams)
- *     std::cout << EnumNameStreamType(md->stream) << std::endl;
+ * const auto& color = tracker->get_stream_data(*session, 0);
+ * std::cout << EnumNameStreamType(color.stream) << " seq=" << color.sequence_number << std::endl;
  * @endcode
  */
 class FrameMetadataTrackerOak : public ITracker
@@ -57,18 +56,28 @@ class FrameMetadataTrackerOak : public ITracker
 
     std::vector<std::string> get_record_channels() const override
     {
-        return { "frame_metadata" };
+        return m_channel_names;
     }
 
     /*!
-     * @brief Get the composed OAK metadata containing all tracked streams.
+     * @brief Get per-stream frame metadata.
+     * @param session Active DeviceIOSession.
+     * @param stream_index Index into the streams vector passed at construction.
+     * @return Reference to the FrameMetadataOakT for that stream.
      */
-    const CameraMetadataOakT& get_data(const DeviceIOSession& session) const;
+    const FrameMetadataOakT& get_stream_data(const DeviceIOSession& session, size_t stream_index) const;
+
+    //! Number of streams this tracker is configured for.
+    size_t get_stream_count() const
+    {
+        return m_channel_names.size();
+    }
 
 private:
     std::shared_ptr<ITrackerImpl> create_tracker(const OpenXRSessionHandles& handles) const override;
 
     std::vector<SchemaTrackerConfig> m_configs;
+    std::vector<std::string> m_channel_names;
     class Impl;
 };
 

src/core/deviceio/python/deviceio_bindings.cpp

@@ -65,11 +65,14 @@ PYBIND11_MODULE(_deviceio, m)
              py::arg("max_flatbuffer_size") = core::FrameMetadataTrackerOak::DEFAULT_MAX_FLATBUFFER_SIZE,
              "Construct a multi-stream FrameMetadataTrackerOak")
         .def(
-            "get_data",
-            [](core::FrameMetadataTrackerOak& self, PyDeviceIOSession& session) -> const core::CameraMetadataOakT&
-            { return self.get_data(session.native()); },
-            py::arg("session"), py::return_value_policy::reference_internal,
-            "Get composed CameraMetadataOak containing all tracked streams");
+            "get_stream_data",
+            [](core::FrameMetadataTrackerOak& self, PyDeviceIOSession& session,
+               size_t stream_index) -> const core::FrameMetadataOakT&
+            { return self.get_stream_data(session.native(), stream_index); },
+            py::arg("session"), py::arg("stream_index"), py::return_value_policy::reference_internal,
+            "Get FrameMetadataOak for a specific stream by index")
+        .def_property_readonly("stream_count", &core::FrameMetadataTrackerOak::get_stream_count,
+                               "Number of streams this tracker is configured for");
 
     // Generic3AxisPedalTracker class
     py::class_<core::Generic3AxisPedalTracker, core::ITracker, std::shared_ptr<core::Generic3AxisPedalTracker>>(

src/core/deviceio/python/deviceio_init.py

@@ -8,7 +8,7 @@
 Note: HeadTracker.get_head(session) returns HeadPoseT from isaacteleop.schema.
     HandTracker.get_left_hand(session) / get_right_hand(session) return HandPoseT from isaacteleop.schema.
     ControllerTracker.get_left_controller(session) / get_right_controller(session) return ControllerSnapshot from isaacteleop.schema.
-    FrameMetadataTrackerOak.get_data(session) returns CameraMetadataOak from isaacteleop.schema.
+    FrameMetadataTrackerOak.get_stream_data(session, index) returns FrameMetadataOak from isaacteleop.schema.
 Import these types from isaacteleop.schema if you need to work with pose types.
 """
 
@@ -38,7 +38,6 @@
     ControllerSnapshot,
     StreamType,
     FrameMetadataOak,
-    CameraMetadataOak,
     Generic3AxisPedalOutput,
     Timestamp,
 )
@@ -49,7 +48,6 @@
     "ControllerSnapshot",
     "StreamType",
     "FrameMetadataOak",
-    "CameraMetadataOak",
     "Generic3AxisPedalOutput",
     "Timestamp",
     "ITracker",

src/core/schema/fbs/oak.fbs

@@ -9,6 +9,7 @@ namespace core;
 enum StreamType : byte { Color = 0, MonoLeft = 1, MonoRight = 2 }
 
 // Per-frame metadata pushed by the OAK camera plugin (one per stream).
+// Each stream is recorded as a separate MCAP channel using this as the root type.
 table FrameMetadataOak {
   // Which camera sensor this metadata belongs to.
   stream: StreamType (id: 0);
@@ -20,9 +21,4 @@ table FrameMetadataOak {
   sequence_number: uint64 (id: 2);
 }
 
-// Composed metadata from all active OAK camera streams.
-table CameraMetadataOak {
-  streams: [FrameMetadataOak] (id: 0);
-}
-
-root_type CameraMetadataOak;
+root_type FrameMetadataOak;

src/core/schema/python/oak_bindings.h

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 // Python bindings for the OAK FlatBuffer schema.
-// Types: StreamType (enum), FrameMetadataOak (table), CameraMetadataOak (composite table).
+// Types: StreamType (enum), FrameMetadataOak (table).
 
 #pragma once
 
@@ -51,22 +51,6 @@ inline void bind_oak(py::module& m)
                  result += ", sequence_number=" + std::to_string(metadata.sequence_number) + ")";
                  return result;
              });
-
-    py::class_<CameraMetadataOakT>(m, "CameraMetadataOak")
-        .def(py::init<>())
-        .def_property_readonly(
-            "streams",
-            [](const CameraMetadataOakT& self)
-            {
-                std::vector<const FrameMetadataOakT*> out;
-                out.reserve(self.streams.size());
-                for (const auto& entry : self.streams)
-                    out.push_back(entry.get());
-                return out;
-            },
-            "List of per-stream FrameMetadataOak entries")
-        .def("__repr__", [](const CameraMetadataOakT& self)
-             { return "CameraMetadataOak(streams=" + std::to_string(self.streams.size()) + ")"; });
 }
 
 } // namespace core

src/core/schema/python/schema_init.py

@@ -31,7 +31,6 @@
     # Camera-related types.
     StreamType,
     FrameMetadataOak,
-    CameraMetadataOak,
     # Full body-related types.
     BodyJointPico,
     BodyJointPose,
@@ -64,7 +63,6 @@
     # Camera types.
     "StreamType",
     "FrameMetadataOak",
-    "CameraMetadataOak",
     # Full body types.
     "BodyJointPose",
     "BodyJointsPico",