Sxian/clt 2304/implement remote audiovideo tracks videostream audiostream (#11)

* initial commit for video stream and other hooks

* Get things functional, be able to publish remote audio / video tracks, and attach to AudioStream and VideoStream

Author: xianshijing-lk

CMakeLists.txt

@@ -158,6 +158,7 @@ add_custom_command(
 add_library(livekit
   include/livekit/audio_frame.h
   include/livekit/audio_source.h
+  include/livekit/audio_stream.h
   include/livekit/room.h
   include/livekit/room_delegate.h
   include/livekit/ffi_handle.h
@@ -166,6 +167,7 @@ add_library(livekit
   include/livekit/remote_audio_track.h
   include/livekit/participant.h
   include/livekit/local_participant.h
+  include/livekit/remote_participant.h
   include/livekit/livekit.h
   include/livekit/stats.h
   include/livekit/track.h
@@ -174,10 +176,12 @@ add_library(livekit
   include/livekit/remote_track_publication.h
   include/livekit/video_frame.h
   include/livekit/video_source.h
+  include/livekit/video_stream.h
   include/livekit/local_video_track.h
   include/livekit/remote_video_track.h
   src/audio_frame.cpp
   src/audio_source.cpp
+  src/audio_stream.cpp
   src/ffi_handle.cpp
   src/ffi_client.cpp
   src/local_audio_track.cpp
@@ -186,6 +190,7 @@ add_library(livekit
   src/room_proto_converter.cpp
   src/room_proto_converter.h
   src/local_participant.cpp
+  src/remote_participant.cpp
   src/stats.cpp
   src/track.cpp
   src/track_proto_converter.cpp
@@ -195,6 +200,7 @@ add_library(livekit
   src/remote_track_publication.cpp
   src/video_frame.cpp
   src/video_source.cpp
+  src/video_stream.cpp
   src/local_video_track.cpp
   src/remote_video_track.cpp
   src/video_utils.cpp

include/livekit/audio_stream.h

@@ -0,0 +1,107 @@
+/*
+ * Copyright 2025 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an “AS IS” BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <condition_variable>
+#include <cstdint>
+#include <deque>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <string>
+
+#include "audio_frame.h"
+#include "ffi_handle.h"
+#include "participant.h"
+#include "track.h"
+
+namespace livekit {
+
+namespace proto {
+class FfiEvent;
+}
+
+struct AudioFrameEvent {
+  AudioFrame frame;
+};
+
+class AudioStream {
+public:
+  struct Options {
+    std::size_t capacity{0}; // 0 = unbounded
+    int sample_rate{48000};
+    int num_channels{1};
+    std::string noise_cancellation_module;       // empty = disabled
+    std::string noise_cancellation_options_json; // empty = no options
+  };
+
+  // Factory: create an AudioStream bound to a specific Track
+  static std::unique_ptr<AudioStream>
+  from_track(const std::shared_ptr<Track> &track, const Options &options);
+
+  // Factory: create an AudioStream from a Participant + TrackSource
+  static std::unique_ptr<AudioStream> from_participant(Participant &participant,
+                                                       TrackSource track_source,
+                                                       const Options &options);
+
+  ~AudioStream();
+
+  AudioStream(const AudioStream &) = delete;
+  AudioStream &operator=(const AudioStream &) = delete;
+  AudioStream(AudioStream &&) noexcept;
+  AudioStream &operator=(AudioStream &&) noexcept;
+
+  /// Blocking read: returns true if a frame was delivered,
+  /// false if the stream has ended (EOS or closed).
+  bool read(AudioFrameEvent &out_event);
+
+  /// Signal that we are no longer interested in frames.
+  /// Disposes the underlying FFI stream and removes the listener.
+  void close();
+
+private:
+  AudioStream() = default;
+
+  void init_from_track(const std::shared_ptr<Track> &track,
+                       const Options &options);
+  void init_from_participant(Participant &participant, TrackSource track_source,
+                             const Options &options);
+
+  // FFI event handler (registered with FfiClient)
+  void on_ffi_event(const proto::FfiEvent &event);
+
+  // Queue helpers
+  void push_frame(AudioFrameEvent &&ev);
+  void push_eos();
+
+  mutable std::mutex mutex_;
+  std::condition_variable cv_;
+  std::deque<AudioFrameEvent> queue_;
+  std::size_t capacity_{0};
+  bool eof_{false};
+  bool closed_{false};
+
+  Options options_;
+
+  // Underlying FFI audio stream handle
+  FfiHandle stream_handle_;
+
+  // Listener id registered on FfiClient
+  std::int64_t listener_id_{0};
+};
+
+} // namespace livekit

include/livekit/livekit.h

@@ -21,6 +21,7 @@
 #include "local_track_publication.h"
 #include "local_video_track.h"
 #include "participant.h"
+#include "remote_participant.h"
 #include "room.h"
 #include "room_delegate.h"
 #include "track_publication.h"

include/livekit/local_track_publication.h

@@ -28,7 +28,8 @@ class Track;
 
 class LocalTrackPublication : public TrackPublication {
 public:
-  /// Construct from an OwnedTrackPublication proto.
+  /// Note, this RemoteTrackPublication is constructed internally only;
+  /// safe to accept proto::OwnedTrackPublication.
   explicit LocalTrackPublication(const proto::OwnedTrackPublication &owned);
 
   /// Typed accessor for the attached LocalTrack (if any).

include/livekit/participant.h

@@ -41,7 +41,7 @@ class Participant {
         metadata_(std::move(metadata)), attributes_(std::move(attributes)),
         kind_(kind), reason_(reason) {}
 
-  // Plain getters/setters (caller ensures threading)
+  // Plain getters (caller ensures threading)
   const std::string &sid() const noexcept { return sid_; }
   const std::string &name() const noexcept { return name_; }
   const std::string &identity() const noexcept { return identity_; }
@@ -55,6 +55,24 @@ class Participant {
 
   uintptr_t ffiHandleId() const noexcept { return handle_.get(); }
 
+  // Setters (caller ensures threading)
+  void set_name(std::string name) noexcept { name_ = std::move(name); }
+  void set_metadata(std::string metadata) noexcept {
+    metadata_ = std::move(metadata);
+  }
+  void
+  set_attributes(std::unordered_map<std::string, std::string> attrs) noexcept {
+    attributes_ = std::move(attrs);
+  }
+  void set_attribute(const std::string &key, const std::string &value) {
+    attributes_[key] = value;
+  }
+  void remove_attribute(const std::string &key) { attributes_.erase(key); }
+  void set_kind(ParticipantKind kind) noexcept { kind_ = kind; }
+  void set_disconnect_reason(DisconnectReason reason) noexcept {
+    reason_ = reason;
+  }
+
 private:
   FfiHandle handle_;
   std::string sid_, name_, identity_, metadata_;

include/livekit/remote_participant.h

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2025 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an “AS IS” BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "participant.h"
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+
+namespace livekit {
+
+class RemoteTrackPublication;
+
+class RemoteParticipant : public Participant {
+public:
+  using TrackPublicationMap =
+      std::unordered_map<std::string, std::shared_ptr<RemoteTrackPublication>>;
+
+  RemoteParticipant(FfiHandle handle, std::string sid, std::string name,
+                    std::string identity, std::string metadata,
+                    std::unordered_map<std::string, std::string> attributes,
+                    ParticipantKind kind, DisconnectReason reason);
+
+  // A dictionary of track publications associated with the participant.
+  const TrackPublicationMap &track_publications() const noexcept {
+    return track_publications_;
+  }
+
+  // Optional: non-const access if you want to mutate in-place.
+  TrackPublicationMap &mutable_track_publications() noexcept {
+    return track_publications_;
+  }
+
+  // C++ equivalent of Python's __repr__
+  std::string to_string() const;
+
+private:
+  TrackPublicationMap track_publications_;
+};
+
+// Convenience for logging / streaming
+std::ostream &operator<<(std::ostream &os,
+                         const RemoteParticipant &participant);
+
+} // namespace livekit

include/livekit/remote_track_publication.h

@@ -28,6 +28,8 @@ class Track;
 
 class RemoteTrackPublication : public TrackPublication {
 public:
+  /// Note, this RemoteTrackPublication is constructed internally only;
+  /// safe to accept proto::OwnedTrackPublication.
   explicit RemoteTrackPublication(const proto::OwnedTrackPublication &owned);
 
   /// Typed accessor for the attached RemoteTrack (if any).

include/livekit/room.h

@@ -32,6 +32,7 @@ class FfiEvent;
 }
 
 class LocalParticipant;
+class RemoteParticipant;
 
 class Room {
 public:
@@ -43,6 +44,7 @@ class Room {
   // Accessors
   RoomInfoData room_info() const;
   LocalParticipant *local_participant() const;
+  RemoteParticipant *remote_participant(const std::string &identity) const;
 
 private:
   mutable std::mutex lock_;
@@ -51,6 +53,8 @@ class Room {
   RoomInfoData room_info_;
   std::shared_ptr<FfiHandle> room_handle_;
   std::unique_ptr<LocalParticipant> local_participant_;
+  std::unordered_map<std::string, std::unique_ptr<RemoteParticipant>>
+      remote_participants_;
 
   void OnEvent(const proto::FfiEvent &event);
 };

include/livekit/track_publication.h

@@ -22,7 +22,7 @@
 #include <vector>
 
 #include "livekit/ffi_handle.h"
-#include "livekit/track.h" // TrackKind, TrackSource, AudioTrackFeature
+#include "livekit/track.h"
 
 namespace livekit {
 

include/livekit/video_frame.h

@@ -44,6 +44,10 @@ struct VideoPlaneInfo {
   std::uint32_t size;      // plane size in bytes
 };
 
+namespace proto {
+class OwnedVideoBuffer;
+}
+
 /**
  * Public SDK representation of a video frame.
  *
@@ -63,16 +67,6 @@ class LKVideoFrame {
   LKVideoFrame(LKVideoFrame &&) noexcept = default;
   LKVideoFrame &operator=(LKVideoFrame &&) noexcept = default;
 
-  /* LKVideoFrame(LKVideoFrame&& other) noexcept
-      : width_(other.width_),
-        height_(other.height_),
-        type_(other.type_),
-        data_(std::move(other.data_)) {
-      other.width_ = 0;
-      other.height_ = 0;
-  }
-  LKVideoFrame& operator=(LKVideoFrame&& other) noexcept;*/
-
   /**
    * Allocate a new frame with the correct buffer size for the given format.
    * Data is zero-initialized.
@@ -123,6 +117,12 @@ class LKVideoFrame {
    */
   LKVideoFrame convert(VideoBufferType dst, bool flip_y = false) const;
 
+protected:
+  friend class VideoStream;
+  // Only internal classes (e.g., VideoStream)
+  // should construct frames directly from FFI buffers.
+  static LKVideoFrame fromOwnedInfo(const proto::OwnedVideoBuffer &owned);
+
 private:
   int width_;
   int height_;