audio_stream

Author: theomonnom

examples/basic_room/room.py

@@ -10,6 +10,9 @@
 async def main():
     room = livekit.Room()
 
+    audio_stream = None
+    video_stream = None
+
     logging.info("connecting to %s", URL)
     try:
         await room.connect(URL, TOKEN)
@@ -41,11 +44,21 @@ def on_track_unpublished(publication: livekit.LocalTrackPublication, participant
     def on_track_subscribed(track: livekit.Track, publication: livekit.RemoteTrackPublication, participant: livekit.RemoteParticipant):
         logging.info("track subscribed: %s", publication.sid)
         if track.kind == livekit.TrackKind.KIND_VIDEO:
+            nonlocal video_stream
             video_stream = livekit.VideoStream(track)
 
             @video_stream.on("frame_received")
             def on_video_frame(frame: livekit.VideoFrame):
-                # e.g: Do something with the frames here :)
+                # received a video frame from the track
+                pass
+        elif track.kind == livekit.TrackKind.KIND_AUDIO:
+            print("Subscribed to an Audio Track")
+            nonlocal audio_stream
+            audio_stream = livekit.AudioStream(track)
+
+            @audio_stream.on('frame_received')
+            def on_audio_frame(frame: livekit.AudioFrame):
+                # received an audio frame from the track
                 pass
 
     @room.on("track_unsubscribed")

examples/face_landmark/room.py

@@ -75,9 +75,12 @@ async def room():
     await room.connect(URL, TOKEN)
     print("connected to room: " + room.name)
 
+    video_stream = None
+
     @room.on("track_subscribed")
     def on_track_subscribed(track: livekit.Track, publication: livekit.RemoteTrackPublication, participant: livekit.RemoteParticipant):
         if track.kind == livekit.TrackKind.KIND_VIDEO:
+            nonlocal video_stream
             video_stream = livekit.VideoStream(track)
 
             @video_stream.on("frame_received")

livekit/__init__.py

@@ -19,3 +19,7 @@
                           PlanarYuv8Buffer, PlanarYuv16Buffer, I420Buffer, I420ABuffer, I422Buffer, I010Buffer, NV12Buffer)
 from .video_stream import VideoStream
 from .video_source import VideoSource
+
+from .audio_frame import AudioFrame
+from .audio_stream import AudioStream
+from .audio_source import AudioSource

livekit/audio_frame.py

@@ -0,0 +1,14 @@
+import ctypes
+
+
+class AudioFrame():
+    def __init__(self, sample_rate: int, num_channels: int, samples_per_channel: int, data=None, ffi_handle=None):
+        self._ffi_handle = ffi_handle
+        self.sample_rate = sample_rate
+        self.num_channels = num_channels
+        self.samples_per_channel = samples_per_channel
+        if data is None:
+            self.data = (ctypes.c_int16 *
+                         (num_channels * samples_per_channel))()
+        else:
+            self.data = data

livekit/audio_source.py

@@ -0,0 +1,5 @@
+
+
+class AudioSource():
+    def __init__(self):
+        pass

livekit/audio_stream.py

@@ -1,14 +1,15 @@
 from pyee.asyncio import AsyncIOEventEmitter
 from ._ffi_client import (FfiClient, FfiHandle)
-from livekit import (Track, Room, Participant)
-from ._proto import track_pb2 as proto_track
+from livekit import Track
 from ._proto import ffi_pb2 as proto_ffi
-from ._proto import video_frame_pb2 as proto_video_frame
-from .video_frame import (VideoFrame, VideoFrameBuffer)
+from ._proto import audio_frame_pb2 as proto_audio_frame
+from .audio_frame import AudioFrame
+import ctypes
+import weakref
 
 
 class AudioStream(AsyncIOEventEmitter):
-    _streams: dict[int, 'AudioStream'] = {}
+    _streams: dict[int, weakref.ref['AudioStream']] = {}
     _initialized = False
 
     @classmethod
@@ -23,9 +24,46 @@ def initalize(cls):
                                 cls._on_audio_stream_event)
 
     @classmethod
-    def _on_audio_stream_event(cls, event: proto_video_frame.AudioStreamEvent):
-        pass
+    def _on_audio_stream_event(cls, event: proto_audio_frame.AudioStreamEvent):
+        stream = cls._streams.get(event.handle.id)
+        if stream is None:
+            return
+
+        stream = stream()
+        if stream is None:
+            return
+
+        which = event.WhichOneof('message')
+        if which == 'frame_received':
+            frame_info = event.frame_received.frame
+            ffi_handle = FfiHandle(frame_info.handle.id)
+            data_len = frame_info.num_channels * frame_info.samples_per_channel
+            data = ctypes.cast(frame_info.data_ptr,
+                               ctypes.POINTER(ctypes.c_uint16 * data_len)).contents
+            frame = AudioFrame(
+                frame_info.sample_rate, frame_info.num_channels, frame_info.samples_per_channel, data, ffi_handle)
+            stream._on_frame_received(frame)
 
     def __init__(self, track: Track):
         super().__init__()
         self.initalize()
+
+        req = proto_ffi.FfiRequest()
+        new_audio_stream = req.new_audio_stream
+        new_audio_stream.track_handle.id = track._ffi_handle.handle
+        new_audio_stream.type = proto_audio_frame.AudioStreamType.AUDIO_STREAM_NATIVE
+
+        ffi_client = FfiClient()
+        resp = ffi_client.request(req)
+        stream_info = resp.new_audio_stream.stream
+
+        self._streams[stream_info.handle.id] = weakref.ref(self)
+        self._ffi_handle = FfiHandle(stream_info.handle.id)
+        self._info = stream_info
+        self._track = track
+
+    def _on_frame_received(self, frame: AudioFrame):
+        self.emit('frame_received', frame)
+
+    def __del__(self):
+        self._streams.pop(self._ffi_handle.handle, None)