Merge branch 'main' into lukas/token-refresh

Author: lukasIO

livekit-api/livekit/api/version.py

@@ -1 +1 @@
-__version__ = "1.0.7"
+__version__ = "1.1.0"

livekit-api/setup.py

@@ -53,7 +53,7 @@
         "aiohttp>=3.9.0",
         "protobuf>=4",
         "types-protobuf>=4",
-        "livekit-protocol>=1.0.4,<2.0.0",
+        "livekit-protocol>=1.1.1,<2.0.0",
     ],
     package_data={
         "livekit.api": ["py.typed", "*.pyi", "**/*.pyi"],

livekit-protocol/livekit/protocol/version.py

@@ -108,6 +108,7 @@
     ByteStreamWriter,
     ByteStreamReader,
 )
+from .frame_processor import FrameProcessor
 
 __all__ = [
     "ConnectionQuality",
@@ -184,6 +185,7 @@
     "ByteStreamReader",
     "ByteStreamWriter",
     "AudioProcessingModule",
+    "FrameProcessor",
     "__version__",
 ]
 

livekit-rtc/livekit/rtc/__init__.py

@@ -25,8 +25,10 @@
 from ._proto.track_pb2 import TrackSource
 from ._utils import RingQueue, task_done_logger
 from .audio_frame import AudioFrame
+from .log import logger
 from .participant import Participant
 from .track import Track
+from .frame_processor import FrameProcessor
 
 
 @dataclass
@@ -62,7 +64,7 @@ def __init__(
         sample_rate: int = 48000,
         num_channels: int = 1,
         frame_size_ms: int | None = None,
-        noise_cancellation: Optional[NoiseCancellationOptions] = None,
+        noise_cancellation: Optional[NoiseCancellationOptions | FrameProcessor[AudioFrame]] = None,
         **kwargs,
     ) -> None:
         """Initialize an `AudioStream` instance.
@@ -76,8 +78,8 @@ def __init__(
             sample_rate (int, optional): The sample rate for the audio stream in Hz.
                 Defaults to 48000.
             num_channels (int, optional): The number of audio channels. Defaults to 1.
-            noise_cancellation (Optional[NoiseCancellationOptions], optional):
-                If noise cancellation is used, pass a `NoiseCancellationOptions` instance
+            noise_cancellation (Optional[NoiseCancellationOptions | FrameProcessor[AudioFrame]], optional):
+                If noise cancellation is used, pass a `NoiseCancellationOptions` or `FrameProcessor[AudioFrame]` instance
                 created by the noise cancellation module.
 
         Example:
@@ -105,9 +107,12 @@ def __init__(
 
         self._audio_filter_module = None
         self._audio_filter_options = None
-        if noise_cancellation is not None:
+        if isinstance(noise_cancellation, NoiseCancellationOptions):
             self._audio_filter_module = noise_cancellation.module_id
             self._audio_filter_options = noise_cancellation.options
+        elif isinstance(noise_cancellation, FrameProcessor):
+            self._processor = noise_cancellation
+
         self._task = self._loop.create_task(self._run())
         self._task.add_done_callback(task_done_logger)
 
@@ -132,7 +137,7 @@ def from_participant(
         sample_rate: int = 48000,
         num_channels: int = 1,
         frame_size_ms: int | None = None,
-        noise_cancellation: Optional[NoiseCancellationOptions] = None,
+        noise_cancellation: Optional[NoiseCancellationOptions | FrameProcessor[AudioFrame]] = None,
     ) -> AudioStream:
         """Create an `AudioStream` from a participant's audio track.
 
@@ -182,7 +187,7 @@ def from_track(
         sample_rate: int = 48000,
         num_channels: int = 1,
         frame_size_ms: int | None = None,
-        noise_cancellation: Optional[NoiseCancellationOptions] = None,
+        noise_cancellation: Optional[NoiseCancellationOptions | FrameProcessor[AudioFrame]] = None,
     ) -> AudioStream:
         """Create an `AudioStream` from an existing audio track.
 
@@ -268,6 +273,14 @@ async def _run(self):
             if audio_event.HasField("frame_received"):
                 owned_buffer_info = audio_event.frame_received.frame
                 frame = AudioFrame._from_owned_info(owned_buffer_info)
+                if self._processor is not None and self._processor.enabled:
+                    try:
+                        frame = self._processor._process(frame)
+                    except Exception:
+                        logger.warning(
+                            "Frame processing failed, passing through original frame",
+                            exc_info=True,
+                        )
                 event = AudioFrameEvent(frame)
                 self._queue.put(event)
             elif audio_event.HasField("eos"):

livekit-rtc/livekit/rtc/audio_stream.py

@@ -0,0 +1,33 @@
+from abc import ABC, abstractmethod
+from typing import Generic, TypeVar, Union
+from .audio_frame import AudioFrame
+from .video_frame import VideoFrame
+
+
+T = TypeVar("T", bound=Union[AudioFrame, VideoFrame])
+
+
+class FrameProcessor(Generic[T], ABC):
+    @property
+    @abstractmethod
+    def enabled(self) -> bool: ...
+
+    @enabled.setter
+    @abstractmethod
+    def enabled(self, value: bool) -> None: ...
+
+    def _on_stream_info_updated(
+        self,
+        *,
+        room_name: str,
+        participant_identity: str,
+        publication_sid: str,
+    ): ...
+
+    def _on_credentials_updated(self, *, token: str, url: str): ...
+
+    @abstractmethod
+    def _process(self, frame: T) -> T: ...
+
+    @abstractmethod
+    def _close(self): ...

livekit-rtc/livekit/rtc/frame_processor.py

@@ -20,3 +20,6 @@ manylinux-s390x-image = "manylinux_2_28"
 manylinux-pypy_x86_64-image = "manylinux_2_28"
 manylinux-pypy_i686-image = "manylinux_2_28"
 manylinux-pypy_aarch64-image = "manylinux_2_28"
+
+[tool.cibuildwheel.linux]
+before-build = "pip install requests && python rust-sdks/download_ffi.py --output livekit/rtc/resources && yum install -y libX11-devel libXrandr-devel libXext-devel libXfixes-devel libXdamage-devel libXcomposite-devel libXi-devel mesa-libGL-devel || apt install -y xorg-dev"