wip

Author: theomonnom

libwebrtc/src/lib.rs

@@ -66,7 +66,7 @@ pub mod video_track;
 pub mod native {
     pub use webrtc_sys::webrtc::ffi::create_random_uuid;
 
-    pub use crate::imp::{aec, audio_resampler, frame_cryptor, yuv_helper};
+    pub use crate::imp::{apm, audio_resampler, frame_cryptor, yuv_helper};
 }
 
 #[cfg(target_os = "android")]

libwebrtc/src/native/aec.rs

@@ -0,0 +1,100 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use cxx::UniquePtr;
+use webrtc_sys::apm::ffi as sys_apm;
+
+use crate::{RtcError, RtcErrorType};
+
+pub struct AudioProcessingModule {
+    sys_handle: UniquePtr<sys_apm::AudioProcessingModule>,
+}
+
+impl AudioProcessingModule {
+    pub fn new(
+        echo_canceller_enabled: bool,
+        gain_controller_enabled: bool,
+        high_pass_filter_enabled: bool,
+        noise_suppression_enabled: bool,
+    ) -> Self {
+        Self {
+            sys_handle: sys_apm::create_apm(
+                echo_canceller_enabled,
+                gain_controller_enabled,
+                high_pass_filter_enabled,
+                noise_suppression_enabled,
+            ),
+        }
+    }
+
+    pub fn process_stream(
+        &mut self,
+        data: &mut [i16],
+        sample_rate: i32,
+        num_channels: i32,
+    ) -> Result<(), RtcError> {
+        let samples_count = (sample_rate as usize / 100) * num_channels as usize;
+        assert_eq!(data.len(), samples_count, "slice must have 10ms worth of samples");
+
+        if unsafe {
+            // using the same slice for src and dst is safe
+            self.sys_handle.pin_mut().process_stream(
+                data.as_mut_ptr(),
+                data.len(),
+                data.as_mut_ptr(),
+                data.len(),
+                sample_rate,
+                num_channels,
+            )
+        } == 0
+        {
+            Ok(())
+        } else {
+            Err(RtcError {
+                error_type: RtcErrorType::Internal,
+                message: "Failed to process stream".to_string(),
+            })
+        }
+    }
+
+    pub fn process_reverse_stream(
+        &mut self,
+        data: &mut [i16],
+        sample_rate: i32,
+        num_channels: i32,
+    ) -> Result<(), RtcError> {
+        let samples_count = (sample_rate as usize / 100) * num_channels as usize;
+        assert_eq!(data.len(), samples_count, "slice must have 10ms worth of samples");
+
+        if unsafe {
+            // using the same slice for src and dst is safe
+            self.sys_handle.pin_mut().process_reverse_stream(
+                data.as_mut_ptr(),
+                data.len(),
+                data.as_mut_ptr(),
+                data.len(),
+                sample_rate,
+                num_channels,
+            )
+        } == 0
+        {
+            Ok(())
+        } else {
+            Err(RtcError {
+                error_type: RtcErrorType::Internal,
+                message: "Failed to process reverse stream".to_string(),
+            })
+        }
+    }
+}

libwebrtc/src/native/apm.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub mod aec;
+pub mod apm;
 #[cfg(target_os = "android")]
 pub mod android;
 pub mod audio_resampler;

libwebrtc/src/native/mod.rs

@@ -120,6 +120,7 @@ impl EgressClient {
                     segment_outputs,
                     image_outputs,
                     output: None, // Deprecated
+                    ..Default::default()
                 },
                 self.base
                     .auth_header(VideoGrants { room_record: true, ..Default::default() }, None)?,

livekit-api/src/services/egress.rs

@@ -452,6 +452,7 @@ impl SIPClient {
                     headers: Default::default(),
                     include_headers: Default::default(),
                     media_encryption: Default::default(),
+                    ..Default::default()
                 },
                 self.base.auth_header(
                     Default::default(),

livekit-api/src/services/sip.rs

@@ -94,24 +94,38 @@ message RemixAndResampleResponse {
 // AEC
 
 
-message NewAecRequest {
-  required uint32 sample_rate = 1;
-  required uint32 num_channels = 2;
+message NewApmRequest {
+  required bool echo_canceller_enabled = 1;
+  required bool gain_controller_enabled = 2;
+  required bool high_pass_filter_enabled = 3;
+  required bool noise_suppression_enabled = 4;
 }
-message NewAecResponse {
-  required OwnedAec aec = 1;
+message NewApmResponse {
+  required OwnedApm apm = 1;
 }
 
-message CancelEchoRequest {
-  required uint64 aec_handle = 1;
-  required uint64 cap_ptr = 2; // *mut i16
-  required uint32 cap_size = 3; // in bytes
-  required uint64 rend_ptr = 4; // *const i16
-  required uint32 rend_size = 5;
+message ApmProcessStreamRequest {
+  required uint64 apm_handle = 1;
+  required uint64 data_ptr = 2; // *mut i16
+  required uint32 size = 3; // in bytes
+  required uint32 sample_rate = 4;
+  required uint32 num_channels = 5;
 }
 
-message CancelEchoResponse {
-  optional string error = 3;
+message ApmProcessStreamResponse {
+  optional string error = 1;
+}
+
+message ApmProcessReverseStreamRequest {
+  required uint64 apm_handle = 1;
+  required uint64 data_ptr = 2; // *mut i16
+  required uint32 size = 3; // in bytes
+  required uint32 sample_rate = 4;
+  required uint32 num_channels = 5;
+}
+
+message ApmProcessReverseStreamResponse {
+  optional string error = 1;
 }
 
 // New resampler using SoX (much better quality)
@@ -267,7 +281,7 @@ message OwnedAudioResampler {
 // AEC
 //
 
-message OwnedAec {
+message OwnedApm {
   required FfiOwnedHandle handle = 1;
 }
 

livekit-ffi/protocol/audio_frame.proto

@@ -122,6 +122,12 @@ message FfiRequest {
 
     // Audio Filter Plugin
     LoadAudioFilterPluginRequest load_audio_filter_plugin = 49;
+
+    NewApmRequest new_apm = 50;
+    ApmProcessStreamRequest apm_process_stream = 51;
+    ApmProcessReverseStreamRequest apm_process_reverse_stream = 52;
+
+    // NEXT_ID: 53
   }
 }
 
@@ -191,6 +197,12 @@ message FfiResponse {
 
     // Audio Filter Plugin
     LoadAudioFilterPluginResponse load_audio_filter_plugin = 48;
+
+    NewApmResponse new_apm = 49;
+    ApmProcessStreamResponse apm_process_stream = 50;
+    ApmProcessReverseStreamResponse apm_process_reverse_stream = 51;
+
+    // NEXT_ID: 52
   }
 }