Implement AudioConstraints.autoGainControl for native platforms (#157)

Additionally:
- fix audio preprocessing on native platforms

Co-authored-by: evdokimovs <[email protected]>

Author: rogurotus

crates/libwebrtc-sys/include/adm.h

@@ -75,7 +75,8 @@ class OpenALAudioDeviceModule : public ExtendedADM {
 
   static rtc::scoped_refptr<OpenALAudioDeviceModule> Create(
       AudioLayer audio_layer,
-      webrtc::TaskQueueFactory* task_queue_factory);
+      webrtc::TaskQueueFactory* task_queue_factory,
+      webrtc::AudioProcessing* audio_processing);
 
   // Main initialization and termination.
   int32_t Init() override;
@@ -216,6 +217,7 @@ class OpenALAudioDeviceModule : public ExtendedADM {
   void processRecordingQueued();
 
   std::unique_ptr<webrtc::AudioDeviceBuffer> audio_device_buffer_ = nullptr;
+  webrtc::AudioProcessing* audio_processing_;
 
   std::recursive_mutex _recording_mutex;
   bool _recordingInitialized = false;

crates/libwebrtc-sys/include/audio_device_recorder.h

@@ -29,7 +29,7 @@ constexpr auto kRecordingPart =
 // data to a `bridge::LocalAudioSource`.
 class AudioDeviceRecorder {
  public:
-  AudioDeviceRecorder(std::string deviceId);
+  AudioDeviceRecorder(std::string deviceId, webrtc::AudioProcessing* audio_processing);
 
   // Captures a new batch of audio samples and propagates it to the inner
   // `bridge::LocalAudioSource`.
@@ -53,6 +53,7 @@ class AudioDeviceRecorder {
   bool validateRecordingDeviceId();
 
   rtc::scoped_refptr<bridge::LocalAudioSource> _source;
+  webrtc::AudioProcessing* _audio_processing;
   ALCdevice* _device;
   std::string _deviceId;
   std::recursive_mutex _mutex;

crates/libwebrtc-sys/include/bridge.h

@@ -120,12 +120,15 @@ using RtcpFeedbackMessageType = webrtc::RtcpFeedbackMessageType;
 using ScalabilityMode = webrtc::ScalabilityMode;
 using RtpCapabilities = webrtc::RtpCapabilities;
 using RtpHeaderExtensionCapability = webrtc::RtpHeaderExtensionCapability;
+using RuntimeSetting = webrtc::AudioProcessing::RuntimeSetting;
+using AudioProcessingConfig = webrtc::AudioProcessing::Config;
 
 // Creates a new proxied `AudioDeviceModule` for the given `AudioLayer`.
 std::unique_ptr<AudioDeviceModule> create_audio_device_module(
     Thread& worker_thread,
     AudioLayer audio_layer,
-    TaskQueueFactory& task_queue_factory);
+    TaskQueueFactory& task_queue_factory,
+    const AudioProcessing& ap);
 
 // Initializes the native audio parts required for each platform.
 int32_t init_audio_device_module(const AudioDeviceModule& audio_device_module);
@@ -687,6 +690,23 @@ std::unique_ptr<webrtc::IceCandidateInterface> create_ice_candidate(
     rust::Str candidate,
     rust::String& error);
 
+// Creates a new `AudioProcessingConfig`.
+std::unique_ptr<AudioProcessingConfig> create_audio_processing_config();
+
+// Applies the provided  `AudioProcessingConfig` to the provided
+// `AudioProcessing`.
+void audio_processing_apply_config(const AudioProcessing& ap,
+                                   const AudioProcessingConfig& config);
+
+// Returns `AudioProcessingConfig` of the provided `AudioProcessing`.
+std::unique_ptr<AudioProcessingConfig> audio_processing_get_config(
+    const AudioProcessing& ap);
+
+// Enables/disables AGC (auto gain control) in the provided
+// `AudioProcessingConfig`.
+void config_gain_controller1_set_enabled(AudioProcessingConfig& config,
+                                         bool enabled);
+
 }  // namespace bridge
 
 #endif  // BRIDGE_H_

crates/libwebrtc-sys/include/local_audio_source.h

@@ -5,10 +5,12 @@
 #include <mutex>
 #include <optional>
 
+#include "api/audio/audio_frame.h"
 #include "api/audio_options.h"
 #include "api/media_stream_interface.h"
 #include "api/notifier.h"
 #include "api/scoped_refptr.h"
+#include "modules/audio_processing/include/audio_processing.h"
 #include "rust/cxx.h"
 
 namespace bridge {
@@ -21,7 +23,8 @@ class LocalAudioSource : public webrtc::Notifier<webrtc::AudioSourceInterface> {
  public:
   // Creates a new `LocalAudioSource`.
   static rtc::scoped_refptr<LocalAudioSource> Create(
-      cricket::AudioOptions audio_options);
+      cricket::AudioOptions audio_options,
+      std::optional<webrtc::AudioProcessing*> audio_processing);
 
   SourceState state() const override { return kLive; }
   bool remote() const override { return false; }
@@ -50,6 +53,8 @@ class LocalAudioSource : public webrtc::Notifier<webrtc::AudioSourceInterface> {
   cricket::AudioOptions _options;
   std::recursive_mutex sink_lock_;
   std::list<webrtc::AudioTrackSinkInterface*> sinks_;
+  webrtc::AudioFrame audio_frame_;
+  std::optional<webrtc::AudioProcessing*> audio_processing_;
 
   // Rust side callback that audio level changes are forwarded to.
   std::optional<rust::Box<bridge::DynAudioSourceOnAudioLevelChangeCallback>>

crates/libwebrtc-sys/src/bridge.rs

@@ -1488,6 +1488,7 @@ pub(crate) mod webrtc {
             worker_thread: Pin<&mut Thread>,
             audio_layer: AudioLayer,
             task_queue_factory: Pin<&mut TaskQueueFactory>,
+            ap: &AudioProcessing,
         ) -> UniquePtr<AudioDeviceModule>;
 
         /// Initializes the given [`AudioDeviceModule`].
@@ -1586,6 +1587,7 @@ pub(crate) mod webrtc {
 
     unsafe extern "C++" {
         pub type AudioProcessing;
+        pub type AudioProcessingConfig;
 
         /// Creates a new [`AudioProcessing`].
         pub fn create_audio_processing() -> UniquePtr<AudioProcessing>;
@@ -1597,6 +1599,30 @@ pub(crate) mod webrtc {
         /// will be muted or in some other way not used. This hints the
         /// underlying AGC, AEC, NS processors to halt.
         pub fn set_output_will_be_muted(ap: &AudioProcessing, muted: bool);
+
+        /// Creates a new [`AudioProcessingConfig`].
+        pub fn create_audio_processing_config(
+        ) -> UniquePtr<AudioProcessingConfig>;
+
+        /// Returns [`AudioProcessingConfig`] of the provided
+        /// [`AudioProcessing`].
+        pub fn audio_processing_get_config(
+            ap: &AudioProcessing,
+        ) -> UniquePtr<AudioProcessingConfig>;
+
+        /// Enables/disables AGC (auto gain control) in the provided
+        /// [`AudioProcessingConfig`].
+        pub fn config_gain_controller1_set_enabled(
+            config: Pin<&mut AudioProcessingConfig>,
+            enabled: bool,
+        );
+
+        /// Applies the provided [`AudioProcessingConfig`] to the provided
+        /// [`AudioProcessing`].
+        pub fn audio_processing_apply_config(
+            ap: &AudioProcessing,
+            config: &AudioProcessingConfig,
+        );
     }
 
     unsafe extern "C++" {

crates/libwebrtc-sys/src/cpp/adm.cc

@@ -164,9 +164,11 @@ int32_t OpenALAudioDeviceModule::ActiveAudioLayer(
 
 rtc::scoped_refptr<OpenALAudioDeviceModule> OpenALAudioDeviceModule::Create(
     AudioLayer audio_layer,
-    webrtc::TaskQueueFactory* task_queue_factory) {
+    webrtc::TaskQueueFactory* task_queue_factory,
+    webrtc::AudioProcessing* audio_processing) {
   auto adm = rtc::make_ref_counted<OpenALAudioDeviceModule>();
 
+  adm->audio_processing_ = audio_processing;
   adm->audio_device_buffer_ =
       std::make_unique<webrtc::AudioDeviceBuffer>(task_queue_factory);
 
@@ -711,17 +713,17 @@ rtc::scoped_refptr<bridge::LocalAudioSource>
 OpenALAudioDeviceModule::CreateAudioSource(uint32_t device_index) {
   std::lock_guard<std::recursive_mutex> lk(_recording_mutex);
 
-  std::string deviceId;
+  std::string device_id;
   const auto result = DeviceName(ALC_CAPTURE_DEVICE_SPECIFIER, device_index,
-                                 nullptr, &deviceId);
+                                 nullptr, &device_id);
   if (result != 0) {
     return nullptr;
   }
 
-  auto recorder = std::make_unique<AudioDeviceRecorder>(deviceId);
+  auto recorder = std::make_unique<AudioDeviceRecorder>(device_id, audio_processing_);
   recorder->StartCapture();
   auto source = recorder->GetSource();
-  _recorders[deviceId] = std::move(recorder);
+  _recorders[device_id] = std::move(recorder);
   ensureThreadStarted();
   startCaptureOnThread();
 

crates/libwebrtc-sys/src/cpp/audio_device_recorder.cc

@@ -34,11 +34,14 @@ bool CheckDeviceFailed(ALCdevice* device) {
 }
 }  // namespace recorder
 
-AudioDeviceRecorder::AudioDeviceRecorder(std::string deviceId) {
+AudioDeviceRecorder::AudioDeviceRecorder(
+    std::string deviceId,
+    webrtc::AudioProcessing* audio_processing) {
   _device = alcCaptureOpenDevice(deviceId.empty() ? nullptr : deviceId.c_str(),
                                  kRecordingFrequency, AL_FORMAT_MONO16,
                                  kRecordingFrequency);
-  _source = bridge::LocalAudioSource::Create(cricket::AudioOptions());
+  _source = bridge::LocalAudioSource::Create(cricket::AudioOptions(),
+                                             audio_processing);
   _deviceId = deviceId;
 }
 

crates/libwebrtc-sys/src/cpp/bridge.cc

@@ -114,10 +114,14 @@ std::unique_ptr<VideoTrackSourceInterface> create_device_video_source(
 std::unique_ptr<AudioDeviceModule> create_audio_device_module(
     Thread& worker_thread,
     AudioLayer audio_layer,
-    TaskQueueFactory& task_queue_factory) {
+    TaskQueueFactory& task_queue_factory,
+    const AudioProcessing& ap) {
   AudioDeviceModule adm = worker_thread.BlockingCall([audio_layer,
-                                                      &task_queue_factory] {
-    return ::OpenALAudioDeviceModule::Create(audio_layer, &task_queue_factory);
+                                                      &task_queue_factory,
+                                                      &ap] {
+    return ::OpenALAudioDeviceModule::Create(audio_layer,
+                                             &task_queue_factory,
+                                             ap.get());
   });
 
   if (adm == nullptr) {
@@ -249,9 +253,14 @@ int32_t set_audio_playout_device(const AudioDeviceModule& audio_device_module,
 
 // Calls `AudioProcessingBuilder().Create()`.
 std::unique_ptr<AudioProcessing> create_audio_processing() {
-  auto ap = webrtc::AudioProcessingBuilder().Create();
+  webrtc::AudioProcessing::Config apm_config;
+  apm_config.echo_canceller.enabled = true;
+  apm_config.echo_canceller.mobile_mode = false;
+  apm_config.gain_controller1.enabled = true;
+  apm_config.gain_controller1.enable_limiter = true;
 
-  return std::make_unique<AudioProcessing>(ap);
+  auto apm = webrtc::AudioProcessingBuilder().SetConfig(apm_config).Create();
+  return std::make_unique<AudioProcessing>(apm);
 }
 
 // Calls `AudioProcessing->set_output_will_be_muted()`.
@@ -346,7 +355,7 @@ void dispose_audio_source(const AudioDeviceModule& audio_device_module,
 // Creates a new fake `AudioSource`.
 std::unique_ptr<AudioSourceInterface> create_fake_audio_source() {
   return std::make_unique<AudioSourceInterface>(
-      bridge::LocalAudioSource::Create(cricket::AudioOptions()));
+      bridge::LocalAudioSource::Create(cricket::AudioOptions(), nullptr));
 }
 
 // Calls `PeerConnectionFactoryInterface->CreateVideoTrack`.
@@ -1161,4 +1170,37 @@ std::unique_ptr<std::string> display_source_title(const DisplaySource& source) {
   return std::make_unique<std::string>(source.title);
 }
 
+// Creates a new `AudioProcessingConfig`.
+std::unique_ptr<AudioProcessingConfig> create_audio_processing_config() {
+  webrtc::AudioProcessing::Config apm_config;
+  apm_config.echo_canceller.enabled = true;
+  apm_config.echo_canceller.mobile_mode = false;
+  apm_config.gain_controller1.enabled = true;
+  apm_config.gain_controller1.mode ==
+      webrtc::AudioProcessing::Config::GainController1::kAdaptiveDigital;
+  apm_config.gain_controller1.enable_limiter = true;
+  return std::make_unique<AudioProcessingConfig>(apm_config);
+}
+
+// Enables/disables AGC (auto gain control) in the provided
+// `AudioProcessingConfig`.
+void config_gain_controller1_set_enabled(AudioProcessingConfig& config,
+                                         bool enabled) {
+  config.gain_controller1.enabled = enabled;
+  config.gain_controller1.analog_gain_controller.enabled = enabled;
+}
+
+// Returns `AudioProcessingConfig` of the provided `AudioProcessing`.
+std::unique_ptr<AudioProcessingConfig> audio_processing_get_config(
+    const AudioProcessing& ap) {
+  return std::make_unique<AudioProcessingConfig>(ap->GetConfig());
+}
+
+// Applies the provided  `AudioProcessingConfig` to the provided
+// `AudioProcessing`.
+void audio_processing_apply_config(const AudioProcessing& ap,
+                                   const AudioProcessingConfig& config) {
+  ap->ApplyConfig(config);
+}
+
 }  // namespace bridge

crates/libwebrtc-sys/src/cpp/local_audio_source.cc

@@ -14,8 +14,10 @@ float calculate_audio_level(int16_t* data, int size) {
 }
 
 rtc::scoped_refptr<LocalAudioSource> LocalAudioSource::Create(
-    cricket::AudioOptions audio_options) {
+    cricket::AudioOptions audio_options,
+    std::optional<webrtc::AudioProcessing*> audio_processing) {
   auto source = rtc::make_ref_counted<LocalAudioSource>();
+  source->audio_processing_ = audio_processing;
   source->_options = audio_options;
   return source;
 }
@@ -54,8 +56,21 @@ void LocalAudioSource::OnData(const void* audio_data,
   }
 
   for (auto* sink : sinks_) {
-    sink->OnData(audio_data, bits_per_sample, sample_rate, number_of_channels,
-                 number_of_frames);
+    audio_frame_.UpdateFrame(
+        0, (const int16_t*)audio_data, sample_rate / 100, sample_rate,
+        webrtc::AudioFrame::SpeechType::kNormalSpeech,
+        webrtc::AudioFrame::VADActivity::kVadUnknown, number_of_channels);
+    if (*audio_processing_) {
+      webrtc::StreamConfig input_config(sample_rate, number_of_channels);
+      webrtc::StreamConfig output_config(sample_rate, number_of_channels);
+
+      int result =
+          (*audio_processing_)
+              ->ProcessStream(audio_frame_.data(), input_config, output_config,
+                              audio_frame_.mutable_data());
+    }
+    sink->OnData(audio_frame_.data(), bits_per_sample, sample_rate,
+                 number_of_channels, number_of_frames);
   }
 }
 

crates/libwebrtc-sys/src/lib.rs

@@ -217,11 +217,15 @@ impl AudioDeviceModule {
         worker_thread: &mut Thread,
         audio_layer: AudioLayer,
         task_queue_factory: &mut TaskQueueFactory,
+        audio_processing: Option<&AudioProcessing>,
     ) -> anyhow::Result<Self> {
         let ptr = webrtc::create_audio_device_module(
             worker_thread.0.pin_mut(),
             audio_layer,
             task_queue_factory.0.pin_mut(),
+            &audio_processing
+                .unwrap_or(&AudioProcessing(UniquePtr::null()))
+                .0,
         );
 
         if ptr.is_null() {
@@ -501,6 +505,23 @@ unsafe impl Sync for webrtc::AudioDeviceModule {}
 /// voice processing components designed for real-time communications software.
 pub struct AudioProcessing(UniquePtr<webrtc::AudioProcessing>);
 
+/// Representation of an audio processing config.
+pub struct AudioProcessingConfig(UniquePtr<webrtc::AudioProcessingConfig>);
+
+impl Default for AudioProcessingConfig {
+    fn default() -> Self {
+        Self(webrtc::create_audio_processing_config())
+    }
+}
+
+impl AudioProcessingConfig {
+    /// Enables/disables AGC (auto gain control) in this
+    /// [`AudioProcessingConfig`].
+    pub fn set_gain_controller_enabled(&mut self, enabled: bool) {
+        webrtc::config_gain_controller1_set_enabled(self.0.pin_mut(), enabled);
+    }
+}
+
 impl AudioProcessing {
     /// Creates a new [`AudioProcessing`].
     pub fn new() -> anyhow::Result<Self> {
@@ -521,6 +542,18 @@ impl AudioProcessing {
     pub fn set_output_will_be_muted(&self, muted: bool) {
         webrtc::set_output_will_be_muted(&self.0, muted);
     }
+
+    /// Returns [`AudioProcessingConfig`] of this [`AudioProcessing`].
+    #[must_use]
+    pub fn config(&self) -> AudioProcessingConfig {
+        AudioProcessingConfig(webrtc::audio_processing_get_config(&self.0))
+    }
+
+    /// Applies the provided [`AudioProcessingConfig`] to this
+    /// [`AudioProcessing`].
+    pub fn apply_config(&self, config: &AudioProcessingConfig) {
+        webrtc::audio_processing_apply_config(&self.0, &config.0);
+    }
 }
 
 unsafe impl Send for webrtc::AudioProcessing {}