Allow to use audio units other than Voice-Processing I/O unit

Author: akihikodaki

README.pixiv.md

@@ -13,6 +13,8 @@ This is a fork of WebRTC made by [pixiv Inc](https://www.pixiv.co.jp/).
   to retrieve the last frame at an arbitrary time is introduced.
 - APIs necessary to deliver recorded audio data on iOS's broadcast extension is
   added.
+- An audio unit component other than the Voice-Processing I/O unit can be used
+  to record audio on iOS.
 
 # Delivering audio data on iOS's broadcast extension
 

sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h

@@ -0,0 +1,23 @@
+/*
+ *  Copyright 2019 pixiv Inc. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a license that can be
+ *  found in the LICENSE.pixiv file in the root of the source tree.
+ */
+
+#import "RTCAudioDeviceModule.h"
+
+#if defined(WEBRTC_IOS)
+#include "sdk/objc/native/src/audio/audio_device_module_ios.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface RTCAudioDeviceModule ()
+
+@property(nonatomic, readonly) rtc::scoped_refptr<webrtc::ios_adm::AudioDeviceModuleIOS>
+    nativeModule;
+
+@end
+
+NS_ASSUME_NONNULL_END
+#endif

sdk/objc/api/peerconnection/RTCAudioDeviceModule.h

@@ -0,0 +1,24 @@
+/*
+ *  Copyright 2019 pixiv Inc. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a license that can be
+ *  found in the LICENSE.pixiv file in the root of the source tree.
+ */
+
+#import <CoreMedia/CoreMedia.h>
+#import <Foundation/Foundation.h>
+
+#import "RTCMacros.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+RTC_OBJC_EXPORT
+
+@interface RTCAudioDeviceModule : NSObject
+
+- (void)deliverRecordedData:(CMSampleBufferRef)sampleBuffer;
+@property(nonatomic, assign) OSType audioUnitSubType;
+
+@end
+
+NS_ASSUME_NONNULL_END

sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm

@@ -0,0 +1,57 @@
+/*
+ *  Copyright 2019 pixiv Inc. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a license that can be
+ *  found in the LICENSE.pixiv file in the root of the source tree.
+ */
+
+#include <AudioUnit/AudioUnit.h>
+
+#import "RTCAudioDeviceModule+Private.h"
+#include "rtc_base/ref_counted_object.h"
+
+@implementation RTCAudioDeviceModule {
+#if defined(WEBRTC_IOS)
+  rtc::scoped_refptr<webrtc::ios_adm::AudioDeviceModuleIOS> _nativeModule;
+#endif
+}
+
+- (instancetype)init {
+#if defined(WEBRTC_IOS)
+  self = [super init];
+  _nativeModule = new rtc::RefCountedObject<webrtc::ios_adm::AudioDeviceModuleIOS>();
+  return self;
+#else
+  return nullptr;
+#endif
+}
+
+- (void)deliverRecordedData:(CMSampleBufferRef)sampleBuffer {
+#if defined(WEBRTC_IOS)
+  _nativeModule->OnDeliverRecordedExternalData(sampleBuffer);
+#endif
+}
+
+- (void)setAudioUnitSubType:(OSType)audioUnitSubType {
+#if defined(WEBRTC_IOS)
+  _nativeModule->SetAudioUnitSubType(audioUnitSubType);
+#endif
+}
+
+- (OSType)audioUnitSubType {
+#if defined(WEBRTC_IOS)
+  return _nativeModule->GetAudioUnitSubType();
+#else
+  return kAudioUnitSubType_VoiceProcessingIO;
+#endif
+}
+
+#pragma mark - Private
+
+#if defined(WEBRTC_IOS)
+- (rtc::scoped_refptr<webrtc::ios_adm::AudioDeviceModuleIOS>)nativeModule {
+  return _nativeModule;
+}
+#endif
+
+@end

sdk/objc/native/src/audio/audio_device_ios.h

@@ -162,6 +162,8 @@ class AudioDeviceIOS : public AudioDeviceGeneric,
 
   bool IsInterrupted();
 
+  OSType audio_unit_sub_type;
+
  private:
   // Called by the relevant AudioSessionObserver methods on |thread_|.
   void HandleInterruptionBegin();

sdk/objc/native/src/audio/audio_device_ios.mm

@@ -781,7 +781,7 @@ static void LogDeviceInfo() {
   RTC_DCHECK(!audio_unit_);
 
   audio_unit_.reset(new VoiceProcessingAudioUnit(this));
-  if (!audio_unit_->Init()) {
+  if (!audio_unit_->Init(audio_unit_sub_type)) {
     audio_unit_.reset();
     return false;
   }

sdk/objc/native/src/audio/audio_device_module_ios.h

@@ -133,6 +133,8 @@ class AudioDeviceModuleIOS : public AudioDeviceModule {
 
 #if defined(WEBRTC_IOS)
   void OnDeliverRecordedExternalData(CMSampleBufferRef sample_buffer);
+  OSType GetAudioUnitSubType() const;
+  void SetAudioUnitSubType(OSType sub_type);
   int GetPlayoutAudioParameters(AudioParameters* params) const override;
   int GetRecordAudioParameters(AudioParameters* params) const override;
 #endif  // WEBRTC_IOS
@@ -141,6 +143,10 @@ class AudioDeviceModuleIOS : public AudioDeviceModule {
   const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
   std::unique_ptr<AudioDeviceIOS> audio_device_;
   std::unique_ptr<AudioDeviceBuffer> audio_device_buffer_;
+
+#if defined(WEBRTC_IOS)
+  OSType audio_unit_sub_type_ = kAudioUnitSubType_VoiceProcessingIO;
+#endif // WEBRTC_IOS
 };
 }  // namespace ios_adm
 }  // namespace webrtc

sdk/objc/native/src/audio/audio_device_module_ios.mm

@@ -75,6 +75,8 @@
     audio_device_.reset(new ios_adm::AudioDeviceIOS());
     RTC_CHECK(audio_device_);
 
+    audio_device_->audio_unit_sub_type = audio_unit_sub_type_;
+
     this->AttachAudioBuffer();
 
     AudioDeviceGeneric::InitStatus status = audio_device_->Init();
@@ -655,6 +657,18 @@
     audio_device_->OnDeliverRecordedExternalData(sample_buffer);
   }
 
+  OSType AudioDeviceModuleIOS::GetAudioUnitSubType() const {
+    return audio_unit_sub_type_;
+  }
+
+  void AudioDeviceModuleIOS::SetAudioUnitSubType(OSType sub_type) {
+    audio_unit_sub_type_ = sub_type;
+
+    if (audio_device_) {
+      audio_device_->audio_unit_sub_type = sub_type;
+    }
+  }
+
   int AudioDeviceModuleIOS::GetPlayoutAudioParameters(
       AudioParameters* params) const {
     RTC_LOG(INFO) << __FUNCTION__;

sdk/objc/native/src/audio/voice_processing_audio_unit.h

@@ -69,7 +69,7 @@ class VoiceProcessingAudioUnit {
   // audio. The selected stream format is selected to avoid internal resampling
   // and to match the 10ms callback rate for WebRTC as well as possible.
   // Does not intialize the audio unit.
-  bool Init();
+  bool Init(OSType audio_unit_sub_type);
 
   VoiceProcessingAudioUnit::State GetState() const;
 
@@ -132,6 +132,7 @@ class VoiceProcessingAudioUnit {
   VoiceProcessingAudioUnitObserver* observer_;
   AudioUnit vpio_unit_;
   VoiceProcessingAudioUnit::State state_;
+  OSType audio_unit_sub_type_;
 };
 }  // namespace ios_adm
 }  // namespace webrtc

sdk/objc/native/src/audio/voice_processing_audio_unit.mm

@@ -84,14 +84,16 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) {
 
 const UInt32 VoiceProcessingAudioUnit::kBytesPerSample = 2;
 
-bool VoiceProcessingAudioUnit::Init() {
+bool VoiceProcessingAudioUnit::Init(OSType audio_unit_sub_type) {
   RTC_DCHECK_EQ(state_, kInitRequired);
 
+  audio_unit_sub_type_ = audio_unit_sub_type;
+
   // Create an audio component description to identify the Voice Processing
   // I/O audio unit.
   AudioComponentDescription vpio_unit_description;
   vpio_unit_description.componentType = kAudioUnitType_Output;
-  vpio_unit_description.componentSubType = kAudioUnitSubType_VoiceProcessingIO;
+  vpio_unit_description.componentSubType = audio_unit_sub_type;
   vpio_unit_description.componentManufacturer = kAudioUnitManufacturer_Apple;
   vpio_unit_description.componentFlags = 0;
   vpio_unit_description.componentFlagsMask = 0;
@@ -250,62 +252,64 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) {
     RTCLog(@"Voice Processing I/O unit is now initialized.");
   }
 
-  // AGC should be enabled by default for Voice Processing I/O units but it is
-  // checked below and enabled explicitly if needed. This scheme is used
-  // to be absolutely sure that the AGC is enabled since we have seen cases
-  // where only zeros are recorded and a disabled AGC could be one of the
-  // reasons why it happens.
-  int agc_was_enabled_by_default = 0;
-  UInt32 agc_is_enabled = 0;
-  result = GetAGCState(vpio_unit_, &agc_is_enabled);
-  if (result != noErr) {
-    RTCLogError(@"Failed to get AGC state (1st attempt). "
-                 "Error=%ld.",
-                (long)result);
-    // Example of error code: kAudioUnitErr_NoConnection (-10876).
-    // All error codes related to audio units are negative and are therefore
-    // converted into a postive value to match the UMA APIs.
-    RTC_HISTOGRAM_COUNTS_SPARSE_100000(
-        "WebRTC.Audio.GetAGCStateErrorCode1", (-1) * result);
-  } else if (agc_is_enabled) {
-    // Remember that the AGC was enabled by default. Will be used in UMA.
-    agc_was_enabled_by_default = 1;
-  } else {
-    // AGC was initially disabled => try to enable it explicitly.
-    UInt32 enable_agc = 1;
-    result =
-        AudioUnitSetProperty(vpio_unit_,
-                             kAUVoiceIOProperty_VoiceProcessingEnableAGC,
-                             kAudioUnitScope_Global, kInputBus, &enable_agc,
-                             sizeof(enable_agc));
-    if (result != noErr) {
-      RTCLogError(@"Failed to enable the built-in AGC. "
-                   "Error=%ld.",
-                  (long)result);
-      RTC_HISTOGRAM_COUNTS_SPARSE_100000(
-          "WebRTC.Audio.SetAGCStateErrorCode", (-1) * result);
-    }
+  if (audio_unit_sub_type_ == kAudioUnitSubType_VoiceProcessingIO) {
+    // AGC should be enabled by default for Voice Processing I/O units but it is
+    // checked below and enabled explicitly if needed. This scheme is used
+    // to be absolutely sure that the AGC is enabled since we have seen cases
+    // where only zeros are recorded and a disabled AGC could be one of the
+    // reasons why it happens.
+    int agc_was_enabled_by_default = 0;
+    UInt32 agc_is_enabled = 0;
     result = GetAGCState(vpio_unit_, &agc_is_enabled);
     if (result != noErr) {
-      RTCLogError(@"Failed to get AGC state (2nd attempt). "
+      RTCLogError(@"Failed to get AGC state (1st attempt). "
                    "Error=%ld.",
                   (long)result);
+      // Example of error code: kAudioUnitErr_NoConnection (-10876).
+      // All error codes related to audio units are negative and are therefore
+      // converted into a postive value to match the UMA APIs.
       RTC_HISTOGRAM_COUNTS_SPARSE_100000(
-          "WebRTC.Audio.GetAGCStateErrorCode2", (-1) * result);
+          "WebRTC.Audio.GetAGCStateErrorCode1", (-1) * result);
+    } else if (agc_is_enabled) {
+      // Remember that the AGC was enabled by default. Will be used in UMA.
+      agc_was_enabled_by_default = 1;
+    } else {
+      // AGC was initially disabled => try to enable it explicitly.
+      UInt32 enable_agc = 1;
+      result =
+          AudioUnitSetProperty(vpio_unit_,
+                               kAUVoiceIOProperty_VoiceProcessingEnableAGC,
+                               kAudioUnitScope_Global, kInputBus, &enable_agc,
+                               sizeof(enable_agc));
+      if (result != noErr) {
+        RTCLogError(@"Failed to enable the built-in AGC. "
+                     "Error=%ld.",
+                    (long)result);
+        RTC_HISTOGRAM_COUNTS_SPARSE_100000(
+            "WebRTC.Audio.SetAGCStateErrorCode", (-1) * result);
+      }
+      result = GetAGCState(vpio_unit_, &agc_is_enabled);
+      if (result != noErr) {
+        RTCLogError(@"Failed to get AGC state (2nd attempt). "
+                     "Error=%ld.",
+                    (long)result);
+        RTC_HISTOGRAM_COUNTS_SPARSE_100000(
+            "WebRTC.Audio.GetAGCStateErrorCode2", (-1) * result);
+      }
     }
-  }
 
-  // Track if the built-in AGC was enabled by default (as it should) or not.
-  RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.BuiltInAGCWasEnabledByDefault",
-                        agc_was_enabled_by_default);
-  RTCLog(@"WebRTC.Audio.BuiltInAGCWasEnabledByDefault: %d",
-         agc_was_enabled_by_default);
-  // As a final step, add an UMA histogram for tracking the AGC state.
-  // At this stage, the AGC should be enabled, and if it is not, more work is
-  // needed to find out the root cause.
-  RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.BuiltInAGCIsEnabled", agc_is_enabled);
-  RTCLog(@"WebRTC.Audio.BuiltInAGCIsEnabled: %u",
-         static_cast<unsigned int>(agc_is_enabled));
+    // Track if the built-in AGC was enabled by default (as it should) or not.
+    RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.BuiltInAGCWasEnabledByDefault",
+                          agc_was_enabled_by_default);
+    RTCLog(@"WebRTC.Audio.BuiltInAGCWasEnabledByDefault: %d",
+           agc_was_enabled_by_default);
+    // As a final step, add an UMA histogram for tracking the AGC state.
+    // At this stage, the AGC should be enabled, and if it is not, more work is
+    // needed to find out the root cause.
+    RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.BuiltInAGCIsEnabled", agc_is_enabled);
+    RTCLog(@"WebRTC.Audio.BuiltInAGCIsEnabled: %u",
+           static_cast<unsigned int>(agc_is_enabled));
+  }
 
   state_ = kInitialized;
   return true;