Merge pull request #10 from GetStream/feature/noise-cancellation-support

Internal webRTC builds used + noise cancellation support on Android

Author: Brazol

android/build.gradle

@@ -2,7 +2,7 @@ group 'io.getstream.webrtc.flutter'
 version '1.0-SNAPSHOT'
 
 buildscript {
-    ext.kotlin_version = '1.7.10'
+    ext.kotlin_version = '1.9.10'
     repositories {
         google()
         mavenCentral()
@@ -52,7 +52,7 @@ android {
 }
 
 dependencies {
-    implementation 'io.github.webrtc-sdk:android:125.6422.03'
+    implementation 'io.getstream:stream-webrtc-android:1.3.8'
     implementation 'com.github.davidliu:audioswitch:89582c47c9a04c62f90aa5e57251af4800a62c9a'
     implementation 'androidx.annotation:annotation:1.1.0'
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"

android/src/main/java/io/getstream/webrtc/flutter/FlutterWebRTCPlugin.java

@@ -11,6 +11,7 @@
 import androidx.lifecycle.Lifecycle;
 import androidx.lifecycle.LifecycleOwner;
 
+import io.getstream.webrtc.flutter.audio.AudioProcessingFactoryProvider;
 import io.getstream.webrtc.flutter.audio.AudioProcessingController;
 import io.getstream.webrtc.flutter.audio.AudioSwitchManager;
 import io.getstream.webrtc.flutter.utils.AnyThreadSink;
@@ -49,8 +50,12 @@ public FlutterWebRTCPlugin() {
 
     public static FlutterWebRTCPlugin sharedSingleton;
 
-    public AudioProcessingController getAudioProcessingController() {
-        return methodCallHandler.audioProcessingController;
+    public void setAudioProcessingFactoryProvider(AudioProcessingFactoryProvider provider) {
+        methodCallHandler.audioProcessingFactoryProvider = provider;
+    }
+
+    public AudioProcessingFactoryProvider getAudioProcessingFactoryProvider() {
+        return methodCallHandler.audioProcessingFactoryProvider;
     }
 
     public MediaStreamTrack getTrackForId(String trackId, String peerConnectionId) {

android/src/main/java/io/getstream/webrtc/flutter/MethodCallHandlerImpl.java

@@ -20,11 +20,12 @@
 import androidx.annotation.RequiresApi;
 
 import io.getstream.webrtc.flutter.audio.AudioDeviceKind;
+import io.getstream.webrtc.flutter.audio.AudioProcessingFactoryProvider;
 import io.getstream.webrtc.flutter.audio.AudioProcessingController;
 import io.getstream.webrtc.flutter.audio.AudioSwitchManager;
 import io.getstream.webrtc.flutter.audio.AudioUtils;
 import io.getstream.webrtc.flutter.audio.LocalAudioTrack;
-import io.getstream.webrtc.flutter.audio.PlaybackSamplesReadyCallbackAdapter;
+// import io.getstream.webrtc.flutter.audio.PlaybackSamplesReadyCallbackAdapter;
 import io.getstream.webrtc.flutter.audio.RecordSamplesReadyCallbackAdapter;
 import io.getstream.webrtc.flutter.record.AudioChannel;
 import io.getstream.webrtc.flutter.record.FrameCapturer;
@@ -110,7 +111,7 @@ public class MethodCallHandlerImpl implements MethodCallHandler, StateProvider {
 
   public RecordSamplesReadyCallbackAdapter recordSamplesReadyCallbackAdapter;
 
-  public PlaybackSamplesReadyCallbackAdapter playbackSamplesReadyCallbackAdapter;
+  // public PlaybackSamplesReadyCallbackAdapter playbackSamplesReadyCallbackAdapter;
 
   /**
    * The implementation of {@code getUserMedia} extracted into a separate file in order to reduce
@@ -130,7 +131,7 @@ public class MethodCallHandlerImpl implements MethodCallHandler, StateProvider {
 
   private CustomVideoDecoderFactory videoDecoderFactory;
 
-  public AudioProcessingController audioProcessingController;
+  public AudioProcessingFactoryProvider audioProcessingFactoryProvider;
 
   MethodCallHandlerImpl(Context context, BinaryMessenger messenger, TextureRegistry textureRegistry) {
     this.context = context;
@@ -198,7 +199,7 @@ private void initialize(boolean bypassVoiceProcessing, int networkIgnoreMask, bo
     JavaAudioDeviceModule.Builder audioDeviceModuleBuilder = JavaAudioDeviceModule.builder(context);
 
     recordSamplesReadyCallbackAdapter = new RecordSamplesReadyCallbackAdapter();
-    playbackSamplesReadyCallbackAdapter = new PlaybackSamplesReadyCallbackAdapter();
+    // playbackSamplesReadyCallbackAdapter = new PlaybackSamplesReadyCallbackAdapter();
 
     if(bypassVoiceProcessing) {
       audioDeviceModuleBuilder.setUseHardwareAcousticEchoCanceler(false)
@@ -215,7 +216,7 @@ private void initialize(boolean bypassVoiceProcessing, int networkIgnoreMask, bo
     }
 
     audioDeviceModuleBuilder.setSamplesReadyCallback(recordSamplesReadyCallbackAdapter);
-    audioDeviceModuleBuilder.setPlaybackSamplesReadyCallback(playbackSamplesReadyCallbackAdapter);
+    // audioDeviceModuleBuilder.setPlaybackSamplesReadyCallback(playbackSamplesReadyCallbackAdapter);
 
     recordSamplesReadyCallbackAdapter.addCallback(getUserMediaImpl.inputSamplesInterceptor);
 
@@ -266,9 +267,12 @@ public void onWebRtcAudioRecordSamplesReady(JavaAudioDeviceModule.AudioSamples a
     videoEncoderFactory.setForceSWCodec(forceSWCodec);
     videoEncoderFactory.setForceSWCodecList(forceSWCodecList);
 
-    audioProcessingController = new AudioProcessingController();
 
-    factoryBuilder.setAudioProcessingFactory(audioProcessingController.externalAudioProcessingFactory);
+    if(audioProcessingFactoryProvider == null) {
+        audioProcessingFactoryProvider = new AudioProcessingController();
+    }
+
+    factoryBuilder.setAudioProcessingFactory(audioProcessingFactoryProvider.getFactory());
 
     mFactory = factoryBuilder
             .setAudioDeviceModule(audioDeviceModule)

android/src/main/java/io/getstream/webrtc/flutter/audio/AudioProcessingController.java

@@ -1,8 +1,10 @@
 package io.getstream.webrtc.flutter.audio;
 
+import org.webrtc.AudioProcessingFactory;
 import org.webrtc.ExternalAudioProcessingFactory;
+import io.getstream.webrtc.flutter.audio.AudioProcessingFactoryProvider;
 
-public class AudioProcessingController {
+public class AudioProcessingController implements AudioProcessingFactoryProvider {
     /**
      * This is the audio processing module that will be applied to the audio stream after it is captured from the microphone.
      * This is useful for adding echo cancellation, noise suppression, etc.
@@ -20,5 +22,9 @@ public AudioProcessingController() {
         this.externalAudioProcessingFactory.setCapturePostProcessing(capturePostProcessing);
         this.externalAudioProcessingFactory.setRenderPreProcessing(renderPreProcessing);
     }
-    
-}
+
+    @Override
+    public AudioProcessingFactory getFactory() {
+        return this.externalAudioProcessingFactory;
+    }
+}

android/src/main/java/io/getstream/webrtc/flutter/audio/AudioProcessingFactoryProvider.java

@@ -0,0 +1,8 @@
+package io.getstream.webrtc.flutter.audio;
+
+import org.webrtc.AudioProcessingFactory;
+
+// Define the common interface
+public interface AudioProcessingFactoryProvider {
+    AudioProcessingFactory getFactory();
+}

android/src/main/java/io/getstream/webrtc/flutter/audio/PlaybackSamplesReadyCallbackAdapter.java

@@ -1,32 +1,32 @@
-package io.getstream.webrtc.flutter.audio;
+// package io.getstream.webrtc.flutter.audio;
 
-import org.webrtc.audio.JavaAudioDeviceModule;
+// import org.webrtc.audio.JavaAudioDeviceModule;
 
-import java.util.ArrayList;
-import java.util.List;
+// import java.util.ArrayList;
+// import java.util.List;
 
-public class PlaybackSamplesReadyCallbackAdapter
-        implements JavaAudioDeviceModule.PlaybackSamplesReadyCallback {
-    public PlaybackSamplesReadyCallbackAdapter() {}
+// public class PlaybackSamplesReadyCallbackAdapter
+//         implements JavaAudioDeviceModule.PlaybackSamplesReadyCallback {
+//     public PlaybackSamplesReadyCallbackAdapter() {}
 
-    List<JavaAudioDeviceModule.PlaybackSamplesReadyCallback> callbacks = new ArrayList<>();
+//     List<JavaAudioDeviceModule.PlaybackSamplesReadyCallback> callbacks = new ArrayList<>();
 
-    public void addCallback(JavaAudioDeviceModule.PlaybackSamplesReadyCallback callback) {
-        synchronized (callbacks) {
-            callbacks.add(callback);
-        }
-    }
+//     public void addCallback(JavaAudioDeviceModule.PlaybackSamplesReadyCallback callback) {
+//         synchronized (callbacks) {
+//             callbacks.add(callback);
+//         }
+//     }
 
-    public void removeCallback(JavaAudioDeviceModule.PlaybackSamplesReadyCallback callback) {
-        synchronized (callbacks) {
-            callbacks.remove(callback);
-        }
-    }
+//     public void removeCallback(JavaAudioDeviceModule.PlaybackSamplesReadyCallback callback) {
+//         synchronized (callbacks) {
+//             callbacks.remove(callback);
+//         }
+//     }
 
-    @Override
-    public void onWebRtcAudioTrackSamplesReady(JavaAudioDeviceModule.AudioSamples audioSamples) {
-        for (JavaAudioDeviceModule.PlaybackSamplesReadyCallback callback : callbacks) {
-            callback.onWebRtcAudioTrackSamplesReady(audioSamples);
-        }
-    }
-}
+//     @Override
+//     public void onWebRtcAudioTrackSamplesReady(JavaAudioDeviceModule.AudioSamples audioSamples) {
+//         for (JavaAudioDeviceModule.PlaybackSamplesReadyCallback callback : callbacks) {
+//             callback.onWebRtcAudioTrackSamplesReady(audioSamples);
+//         }
+//     }
+// }

common/darwin/Classes/FlutterRTCMediaStream.h

@@ -7,7 +7,7 @@
 
 @interface FlutterWebRTCPlugin (RTCMediaStream)
 
-- (RTCVideoTrack*)cloneTrack:(nonnull NSString*)trackId;
+- (RTCVideoTrack* _Nullable)cloneTrack:(nonnull NSString*)trackId;
 
 - (void)getUserMedia:(nonnull NSDictionary*)constraints result:(nonnull FlutterResult)result;
 

common/darwin/Classes/FlutterRTCMediaStream.m

@@ -341,9 +341,10 @@ - (RTCMediaStreamTrack*)cloneTrack:(nonnull NSString*)trackId {
     LocalVideoTrack* originalLocalTrack = self.localTracks[trackId];
 
     if (originalTrack != nil && [originalTrack.kind isEqualToString:@"audio"]) {
-      RTCAudioTrack *originalAudioTrack = (RTCAudioTrack *)originalTrack;
+      RTCAudioTrack* originalAudioTrack = (RTCAudioTrack *)originalTrack;
+      RTCAudioSource* originalAudioSource = originalAudioTrack.source;
 
-      RTCAudioTrack* audioTrack = [self.peerConnectionFactory audioTrackWithTrackId:trackId];
+      RTCAudioTrack* audioTrack = [self.peerConnectionFactory audioTrackWithSource:originalAudioSource trackId:newTrackId];
       LocalAudioTrack *localAudioTrack = [[LocalAudioTrack alloc] initWithTrack:audioTrack];
 
       audioTrack.settings = originalAudioTrack.settings;

common/darwin/Classes/FlutterWebRTCPlugin.h

@@ -49,7 +49,7 @@ typedef void (^CapturerStopHandler)(CompletionHandler _Nonnull handler);
 @property(nonatomic, strong) RTCCameraVideoCapturer* _Nullable videoCapturer;
 @property(nonatomic, strong) FlutterRTCFrameCapturer* _Nullable frameCapturer;
 @property(nonatomic, strong) AVAudioSessionPort _Nullable preferredInput;
-@property (nonatomic, strong) VideoEffectProcessor* videoEffectProcessor;
+@property (nonatomic, strong) VideoEffectProcessor* _Nullable videoEffectProcessor;
 
 @property(nonatomic, strong) NSString * _Nonnull focusMode;
 @property(nonatomic, strong) NSString * _Nonnull exposureMode;

example/android/settings.gradle

@@ -19,7 +19,7 @@ pluginManagement {
 plugins {
     id "dev.flutter.flutter-plugin-loader" version "1.0.0"
     id "com.android.application" version "8.3.0" apply false
-    id "org.jetbrains.kotlin.android" version "1.7.10" apply false
+    id "org.jetbrains.kotlin.android" version "1.9.10" apply false
 }
 
 include ":app"