feat: preconnect audio buffer (#256)

davidliu · web-flow · commit 917defa8a319 · 2025-07-29T19:59:36.000+09:00
* feat: preconnect audio buffer

* chore: lint

* fix: 0.80 react-native compatibility

* chore: update livekit-client peer dependency to 2.15.4

* chore: cleanup export
diff --git a/android/src/main/java/com/livekit/reactnative/LivekitReactNativeModule.kt b/android/src/main/java/com/livekit/reactnative/LivekitReactNativeModule.kt
@@ -8,14 +8,19 @@ import com.facebook.react.bridge.ReactApplicationContext
 import com.facebook.react.bridge.ReactContextBaseJavaModule
 import com.facebook.react.bridge.ReactMethod
 import com.facebook.react.bridge.ReadableMap
+import com.facebook.react.modules.core.DeviceEventManagerModule
 import com.livekit.reactnative.audio.AudioDeviceKind
 import com.livekit.reactnative.audio.AudioManagerUtils
 import com.livekit.reactnative.audio.AudioSwitchManager
+import com.livekit.reactnative.audio.events.Events
 import com.livekit.reactnative.audio.processing.AudioSinkManager
+import com.livekit.reactnative.audio.processing.AudioSinkProcessor
 import com.livekit.reactnative.audio.processing.MultibandVolumeProcessor
 import com.livekit.reactnative.audio.processing.VolumeProcessor
 import com.oney.WebRTCModule.WebRTCModuleOptions
 import org.webrtc.audio.WebRtcAudioTrackHelper
+import java.lang.Thread.sleep
+import kotlin.concurrent.thread
 import kotlin.time.Duration.Companion.milliseconds
 
 // NOTE: As of 0.80 react-native new architecture requires all
@@ -134,6 +139,23 @@ class LivekitReactNativeModule(reactContext: ReactApplicationContext) : ReactCon
         promise.resolve(null)
     }
 
+    @ReactMethod(isBlockingSynchronousMethod = true)
+    fun createAudioSinkListener(pcId: Int, trackId: String): String {
+        val processor = AudioSinkProcessor(reactApplicationContext)
+        val reactTag = audioSinkManager.registerSink(processor)
+        audioSinkManager.attachSinkToTrack(processor, pcId, trackId)
+        processor.reactTag = reactTag
+
+        return reactTag
+    }
+
+    @ReactMethod(isBlockingSynchronousMethod = true)
+    fun deleteAudioSinkListener(reactTag: String, pcId: Int, trackId: String): Boolean {
+        audioSinkManager.detachSinkFromTrack(reactTag, pcId, trackId)
+        audioSinkManager.unregisterSink(reactTag)
+        return true
+    }
+
     @ReactMethod(isBlockingSynchronousMethod = true)
     fun createVolumeProcessor(pcId: Int, trackId: String): String {
         val processor = VolumeProcessor(reactApplicationContext)
diff --git a/android/src/main/java/com/livekit/reactnative/audio/events/Events.kt b/android/src/main/java/com/livekit/reactnative/audio/events/Events.kt
@@ -3,4 +3,5 @@ package com.livekit.reactnative.audio.events
 enum class Events {
     LK_VOLUME_PROCESSED,
     LK_MULTIBAND_PROCESSED,
+    LK_AUDIO_DATA,
 }
diff --git a/android/src/main/java/com/livekit/reactnative/audio/processing/AudioSinkProcessor.kt b/android/src/main/java/com/livekit/reactnative/audio/processing/AudioSinkProcessor.kt
@@ -0,0 +1,57 @@
+package com.livekit.reactnative.audio.processing
+
+import com.facebook.react.bridge.Arguments
+import com.facebook.react.bridge.ReactContext
+import com.facebook.react.modules.core.DeviceEventManagerModule
+import com.livekit.reactnative.audio.events.Events
+import org.webrtc.AudioTrackSink
+import java.nio.ByteBuffer
+import java.util.Arrays
+import kotlin.io.encoding.Base64
+import kotlin.io.encoding.ExperimentalEncodingApi
+
+class AudioSinkProcessor(private val reactContext: ReactContext) : BaseAudioSinkProcessor() {
+    var reactTag: String? = null
+
+    @OptIn(ExperimentalEncodingApi::class)
+    override fun onAudioData(byteArray: ByteArray) {
+        val reactTag = this.reactTag ?: return
+
+        val encodedString = Base64.encode(byteArray)
+        val event = Arguments.createMap().apply {
+            putString("data", encodedString)
+            putString("id", reactTag)
+        }
+        reactContext.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter::class.java)
+            .emit(Events.LK_AUDIO_DATA.name, event)
+    }
+}
+
+abstract class BaseAudioSinkProcessor : AudioTrackSink {
+    abstract fun onAudioData(byteArray: ByteArray)
+
+    override fun onData(
+        audioData: ByteBuffer,
+        bitsPerSample: Int,
+        sampleRate: Int,
+        numberOfChannels: Int,
+        numberOfFrames: Int,
+        absoluteCaptureTimestampMs: Long
+    ) {
+        val byteArray: ByteArray
+
+        if (audioData.hasArray()) {
+            val audioArray = audioData.array()
+            byteArray = Arrays.copyOfRange(audioArray, audioData.arrayOffset(), audioArray.size)
+        } else {
+            audioData.mark()
+            audioData.position(0)
+
+            byteArray = ByteArray(audioData.remaining())
+            audioData.get(byteArray)
+            audioData.reset()
+        }
+
+        onAudioData(byteArray)
+    }
+}
diff --git a/ci/package.json b/ci/package.json
@@ -13,7 +13,7 @@
   "dependencies": {
     "@livekit/react-native": "*",
     "@livekit/react-native-webrtc": "^125.0.12",
-    "livekit-client": "^2.9.8",
+    "livekit-client": "^2.15.4",
     "react": "18.2.0",
     "react-native": "0.74.2"
   },
diff --git a/ci/yarn.lock b/ci/yarn.lock
@@ -2165,12 +2165,12 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@livekit/protocol@npm:1.34.0":
-  version: 1.34.0
-  resolution: "@livekit/protocol@npm:1.34.0"
+"@livekit/protocol@npm:1.39.3":
+  version: 1.39.3
+  resolution: "@livekit/protocol@npm:1.39.3"
   dependencies:
     "@bufbuild/protobuf": ^1.10.0
-  checksum: e08d0ceb20e5434b352e0c42f89d13f29edbf853a968924aea9792660d79edc13a535288d05a47da97843e1e3a66917adb79ee51b8652442d42123e5f75f649c
+  checksum: 72aa3b9e82203ccec039047518d841d65b10dcc11ab9d661f269eab8a8672aed7b2d072f4f09cc6fb94a2579542b0a9216d7f66476d34cd1675c13e1ebcc119f
   languageName: node
   linkType: hard
 
@@ -3883,7 +3883,7 @@ __metadata:
     eslint: ^8.19.0
     install-local: ^3.0.1
     jest: ^29.6.3
-    livekit-client: ^2.9.8
+    livekit-client: ^2.15.4
     prettier: 2.8.8
     react: 18.2.0
     react-native: 0.74.2
@@ -6956,20 +6956,22 @@ __metadata:
   languageName: node
   linkType: hard
 
-"livekit-client@npm:^2.9.8":
-  version: 2.9.8
-  resolution: "livekit-client@npm:2.9.8"
+"livekit-client@npm:^2.15.4":
+  version: 2.15.4
+  resolution: "livekit-client@npm:2.15.4"
   dependencies:
     "@livekit/mutex": 1.1.1
-    "@livekit/protocol": 1.34.0
+    "@livekit/protocol": 1.39.3
     events: ^3.3.0
     loglevel: ^1.9.2
     sdp-transform: ^2.15.0
     ts-debounce: ^4.0.0
     tslib: 2.8.1
     typed-emitter: ^2.1.0
     webrtc-adapter: ^9.0.1
-  checksum: cc01e43fbf5fa298c6696a3a084852d04cd818c554e9b3b9469ad22e85b8d29a975d96072d8432aff79651337f2950881c266117d0490c8ab94112dfda789ca4
+  peerDependencies:
+    "@types/dom-mediacapture-record": ^1
+  checksum: 7ae2e2f5326c8f7925b359277fd12ddeefb0bc5069b0563a9e184f2b860351bf91733c1fa5fd7b65467b643776368058d485af888e2771fd68c61c409b950e05
   languageName: node
   linkType: hard
 
diff --git a/example/package.json b/example/package.json
@@ -15,7 +15,7 @@
     "@react-navigation/native": "^6.0.8",
     "@react-navigation/native-stack": "^6.5.0",
     "@supersami/rn-foreground-service": "^2.2.5",
-    "livekit-client": "^2.9.8",
+    "livekit-client": "^2.15.4",
     "react": "18.2.0",
     "react-native": "0.74.2",
     "react-native-callkeep": "^4.3.14",
diff --git a/example/yarn.lock b/example/yarn.lock
@@ -2163,12 +2163,12 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@livekit/protocol@npm:1.39.1":
-  version: 1.39.1
-  resolution: "@livekit/protocol@npm:1.39.1"
+"@livekit/protocol@npm:1.39.3":
+  version: 1.39.3
+  resolution: "@livekit/protocol@npm:1.39.3"
   dependencies:
     "@bufbuild/protobuf": ^1.10.0
-  checksum: 2ca25994ac28c05c618272029f4af82197897545096402548b5ae75b2bd97de993d345b0cca902d2b0fb4f0110602c667844a7a540824d6a12b726f6f495b988
+  checksum: 72aa3b9e82203ccec039047518d841d65b10dcc11ab9d661f269eab8a8672aed7b2d072f4f09cc6fb94a2579542b0a9216d7f66476d34cd1675c13e1ebcc119f
   languageName: node
   linkType: hard
 
@@ -6102,12 +6102,12 @@ __metadata:
   languageName: node
   linkType: hard
 
-"livekit-client@npm:^2.9.8":
-  version: 2.13.5
-  resolution: "livekit-client@npm:2.13.5"
+"livekit-client@npm:^2.15.4":
+  version: 2.15.4
+  resolution: "livekit-client@npm:2.15.4"
   dependencies:
     "@livekit/mutex": 1.1.1
-    "@livekit/protocol": 1.39.1
+    "@livekit/protocol": 1.39.3
     events: ^3.3.0
     loglevel: ^1.9.2
     sdp-transform: ^2.15.0
@@ -6117,7 +6117,7 @@ __metadata:
     webrtc-adapter: ^9.0.1
   peerDependencies:
     "@types/dom-mediacapture-record": ^1
-  checksum: aaff2e0729a7dd51b8641153043ab77825614c12af80662c4f1b98b713af7b7d2565210eed1a56c9e068c45afe4b03bb60a49de02117769868b516a9d1a3a1de
+  checksum: 7ae2e2f5326c8f7925b359277fd12ddeefb0bc5069b0563a9e184f2b860351bf91733c1fa5fd7b65467b643776368058d485af888e2771fd68c61c409b950e05
   languageName: node
   linkType: hard
 
@@ -6140,7 +6140,7 @@ __metadata:
     "@types/fastestsmallesttextencoderdecoder": ^1.0.0
     "@types/react": ^18.2.6
     babel-plugin-module-resolver: ^4.1.0
-    livekit-client: ^2.9.8
+    livekit-client: ^2.15.4
     patch-package: ^8.0.0
     react: 18.2.0
     react-native: 0.74.2
diff --git a/ios/LiveKitReactNativeModule.swift b/ios/LiveKitReactNativeModule.swift
@@ -6,6 +6,7 @@ import React
 struct LKEvents {
     static let kEventVolumeProcessed = "LK_VOLUME_PROCESSED";
     static let kEventMultibandProcessed = "LK_MULTIBAND_PROCESSED";
+    static let kEventAudioData = "LK_AUDIO_DATA";
 }
 
 @objc(LivekitReactNativeModule)
@@ -178,6 +179,24 @@ public class LivekitReactNativeModule: RCTEventEmitter {
         session.unlockForConfiguration()
     }
     
+    @objc(createAudioSinkListener:trackId:)
+    public func createAudioSinkListener(_ pcId: NSNumber, trackId: String) -> String {
+        let renderer = AudioSinkRenderer(eventEmitter: self)
+        let reactTag = self.audioRendererManager.registerRenderer(renderer)
+        renderer.reactTag = reactTag
+        self.audioRendererManager.attach(renderer: renderer, pcId: pcId, trackId: trackId)
+        
+        return reactTag
+    }
+
+    @objc(deleteAudioSinkListener:pcId:trackId:)
+    public func deleteAudioSinkListener(_ reactTag: String, pcId: NSNumber, trackId: String) -> Any? {
+        self.audioRendererManager.detach(rendererByTag: reactTag, pcId: pcId, trackId: trackId)
+        self.audioRendererManager.unregisterRenderer(forReactTag: reactTag)
+        
+        return nil
+    }
+
     @objc(createVolumeProcessor:trackId:)
     public func createVolumeProcessor(_ pcId: NSNumber, trackId: String) -> String {
         let renderer = VolumeAudioRenderer(intervalMs: 40.0, eventEmitter: self)
@@ -195,7 +214,7 @@ public class LivekitReactNativeModule: RCTEventEmitter {
         
         return nil
     }
-    
+
     @objc(createMultibandVolumeProcessor:pcId:trackId:)
     public func createMultibandVolumeProcessor(_ options: NSDictionary, pcId: NSNumber, trackId: String) -> String {
         let bands = (options["bands"] as? NSNumber)?.intValue ?? 5
@@ -237,6 +256,7 @@ public class LivekitReactNativeModule: RCTEventEmitter {
         return [
             LKEvents.kEventVolumeProcessed,
             LKEvents.kEventMultibandProcessed,
+            LKEvents.kEventAudioData,
         ]
     }
 }
diff --git a/ios/LivekitReactNativeModule.m b/ios/LivekitReactNativeModule.m
@@ -21,6 +21,12 @@ @interface RCT_EXTERN_MODULE(LivekitReactNativeModule, RCTEventEmitter)
 /// Configure audio config for WebRTC
 RCT_EXTERN_METHOD(setAppleAudioConfiguration:(NSDictionary *) configuration)
 
+RCT_EXTERN__BLOCKING_SYNCHRONOUS_METHOD(createAudioSinkListener:(nonnull NSNumber *)pcId
+                                        trackId:(nonnull NSString *)trackId)
+
+RCT_EXTERN__BLOCKING_SYNCHRONOUS_METHOD(deleteAudioSinkListener:(nonnull NSString *)reactTag
+                                        pcId:(nonnull NSNumber *)pcId
+                                        trackId:(nonnull NSString *)trackId)
 
 RCT_EXTERN__BLOCKING_SYNCHRONOUS_METHOD(createVolumeProcessor:(nonnull NSNumber *)pcId
                                         trackId:(nonnull NSString *)trackId)
diff --git a/ios/audio/AudioSinkRenderer.swift b/ios/audio/AudioSinkRenderer.swift
@@ -0,0 +1,51 @@
+import livekit_react_native_webrtc
+import React
+
+@objc
+public class AudioSinkRenderer: BaseAudioSinkRenderer {
+    private let eventEmitter: RCTEventEmitter
+    
+    @objc
+    public var reactTag: String? = nil
+    
+    @objc
+    public init(eventEmitter: RCTEventEmitter) {
+        self.eventEmitter = eventEmitter
+        super.init()
+    }
+    
+    override public func onData(_ pcmBuffer: AVAudioPCMBuffer) {
+        guard pcmBuffer.format.commonFormat == .pcmFormatInt16,
+              let channelData = pcmBuffer.int16ChannelData else {
+            return
+        }
+        let channelCount = Int(pcmBuffer.format.channelCount)
+        let channels = UnsafeBufferPointer(start: channelData, count: channelCount)
+        let length = Int(pcmBuffer.frameCapacity * pcmBuffer.format.streamDescription.pointee.mBytesPerFrame)
+        let data = NSData(bytes: channels[0], length: length)
+        let base64 = data.base64EncodedString()
+        NSLog("AUDIO DATA!!!!")
+        NSLog("\(data.length)")
+        NSLog(base64)
+        NSLog("\(base64.count)")
+        NSLog("\(length)")
+        eventEmitter.sendEvent(withName: LKEvents.kEventAudioData, body: [
+            "data": base64,
+            "id": reactTag
+        ])
+    }
+}
+
+public class BaseAudioSinkRenderer: NSObject, RTCAudioRenderer {
+    
+    public override init() {
+        super.init()
+    }
+    
+    public func render(pcmBuffer: AVAudioPCMBuffer) {
+        onData(pcmBuffer)
+    }
+    
+    public func onData(_ pcmBuffer: AVAudioPCMBuffer) {
+    }
+}
diff --git a/package.json b/package.json
@@ -44,9 +44,11 @@
   "dependencies": {
     "@livekit/components-react": "^2.8.1",
     "array.prototype.at": "^1.1.1",
+    "event-target-shim": "6.0.2",
     "events": "^3.3.0",
     "loglevel": "^1.8.0",
     "promise.allsettled": "^1.0.5",
+    "react-native-quick-base64": "2.1.1",
     "react-native-url-polyfill": "^1.3.0",
     "typed-emitter": "^2.1.0",
     "web-streams-polyfill": "^4.1.0",
@@ -73,7 +75,7 @@
     "eslint-plugin-prettier": "^4.2.1",
     "husky": "^7.0.4",
     "jest": "^29.6.3",
-    "livekit-client": "^2.9.8",
+    "livekit-client": "^2.15.4",
     "pod-install": "^0.2.2",
     "prettier": "2.8.8",
     "react": "18.2.0",
@@ -85,7 +87,7 @@
   },
   "peerDependencies": {
     "@livekit/react-native-webrtc": "^125.0.12",
-    "livekit-client": "^2.9.0",
+    "livekit-client": "^2.15.4",
     "react": "*",
     "react-native": "*"
   },
diff --git a/src/audio/MediaRecorder.ts b/src/audio/MediaRecorder.ts
diff --git a/src/events/EventEmitter.ts b/src/events/EventEmitter.ts
diff --git a/src/index.tsx b/src/index.tsx
diff --git a/src/polyfills/MediaRecorderShim.ts b/src/polyfills/MediaRecorderShim.ts
diff --git a/yarn.lock b/yarn.lock

Original file line number	Diff line number	Diff line change
`@@ -3,4 +3,5 @@ package com.livekit.reactnative.audio.events`
`3`	`3`	`enum class Events {`
`4`	`4`	`LK_VOLUME_PROCESSED,`
`5`	`5`	`LK_MULTIBAND_PROCESSED,`
	`6`	`+ LK_AUDIO_DATA,`
`6`	`7`	`}`