Skip to content

Commit 0fc303f

Browse files
kaizhi-singtownkaizhi-singtown
committed
feat: add Microphone, Camera
1 parent 20c6fbd commit 0fc303f

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+2311
-135
lines changed

‎android/src/main/cpp/cpp-adapter.cpp‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,3 +107,104 @@ Java_com_webrtc_HybridWebrtcView_subscribeVideo (JNIEnv *env, jobject,
107107
std::string pipeIdStr (env->GetStringUTFChars (pipeId, nullptr));
108108
return subscribe ({ pipeIdStr }, callback, cleanup);
109109
}
110+
111+
extern "C" JNIEXPORT void JNICALL
112+
Java_com_webrtc_HybridMicrophone_publishAudio (JNIEnv *env, jobject,
113+
jstring pipeId,
114+
jbyteArray audioBuffer,
115+
jint size)
116+
117+
{
118+
auto frame = FFmpeg::Frame (AV_SAMPLE_FMT_S16, 48000, 1, size / 2);
119+
jboolean isCopy = JNI_FALSE;
120+
jbyte *audioData = env->GetByteArrayElements (audioBuffer, &isCopy);
121+
memcpy (frame->data[0], audioData, size);
122+
env->ReleaseByteArrayElements (audioBuffer, audioData, JNI_ABORT);
123+
124+
std::string pipeIdStr (env->GetStringUTFChars (pipeId, nullptr));
125+
publish (pipeIdStr, frame);
126+
}
127+
128+
extern "C" JNIEXPORT void JNICALL Java_com_webrtc_Camera_publishVideo (
129+
JNIEnv *env, jobject, jobjectArray pipeIds, jobject image)
130+
{
131+
jclass imageClass = env->GetObjectClass (image);
132+
jmethodID getWidthMethod
133+
= env->GetMethodID (imageClass, "getWidth", "()I");
134+
jmethodID getHeightMethod
135+
= env->GetMethodID (imageClass, "getHeight", "()I");
136+
jint width = env->CallIntMethod (image, getWidthMethod);
137+
jint height = env->CallIntMethod (image, getHeightMethod);
138+
139+
jmethodID getPlanesMethod = env->GetMethodID (
140+
imageClass, "getPlanes", "()[Landroid/media/Image$Plane;");
141+
auto planeArray
142+
= (jobjectArray)env->CallObjectMethod (image, getPlanesMethod);
143+
144+
jobject yPlane = env->GetObjectArrayElement (planeArray, 0);
145+
jobject uPlane = env->GetObjectArrayElement (planeArray, 1);
146+
jobject vPlane = env->GetObjectArrayElement (planeArray, 2);
147+
jclass planeClass = env->GetObjectClass (yPlane);
148+
jmethodID getBufferMethod = env->GetMethodID (planeClass, "getBuffer",
149+
"()Ljava/nio/ByteBuffer;");
150+
jmethodID getRowStrideMethod
151+
= env->GetMethodID (planeClass, "getRowStride", "()I");
152+
jmethodID getPixelStrideMethod
153+
= env->GetMethodID (planeClass, "getPixelStride", "()I");
154+
155+
jobject yByteBuffer = env->CallObjectMethod (yPlane, getBufferMethod);
156+
auto *yBufferPtr
157+
= static_cast<uint8_t *> (env->GetDirectBufferAddress (yByteBuffer));
158+
jint yRowStride = env->CallIntMethod (yPlane, getRowStrideMethod);
159+
160+
jobject uByteBuffer = env->CallObjectMethod (uPlane, getBufferMethod);
161+
auto *uBufferPtr
162+
= static_cast<uint8_t *> (env->GetDirectBufferAddress (uByteBuffer));
163+
jint uRowStride = env->CallIntMethod (uPlane, getRowStrideMethod);
164+
jint uPixelStride = env->CallIntMethod (uPlane, getPixelStrideMethod);
165+
166+
jobject vByteBuffer = env->CallObjectMethod (vPlane, getBufferMethod);
167+
auto *vBufferPtr
168+
= static_cast<uint8_t *> (env->GetDirectBufferAddress (vByteBuffer));
169+
jint vRowStride = env->CallIntMethod (vPlane, getRowStrideMethod);
170+
jint vPixelStride = env->CallIntMethod (vPlane, getPixelStrideMethod);
171+
172+
FFmpeg::Frame frame (AV_PIX_FMT_NV12, width, height);
173+
174+
// Copy Y
175+
for (int y = 0; y < height; ++y)
176+
{
177+
memcpy (frame->data[0] + y * frame->linesize[0],
178+
yBufferPtr + y * yRowStride, width);
179+
}
180+
181+
// Copy UV
182+
for (int y = 0; y < height / 2; ++y)
183+
{
184+
for (int x = 0; x < width / 2; ++x)
185+
{
186+
frame->data[1][y * frame->linesize[1] + x * 2]
187+
= uBufferPtr[y * uRowStride + x * uPixelStride];
188+
frame->data[1][y * frame->linesize[1] + x * 2 + 1]
189+
= vBufferPtr[y * vRowStride + x * vPixelStride];
190+
}
191+
}
192+
193+
env->DeleteLocalRef (yPlane);
194+
env->DeleteLocalRef (uPlane);
195+
env->DeleteLocalRef (vPlane);
196+
env->DeleteLocalRef (planeArray);
197+
env->DeleteLocalRef (imageClass);
198+
env->DeleteLocalRef (planeClass);
199+
200+
jsize pipeIdsLength = env->GetArrayLength (pipeIds);
201+
for (jsize i = 0; i < pipeIdsLength; ++i)
202+
{
203+
auto pipeId = (jstring)env->GetObjectArrayElement (pipeIds, i);
204+
const char *cstr = env->GetStringUTFChars (pipeId, nullptr);
205+
std::string pipeIdStr (cstr);
206+
publish (pipeIdStr, frame);
207+
env->ReleaseStringUTFChars (pipeId, cstr);
208+
env->DeleteLocalRef (pipeId);
209+
}
210+
}

‎android/src/main/java/com/webrtc/HybridCamera.kt‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
package com.webrtc
2+
3+
import android.content.Context
4+
import androidx.annotation.Keep
5+
import android.Manifest
6+
import android.content.pm.PackageManager
7+
import android.hardware.camera2.CameraDevice
8+
import android.hardware.camera2.CameraManager
9+
import android.hardware.camera2.CameraCaptureSession
10+
import android.hardware.camera2.CaptureRequest
11+
import android.media.Image
12+
import android.media.ImageReader
13+
import android.graphics.ImageFormat
14+
import android.os.Handler
15+
import android.os.HandlerThread
16+
import com.facebook.proguard.annotations.DoNotStrip
17+
import com.margelo.nitro.webrtc.HybridCameraSpec
18+
import com.margelo.nitro.core.Promise
19+
import com.margelo.nitro.NitroModules
20+
21+
22+
object Camera {
23+
private const val DEFAULT_WIDTH = 1280
24+
private const val DEFAULT_HEIGHT = 720
25+
26+
private var cameraDevice: CameraDevice? = null
27+
private var captureSession: CameraCaptureSession? = null
28+
private var backgroundThread = HandlerThread("CameraBackground")
29+
private var imageReader =
30+
ImageReader.newInstance(DEFAULT_WIDTH, DEFAULT_HEIGHT, ImageFormat.YUV_420_888, 2)
31+
private val pipeIds = mutableSetOf<String>()
32+
33+
external fun publishVideo(pipeIds: Array<String>, image: Image)
34+
35+
private val cameraStateCallback = object : CameraDevice.StateCallback() {
36+
override fun onOpened(camera: CameraDevice) {
37+
cameraDevice = camera
38+
val captureRequestBuilder = camera.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW)
39+
captureRequestBuilder.addTarget(imageReader.surface)
40+
41+
captureRequestBuilder.set(
42+
CaptureRequest.CONTROL_AF_MODE,
43+
CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE
44+
)
45+
46+
captureRequestBuilder.set(
47+
CaptureRequest.CONTROL_AE_MODE,
48+
CaptureRequest.CONTROL_AE_MODE_ON
49+
)
50+
51+
captureRequestBuilder.set(
52+
CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE,
53+
android.util.Range(30, 30)
54+
)
55+
val captureRequest = captureRequestBuilder.build()
56+
57+
val sessionStateCallback = object : CameraCaptureSession.StateCallback() {
58+
override fun onConfigured(session: CameraCaptureSession) {
59+
captureSession = session
60+
session.setRepeatingRequest(
61+
captureRequest,
62+
null,
63+
Handler(backgroundThread.looper)
64+
)
65+
}
66+
67+
override fun onConfigureFailed(session: CameraCaptureSession) {
68+
throw RuntimeException("Camera configuration failed")
69+
}
70+
}
71+
camera.createCaptureSession(
72+
listOf(imageReader.surface),
73+
sessionStateCallback,
74+
Handler(backgroundThread.looper)
75+
)
76+
}
77+
78+
override fun onDisconnected(camera: CameraDevice) {
79+
camera.close()
80+
}
81+
82+
override fun onError(camera: CameraDevice, error: Int) {
83+
camera.close()
84+
throw RuntimeException("Camera run Error!")
85+
}
86+
}
87+
88+
@Synchronized
89+
fun addPipeId(pipeId: String) {
90+
if (pipeIds.contains(pipeId)) {
91+
return
92+
}
93+
pipeIds.add(pipeId)
94+
if (pipeIds.size == 1) {
95+
startCamera()
96+
}
97+
}
98+
99+
@Synchronized
100+
fun removePipeId(pipeId: String) {
101+
if (!pipeIds.contains(pipeId)) {
102+
return
103+
}
104+
pipeIds.remove(pipeId)
105+
if (pipeIds.isEmpty()) {
106+
stopCamera()
107+
}
108+
}
109+
110+
private fun startCamera() {
111+
val context = NitroModules.applicationContext
112+
?: throw RuntimeException("ReactApplicationContext is not available")
113+
114+
val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager
115+
?: throw RuntimeException("CameraManager is not available")
116+
117+
backgroundThread = HandlerThread("CameraBackground")
118+
backgroundThread.start()
119+
120+
val onImageAvailableListener = ImageReader.OnImageAvailableListener { reader ->
121+
val image = reader.acquireNextImage()
122+
image.use {
123+
publishVideo(pipeIds.toTypedArray(), it)
124+
}
125+
}
126+
imageReader.setOnImageAvailableListener(
127+
onImageAvailableListener,
128+
Handler(backgroundThread.looper)
129+
)
130+
131+
val cameraId = cameraManager.cameraIdList.firstOrNull()
132+
?: throw RuntimeException("Camera not exist!")
133+
cameraManager.openCamera(cameraId, cameraStateCallback, Handler(backgroundThread.looper))
134+
135+
}
136+
137+
private fun stopCamera() {
138+
captureSession?.stopRepeating()
139+
captureSession?.close()
140+
captureSession = null
141+
cameraDevice?.close()
142+
cameraDevice = null
143+
backgroundThread.quitSafely()
144+
backgroundThread.join()
145+
}
146+
}
147+
148+
149+
@Keep
150+
@DoNotStrip
151+
class HybridCamera : HybridCameraSpec() {
152+
private var pipeId: String = ""
153+
154+
override fun open(pipeId: String): Promise<Unit> {
155+
this.pipeId = pipeId
156+
return Promise.async {
157+
requestPermission(android.Manifest.permission.CAMERA)
158+
Camera.addPipeId(pipeId)
159+
}
160+
}
161+
162+
override fun dispose() {
163+
Camera.removePipeId(pipeId)
164+
pipeId = ""
165+
}
166+
}

‎android/src/main/java/com/webrtc/HybridMicrophone.kt‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
package com.webrtc
2+
3+
import android.media.AudioFormat
4+
import android.media.AudioRecord
5+
import android.media.MediaRecorder
6+
import androidx.annotation.Keep
7+
import com.facebook.proguard.annotations.DoNotStrip
8+
import com.margelo.nitro.webrtc.HybridMicrophoneSpec
9+
import kotlinx.coroutines.*
10+
import com.margelo.nitro.core.Promise
11+
import android.media.audiofx.AcousticEchoCanceler
12+
13+
@Keep
14+
@DoNotStrip
15+
class HybridMicrophone : HybridMicrophoneSpec() {
16+
private var audioRecord: AudioRecord? = null
17+
private var recordingJob: Job? = null
18+
private var pipeId: String = ""
19+
private val scope = CoroutineScope(Dispatchers.Default + SupervisorJob())
20+
private var aec: AcousticEchoCanceler? = null
21+
22+
companion object {
23+
private const val SAMPLE_RATE = 44100
24+
private const val CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_MONO
25+
private const val AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT
26+
private val BUFFER_SIZE = AudioRecord.getMinBufferSize(
27+
SAMPLE_RATE,
28+
CHANNEL_CONFIG,
29+
AUDIO_FORMAT
30+
)
31+
}
32+
33+
external fun publishAudio(pipeId: String, data: ByteArray, size: Int)
34+
35+
override fun open(pipeId: String): Promise<Unit> {
36+
this.pipeId = pipeId
37+
return Promise.async {
38+
requestPermission(android.Manifest.permission.RECORD_AUDIO)
39+
40+
audioRecord = AudioRecord(
41+
MediaRecorder.AudioSource.VOICE_COMMUNICATION,
42+
SAMPLE_RATE,
43+
CHANNEL_CONFIG,
44+
AUDIO_FORMAT,
45+
BUFFER_SIZE
46+
)
47+
48+
val recorder = audioRecord
49+
?: throw RuntimeException("AudioRecord is null")
50+
51+
if (recorder.state != AudioRecord.STATE_INITIALIZED) {
52+
throw RuntimeException("AudioRecord initialization failed")
53+
}
54+
55+
if (AcousticEchoCanceler.isAvailable()) {
56+
aec = AcousticEchoCanceler.create(recorder.audioSessionId)
57+
aec?.enabled = true
58+
}
59+
60+
recorder.startRecording()
61+
62+
recordingJob = scope.launch {
63+
val buffer = ByteArray(BUFFER_SIZE)
64+
while (isActive) {
65+
val readResult = recorder.read(buffer, 0, buffer.size)
66+
if (readResult > 0) {
67+
publishAudio(pipeId, buffer, readResult)
68+
}
69+
}
70+
}
71+
}
72+
}
73+
74+
override fun dispose() {
75+
recordingJob?.cancel()
76+
recordingJob = null
77+
78+
audioRecord?.apply {
79+
stop()
80+
release()
81+
}
82+
audioRecord = null
83+
scope.cancel()
84+
}
85+
}

‎android/src/main/java/com/webrtc/HybridWebrtcView.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ class HybridWebrtcView(val context: ThemedReactContext) : HybridWebrtcViewSpec()
3232
external fun unsubscribe(subscriptionId: Int)
3333
external fun subscribeAudio(pipeId: String, track: AudioTrack): Int
3434
external fun subscribeVideo(pipeId: String, surface: Surface): Int
35-
35+
3636
private var _audioPipeId: String? = null
3737
private var _videoPipeId: String? = null
3838
private var videoSubscriptionId: Int = -1

0 commit comments

Comments
 (0)