Skip to content

Commit 20c6fbd

Browse files
kaizhi-singtownkaizhi-singtown
committed
feat: add FramePipe, getMockMeida, MediaStream, MediaStreamTrack, WebrtcView, RTCPeerConnection, RTCRtpTransceiver
1 parent 289ef04 commit 20c6fbd

File tree

108 files changed

+7007
-563
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

108 files changed

+7007
-563
lines changed

‎.github/workflows/ios-build.yml‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -81,16 +81,6 @@ jobs:
8181
- name: Install xcpretty
8282
run: gem install xcpretty
8383

84-
- name: Cache CocoaPods
85-
uses: actions/cache@v4
86-
with:
87-
path: |
88-
~/.cocoapods/repos
89-
example/ios/Pods
90-
key: ${{ runner.os }}-pods-${{ hashFiles('example/ios/Podfile.lock') }}
91-
restore-keys: |
92-
${{ runner.os }}-pods-
93-
9484
- name: Install Pods
9585
working-directory: example/ios
9686
run: pod install

‎Webrtc.podspec‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,10 @@ Pod::Spec.new do |s|
2121
# Implementation (C++ objects)
2222
"cpp/*.{h,hpp,cpp}",
2323
"cpp/FFmpeg/*.{h,hpp,cpp}",
24+
"cpp/Hybrid/*.{h,hpp,cpp}",
2425
]
2526

27+
s.public_header_files = "ios/**/*.h"
2628
s.vendored_frameworks = "3rdparty/output/ios/*.xcframework"
2729
s.pod_target_xcconfig = {
2830
'HEADER_SEARCH_PATHS' => [

‎android/CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ set (CMAKE_CXX_STANDARD 20)
77

88
# Enable Raw Props parsing in react-native (for Nitro Views)
99
add_compile_options(-DRN_SERIALIZABLE_STATE=1)
10+
set(CMAKE_CXX_CLANG_TIDY "${ANDROID_TOOLCHAIN_ROOT}/bin/clang-tidy")
1011

1112
get_filename_component(TOP_PATH ${CMAKE_CURRENT_SOURCE_DIR}/.. ABSOLUTE)
1213
get_filename_component(LIBDATACHANNEL_PATH ${TOP_PATH}/3rdparty/output/android/libdatachannel/${ANDROID_ABI} ABSOLUTE)
@@ -15,6 +16,7 @@ get_filename_component(FFMPEG_PATH ${TOP_PATH}/3rdparty/output/android/ffmpeg/${
1516
file(GLOB CPP_SOURCES
1617
${TOP_PATH}/cpp/*.cpp
1718
${TOP_PATH}/cpp/FFmpeg/*.cpp
19+
${TOP_PATH}/cpp/Hybrid/*.cpp
1820
)
1921

2022
# Define C++ library and add all sources
@@ -31,6 +33,7 @@ include_directories(
3133
"src/main/cpp"
3234
"${TOP_PATH}/cpp"
3335
"${TOP_PATH}/cpp/FFmpeg"
36+
"${TOP_PATH}/cpp/Hybrid"
3437
${LIBDATACHANNEL_PATH}/include
3538
${FFMPEG_PATH}/include
3639
)

‎android/src/main/cpp/cpp-adapter.cpp‎

Lines changed: 106 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,109 @@
1-
#include <jni.h>
1+
#include "FFmpeg.hpp"
2+
#include "FramePipe.hpp"
23
#include "WebrtcOnLoad.hpp"
4+
#include <android/native_window.h>
5+
#include <android/native_window_jni.h>
6+
#include <chrono>
7+
#include <jni.h>
8+
#include <string>
9+
10+
JavaVM *gJvm = nullptr;
11+
12+
JNIEXPORT auto JNICALL JNI_OnLoad (JavaVM *vm, void *) -> jint
13+
{
14+
gJvm = vm;
15+
return margelo::nitro::webrtc::initialize (vm);
16+
}
17+
18+
extern "C" JNIEXPORT void JNICALL
19+
Java_com_webrtc_HybridWebrtcView_unsubscribe (JNIEnv *, jobject,
20+
jint subscriptionId)
21+
{
22+
unsubscribe (subscriptionId);
23+
}
24+
25+
extern "C" JNIEXPORT auto JNICALL
26+
Java_com_webrtc_HybridWebrtcView_subscribeAudio (JNIEnv *env, jobject,
27+
jstring pipeId, jobject track)
28+
-> int
29+
{
30+
auto resampler = std::make_shared<FFmpeg::Resampler> ();
31+
jobject trackGlobal = env->NewGlobalRef (track);
32+
33+
FrameCallback callback
34+
= [trackGlobal, resampler] (const std::string &, int,
35+
const FFmpeg::Frame &raw)
36+
{
37+
JNIEnv *env;
38+
gJvm->AttachCurrentThread (&env, nullptr);
39+
40+
FFmpeg::Frame frame
41+
= resampler->resample (raw, AV_SAMPLE_FMT_S16, 48000, 2);
42+
auto *sample = reinterpret_cast<const jbyte *> (frame->data[0]);
43+
int length = frame->nb_samples * 2 * 2;
44+
jbyteArray byteArray = env->NewByteArray (length);
45+
env->SetByteArrayRegion (byteArray, 0, length, sample);
46+
47+
jclass audioTrackCls = env->GetObjectClass (trackGlobal);
48+
jmethodID writeMethod
49+
= env->GetMethodID (audioTrackCls, "write", "([BII)I");
50+
env->CallIntMethod (trackGlobal, writeMethod, byteArray, 0, length);
51+
};
52+
53+
CleanupCallback cleanup = [trackGlobal] (int)
54+
{
55+
JNIEnv *env;
56+
gJvm->AttachCurrentThread (&env, nullptr);
57+
env->DeleteGlobalRef (trackGlobal);
58+
};
59+
60+
std::string pipeIdStr (env->GetStringUTFChars (pipeId, nullptr));
61+
return subscribe ({ pipeIdStr }, callback);
62+
}
63+
64+
extern "C" JNIEXPORT auto JNICALL
65+
Java_com_webrtc_HybridWebrtcView_subscribeVideo (JNIEnv *env, jobject,
66+
jstring pipeId,
67+
jobject surface) -> jint
68+
{
69+
if (!surface)
70+
{
71+
return -1;
72+
}
73+
ANativeWindow *window = ANativeWindow_fromSurface (env, surface);
74+
if (!window)
75+
{
76+
return -1;
77+
}
78+
79+
auto scaler = std::make_shared<FFmpeg::Scaler> ();
80+
FrameCallback callback
81+
= [window, scaler] (const std::string &, int, const FFmpeg::Frame &raw)
82+
{
83+
FFmpeg::Frame frame
84+
= scaler->scale (raw, AV_PIX_FMT_RGBA, raw->width, raw->height);
85+
86+
ANativeWindow_setBuffersGeometry (window, frame->width, frame->height,
87+
WINDOW_FORMAT_RGBA_8888);
88+
89+
ANativeWindow_Buffer buffer;
90+
if (ANativeWindow_lock (window, &buffer, nullptr) < 0)
91+
{
92+
return;
93+
}
94+
95+
auto *dst = static_cast<uint8_t *> (buffer.bits);
96+
for (int y = 0; y < frame->height; ++y)
97+
{
98+
uint8_t *srcRow = frame->data[0] + y * frame->linesize[0];
99+
uint8_t *dstRow = dst + y * buffer.stride * 4;
100+
memcpy (dstRow, srcRow, frame->width * 4);
101+
}
3102

4-
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void*) {
5-
return margelo::nitro::webrtc::initialize(vm);
103+
ANativeWindow_unlockAndPost (window);
104+
};
105+
CleanupCallback cleanup
106+
= [window] (int) { ANativeWindow_release (window); };
107+
std::string pipeIdStr (env->GetStringUTFChars (pipeId, nullptr));
108+
return subscribe ({ pipeIdStr }, callback, cleanup);
6109
}

‎android/src/main/java/com/webrtc/HybridWebrtcView.kt‎

Lines changed: 82 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,96 @@
11
package com.webrtc
22

3-
import android.graphics.Color
4-
import android.view.View
3+
import android.view.Surface
4+
import android.view.SurfaceView
5+
import android.view.SurfaceHolder
6+
import android.media.AudioFormat
7+
import android.media.AudioManager
8+
import android.media.AudioTrack
59
import androidx.annotation.Keep
610
import com.facebook.proguard.annotations.DoNotStrip
711
import com.facebook.react.uimanager.ThemedReactContext
812
import com.margelo.nitro.webrtc.HybridWebrtcViewSpec
913

1014
@Keep
1115
@DoNotStrip
12-
class HybridWebrtcView(val context: ThemedReactContext): HybridWebrtcViewSpec() {
16+
class HybridWebrtcView(val context: ThemedReactContext) : HybridWebrtcViewSpec() {
1317
// View
14-
override val view: View = View(context)
18+
override val view: SurfaceView = SurfaceView(context)
19+
private var audioTrack = AudioTrack(
20+
AudioManager.STREAM_MUSIC,
21+
48000,
22+
AudioFormat.CHANNEL_OUT_STEREO,
23+
AudioFormat.ENCODING_PCM_16BIT,
24+
AudioTrack.getMinBufferSize(
25+
48000,
26+
AudioFormat.CHANNEL_OUT_STEREO,
27+
AudioFormat.ENCODING_PCM_16BIT
28+
),
29+
AudioTrack.MODE_STREAM
30+
)
1531

16-
// Props
17-
private var _isRed = false
18-
override var isRed: Boolean
19-
get() = _isRed
32+
external fun unsubscribe(subscriptionId: Int)
33+
external fun subscribeAudio(pipeId: String, track: AudioTrack): Int
34+
external fun subscribeVideo(pipeId: String, surface: Surface): Int
35+
36+
private var _audioPipeId: String? = null
37+
private var _videoPipeId: String? = null
38+
private var videoSubscriptionId: Int = -1
39+
private var audioSubscriptionId: Int = -1
40+
41+
override var audioPipeId: String?
42+
get() = _audioPipeId
43+
set(value) {
44+
if (this.audioSubscriptionId > 0) {
45+
this.unsubscribe(this.audioSubscriptionId)
46+
}
47+
if (value.isNullOrEmpty()) {
48+
return;
49+
}
50+
this.audioSubscriptionId = subscribeAudio(value, audioTrack)
51+
this._audioPipeId = value
52+
audioTrack.play()
53+
}
54+
55+
56+
init {
57+
view.holder.addCallback(object : SurfaceHolder.Callback {
58+
override fun surfaceCreated(holder: SurfaceHolder) {
59+
updateVideoPipeId(_videoPipeId, holder.surface)
60+
}
61+
62+
override fun surfaceChanged(
63+
holder: SurfaceHolder,
64+
format: Int,
65+
width: Int,
66+
height: Int
67+
) {
68+
updateVideoPipeId(_videoPipeId, holder.surface)
69+
}
70+
71+
override fun surfaceDestroyed(holder: SurfaceHolder) {
72+
updateVideoPipeId(_videoPipeId, null)
73+
}
74+
})
75+
}
76+
77+
private fun updateVideoPipeId(newVideoPipeId: String?, surface: Surface?) {
78+
if (this.videoSubscriptionId > 0) {
79+
this.unsubscribe(this.videoSubscriptionId)
80+
}
81+
if (surface == null) {
82+
return;
83+
}
84+
if (newVideoPipeId.isNullOrEmpty()) {
85+
return;
86+
}
87+
this.videoSubscriptionId = subscribeVideo(newVideoPipeId, surface)
88+
this._videoPipeId = newVideoPipeId
89+
}
90+
91+
override var videoPipeId: String?
92+
get() = _videoPipeId
2093
set(value) {
21-
_isRed = value
22-
view.setBackgroundColor(
23-
if (value) Color.RED
24-
else Color.BLACK
25-
)
94+
updateVideoPipeId(value, view.holder.surface)
2695
}
2796
}

‎cpp/FFmpeg/Encoder.cpp‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -130,25 +130,20 @@ void Encoder::send (const Frame &frame)
130130

131131
if (encoder->id == AV_CODEC_ID_OPUS)
132132
{
133-
std::optional<Frame> resampled_frame = resampler.resample (
134-
frame, AV_SAMPLE_FMT_FLT, AAC_SAMPLE_RATE, 2);
135-
if (resampled_frame.has_value ())
136-
{
137-
fifo.write (resampled_frame.value ());
138-
}
133+
Frame resampled_frame = resampler.resample (frame, AV_SAMPLE_FMT_FLT,
134+
AAC_SAMPLE_RATE, 2);
135+
fifo.write (resampled_frame);
136+
139137
while (std::optional<Frame> frame = fifo.read (OPUS_NB_SAMPLES))
140138
{
141139
frames.push_back (frame.value ());
142140
}
143141
}
144142
else if (encoder->id == AV_CODEC_ID_AAC)
145143
{
146-
std::optional<Frame> resampled_frame = resampler.resample (
147-
frame, AV_SAMPLE_FMT_FLTP, OPUS_SAMPLE_RATE, 2);
148-
if (resampled_frame.has_value ())
149-
{
150-
fifo.write (resampled_frame.value ());
151-
}
144+
Frame resampled_frame = resampler.resample (frame, AV_SAMPLE_FMT_FLTP,
145+
OPUS_SAMPLE_RATE, 2);
146+
fifo.write (resampled_frame);
152147
while (std::optional<Frame> frame = fifo.read (AAC_NB_SAMPLES))
153148
{
154149
frames.push_back (frame.value ());

0 commit comments

Comments
 (0)