Skip to content
This repository was archived by the owner on Oct 25, 2024. It is now read-only.

Commit 0f82778

Browse files
jianjunzjianjunz
committed
Add wasm APIs for RTP depacketizing.
1 parent 0490093 commit 0f82778

File tree

10 files changed

+152
-58
lines changed

10 files changed

+152
-58
lines changed

‎talk/owt/BUILD.gn

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,8 @@ if (is_wasm) {
340340
"//third_party/webrtc/call",
341341
"//third_party/webrtc/modules/rtp_rtcp:rtp_rtcp",
342342
"//third_party/webrtc/rtc_base:rtc_json",
343+
"//third_party/webrtc/rtc_base:rtc_task_queue_stdlib",
344+
"//third_party/webrtc/api/rtc_event_log:rtc_event_log_factory",
343345
]
344346
sources = [
345347
"sdk/wasm/binding.h",

‎talk/owt/sdk/wasm/binding.h

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,16 @@ namespace owt {
1313
namespace wasm {
1414

1515
EMSCRIPTEN_BINDINGS(Owt) {
16-
emscripten::class_<MediaSession>("MediaSession").constructor<>();
16+
emscripten::class_<MediaSession>("MediaSession")
17+
.constructor<>()
18+
.function("createRtpVideoReceiver",
19+
&MediaSession::CreateRtpVideoReceiver);
1720
emscripten::class_<RtpVideoReceiver>("RtpVideoReceiver")
18-
.function("onRtpPacket", &RtpVideoReceiver::OnRtpPacket);
21+
.smart_ptr<std::shared_ptr<RtpVideoReceiver>>("RtpVideoReceiver")
22+
.function("onRtpPacket", &RtpVideoReceiver::OnRtpPacket,
23+
emscripten::allow_raw_pointers())
24+
.function("setCompleteFrameCallback",
25+
&RtpVideoReceiver::SetCompleteFrameCallback);
1926
}
2027

2128
} // namespace wasm

‎talk/owt/sdk/wasm/gn/BUILD.gn

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,23 +65,23 @@ template("gcc_like_toolchain") {
6565

6666
tool("cc") {
6767
depfile = "{{output}}.d"
68-
command = "$cc_wrapper $cc -MMD -MF $depfile {{defines}} {{include_dirs}} {{cflags}} {{cflags_c}} -DEMSCRIPTEN_HAS_UNBOUND_TYPE_NAMES=0 --bind -fno-stack-protector ${external_cflags} -c {{source}} -o {{output}}"
68+
command = "$cc_wrapper $cc -MMD -MF $depfile {{defines}} {{include_dirs}} {{cflags}} {{cflags_c}} -DEMSCRIPTEN_HAS_UNBOUND_TYPE_NAMES=1 --bind -fno-stack-protector ${external_cflags} -c {{source}} -o {{output}}"
6969
depsformat = "gcc"
7070
outputs = [ "$object_subdir/{{source_name_part}}.o" ]
7171
description = "compile {{source}}"
7272
}
7373

7474
tool("cxx") {
7575
depfile = "{{output}}.d"
76-
command = "$cc_wrapper $cxx -MMD -MF $depfile {{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}} -DEMSCRIPTEN_HAS_UNBOUND_TYPE_NAMES=0 --bind -fno-stack-protector ${external_cflags} ${external_cxxflags} -c {{source}} -o {{output}}"
76+
command = "$cc_wrapper $cxx -MMD -MF $depfile {{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}} -DEMSCRIPTEN_HAS_UNBOUND_TYPE_NAMES=1 --bind -fno-stack-protector ${external_cflags} ${external_cxxflags} -c {{source}} -o {{output}}"
7777
depsformat = "gcc"
7878
outputs = [ "$object_subdir/{{source_name_part}}.o" ]
7979
description = "compile {{source}}"
8080
}
8181

8282
tool("asm") {
8383
depfile = "{{output}}.d"
84-
command = "$cc_wrapper $cc -MMD -MF $depfile {{defines}} {{include_dirs}} {{asmflags}} -DEMSCRIPTEN_HAS_UNBOUND_TYPE_NAMES=0 --bind -fno-stack-protector -c {{source}} -o {{output}}"
84+
command = "$cc_wrapper $cc -MMD -MF $depfile {{defines}} {{include_dirs}} {{asmflags}} -DEMSCRIPTEN_HAS_UNBOUND_TYPE_NAMES=1 --bind -fno-stack-protector -c {{source}} -o {{output}}"
8585
depsformat = "gcc"
8686
outputs = [ "$object_subdir/{{source_name_part}}.o" ]
8787
description = "assemble {{source}}"
@@ -142,7 +142,7 @@ template("gcc_like_toolchain") {
142142
tool("link") {
143143
unstripped_exe =
144144
"{{root_out_dir}}/{{target_output_name}}{{output_extension}}"
145-
command = "$cc_wrapper $cxx $ld_arg ${external_ldflags} {{inputs}} {{solibs}} {{libs}} -o $unstripped_exe"
145+
command = "$cc_wrapper $cxx $ld_arg {{ldflags}} ${external_ldflags} {{inputs}} {{solibs}} {{libs}} -o $unstripped_exe"
146146
outputs = [ unstripped_exe ]
147147
description = "link $unstripped_exe"
148148
if (strip != "") {
@@ -174,7 +174,7 @@ gcc_like_toolchain("wasm") {
174174
cpu = host_cpu
175175
os = host_os
176176
ar = "$emsdk_dir/emscripten/emar --em-config $em_config"
177-
cc = "$emsdk_dir/emscripten/emcc --em-config $em_config"
178-
cxx = "$emsdk_dir/emscripten/em++ --em-config $em_config --bind -s USE_PTHREADS=1"
177+
cc = "$emsdk_dir/emscripten/emcc --em-config $em_config -pthread"
178+
cxx = "$emsdk_dir/emscripten/em++ --em-config $em_config --bind -pthread"
179179
strip = ""
180180
}

‎talk/owt/sdk/wasm/gn/wasm.gni

Lines changed: 28 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -38,26 +38,26 @@ template("wasm_lib") {
3838
assert(invoker.name + "_wasm" == target_name)
3939
_lib_name = invoker.name
4040
if (is_wasm) {
41-
_exports = "['ccall', 'callMain', 'addFunction', 'FS']"
41+
_exports = "['writeArrayToMemory']"
4242
_target_ldflags = [
4343
"-s",
4444
"WASM=1",
45-
"-s",
46-
"DISABLE_EXCEPTION_CATCHING=1",
47-
"-s",
48-
"NO_DYNAMIC_EXECUTION=1",
45+
# "-s",
46+
# "DISABLE_EXCEPTION_CATCHING=1",
4947
"-s",
5048
"INITIAL_MEMORY=33554432",
5149
"-s",
5250
"ALLOW_TABLE_GROWTH=1",
53-
"-s",
54-
"WASM_ASYNC_COMPILATION=0",
51+
# "-s",
52+
# "WASM_ASYNC_COMPILATION=0",
5553
"-s",
5654
"USE_PTHREADS=1",
5755
"-s",
56+
"PROXY_TO_PTHREAD=1",
57+
"-s",
5858
"LLD_REPORT_UNDEFINED",
5959
"-s",
60-
"EXTRA_EXPORTED_RUNTIME_METHODS=" + _exports,
60+
"EXPORTED_RUNTIME_METHODS=" + _exports,
6161

6262
# This forces the MEMFS filesystem library to always use typed arrays
6363
# instead of building strings/arrays when appending to a file. This allows
@@ -66,30 +66,37 @@ template("wasm_lib") {
6666
"MEMFS_APPEND_TO_TYPED_ARRAYS=1",
6767

6868
# Reduces global namespace pollution.
69-
"-s",
70-
"MODULARIZE=1",
69+
# "-s",
70+
# "MODULARIZE=1",
7171

72-
"-fno-stack-protector",
72+
#"-fno-stack-protector",
7373

7474
# This is to prevent that two different wasm modules end up generating
7575
# JS that overrides the same global variable (var Module = ...)
76-
"-s",
77-
"EXPORT_NAME=${target_name}",
76+
# "-s",
77+
# "EXPORT_NAME=${target_name}",
7878

7979
"-lworkerfs.js", # For FS.filesystems.WORKERFS
8080

8181
"--bind",
82+
83+
"-s",
84+
"PTHREAD_POOL_SIZE=2",
85+
86+
"-pthread",
87+
88+
"--source-map-base",
8289
]
8390
if (is_debug) {
8491
_target_ldflags += [
85-
"-s",
86-
"ASSERTIONS=2",
87-
"-s",
88-
"SAFE_HEAP=1",
89-
"-s",
90-
"STACK_OVERFLOW_CHECK=1",
91-
"-gsource-map",
92-
"-O0",
92+
# "-s",
93+
# "ASSERTIONS=2",
94+
# "-s",
95+
# "SAFE_HEAP=1",
96+
# "-s",
97+
# "STACK_OVERFLOW_CHECK=1",
98+
# "-gsource-map",
99+
# "-O0",
93100
]
94101
} else {
95102
_target_ldflags += [
@@ -108,7 +115,6 @@ template("wasm_lib") {
108115
}
109116

110117
_target_cflags = [
111-
"-fno-stack-protector",
112118
"-pthread",
113119
]
114120
if (defined(invoker.cflags)) {

‎talk/owt/sdk/wasm/media_session.cc

Lines changed: 31 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,30 +3,45 @@
33
// SPDX-License-Identifier: Apache-2.0
44

55
#include "talk/owt/sdk/wasm/media_session.h"
6+
#include "third_party/webrtc/api/rtc_event_log/rtc_event_log_factory.h"
7+
#include "third_party/webrtc/api/transport/field_trial_based_config.h"
8+
#include "third_party/webrtc/rtc_base/task_queue_stdlib.h"
69

710
namespace owt {
811
namespace wasm {
912
MediaSession::MediaSession()
10-
: call_(std::unique_ptr<webrtc::Call>(
11-
webrtc::Call::Create(webrtc::Call::Config(nullptr)))),
12-
video_receiver_(nullptr) {}
13+
: task_queue_factory_(webrtc::CreateTaskQueueStdlibFactory()),
14+
event_log_factory_(std::make_unique<webrtc::RtcEventLogFactory>(
15+
task_queue_factory_.get())),
16+
event_log_(event_log_factory_->CreateRtcEventLog(
17+
webrtc::RtcEventLog::EncodingType::NewFormat)),
18+
web_transport_session_(std::make_unique<WebTransportSession>()),
19+
call_(nullptr),
20+
receive_statistics_(
21+
webrtc::ReceiveStatistics::Create(webrtc::Clock::GetRealTimeClock())),
22+
receiver_process_thread_(nullptr),
23+
video_receiver_(nullptr) {
24+
RTC_LOG(LS_ERROR) << "Ctor of Media Session. " << __EMSCRIPTEN_PTHREADS__;
25+
rtc::ThreadManager::Instance()->WrapCurrentThread();
26+
receiver_process_thread_=webrtc::ProcessThread::Create("ReceiverProcessThread");
27+
webrtc::Call::Config config(event_log_.get());
28+
config.task_queue_factory = task_queue_factory_.get();
29+
webrtc::FieldTrialBasedConfig trials;
30+
config.trials = &trials;
31+
call_ = std::unique_ptr<webrtc::Call>(webrtc::Call::Create(config));
32+
}
1333

1434
MediaSession::~MediaSession() {}
1535

16-
RtpVideoReceiver* MediaSession::CreateRtpVideoReceiver() {
36+
std::shared_ptr<RtpVideoReceiver> MediaSession::CreateRtpVideoReceiver() {
1737
webrtc::VideoReceiveStream::Config config(web_transport_session_.get());
18-
webrtc::TaskQueueBase* task_queue = webrtc::TaskQueueBase::Current();
19-
if (!task_queue) {
20-
task_queue = rtc::ThreadManager::Instance()->CurrentThread();
21-
}
22-
std::unique_ptr<webrtc::RtpVideoStreamReceiver2> receiver =
23-
std::make_unique<webrtc::RtpVideoStreamReceiver2>(
24-
task_queue, webrtc::Clock::GetRealTimeClock(),
25-
web_transport_session_.get(), nullptr, nullptr, &config, nullptr,
26-
nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
27-
nullptr);
28-
video_receiver_ = std::make_unique<RtpVideoReceiver>(std::move(receiver));
29-
return video_receiver_.get();
38+
config.rtp.local_ssrc = 1;
39+
// TODO: Get a message queue for receiver.
40+
video_receiver_ = std::make_shared<RtpVideoReceiver>(
41+
web_transport_session_.get(), &config, receive_statistics_.get(),
42+
receiver_process_thread_.get(), web_transport_session_.get());
43+
return video_receiver_;
3044
}
45+
3146
} // namespace wasm
3247
} // namespace owt

‎talk/owt/sdk/wasm/media_session.h

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#include "call/call.h"
99
#include "talk/owt/sdk/wasm/rtp_video_receiver.h"
1010
#include "talk/owt/sdk/wasm/web_transport_session.h"
11+
#include "third_party/webrtc/api/rtc_event_log/rtc_event_log_factory_interface.h"
1112

1213
namespace owt {
1314
namespace wasm {
@@ -16,12 +17,17 @@ class MediaSession {
1617
public:
1718
explicit MediaSession();
1819
virtual ~MediaSession();
19-
RtpVideoReceiver* CreateRtpVideoReceiver();
20+
std::shared_ptr<RtpVideoReceiver> CreateRtpVideoReceiver();
2021

2122
private:
23+
std::unique_ptr<webrtc::TaskQueueFactory> task_queue_factory_;
24+
std::unique_ptr<webrtc::RtcEventLogFactoryInterface> event_log_factory_;
25+
std::unique_ptr<webrtc::RtcEventLog> event_log_;
2226
std::unique_ptr<WebTransportSession> web_transport_session_;
2327
std::unique_ptr<webrtc::Call> call_;
24-
std::unique_ptr<RtpVideoReceiver> video_receiver_;
28+
std::unique_ptr<webrtc::ReceiveStatistics> receive_statistics_;
29+
std::unique_ptr<webrtc::ProcessThread> receiver_process_thread_;
30+
std::shared_ptr<RtpVideoReceiver> video_receiver_;
2531
};
2632

2733
} // namespace wasm

‎talk/owt/sdk/wasm/rtp_video_receiver.cc

Lines changed: 38 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,50 @@
88

99
namespace owt {
1010
namespace wasm {
11+
1112
RtpVideoReceiver::RtpVideoReceiver(
12-
std::unique_ptr<webrtc::RtpVideoStreamReceiver2> receiver)
13-
: receiver_(std::move(receiver)) {}
13+
webrtc::Transport* transport,
14+
const webrtc::VideoReceiveStream::Config* config,
15+
webrtc::ReceiveStatistics* rtp_receive_statistics,
16+
webrtc::ProcessThread* process_thread,
17+
webrtc::NackSender* nack_sender)
18+
: receiver_(nullptr), complete_frame_callback_(emscripten::val::null()) {
19+
receiver_ = std::make_unique<webrtc::RtpVideoStreamReceiver2>(
20+
nullptr, webrtc::Clock::GetRealTimeClock(), transport, nullptr, nullptr,
21+
config, rtp_receive_statistics, nullptr, nullptr, process_thread,
22+
nack_sender, nullptr, this, nullptr, nullptr);
23+
webrtc::VideoCodec codec;
24+
codec.codecType = webrtc::VideoCodecType::kVideoCodecH264;
25+
// TODO: Assign payload value dynamically according to server side
26+
// configuration.
27+
receiver_->AddReceiveCodec(127, codec, std::map<std::string, std::string>(),
28+
false);
29+
receiver_->StartReceive();
30+
}
1431

15-
bool RtpVideoReceiver::OnRtpPacket(std::vector<uint8_t> packet) {
16-
RTC_LOG(LS_INFO) << "On RTP packet, size " << packet.size();
32+
bool RtpVideoReceiver::OnRtpPacket(uintptr_t packet_ptr, size_t packet_size) {
33+
const uint8_t* packet = reinterpret_cast<uint8_t*>(packet_ptr);
1734
// TODO: Create `rtp_packet_received` from `packet`.
1835
webrtc::RtpPacketReceived rtp_packet_received;
36+
rtp_packet_received.Parse(packet, packet_size);
1937
receiver_->OnRtpPacket(rtp_packet_received);
2038
return false;
2139
}
40+
41+
void RtpVideoReceiver::OnCompleteFrame(
42+
std::unique_ptr<webrtc::video_coding::EncodedFrame> frame) {
43+
if(complete_frame_callback_.isNull()){
44+
RTC_LOG(LS_WARNING) << "No callback registered for complete frames.";
45+
return;
46+
}
47+
emscripten::val buffer(emscripten::typed_memory_view(
48+
frame->EncodedImage().size(), frame->EncodedImage().data()));
49+
complete_frame_callback_(buffer);
50+
}
51+
52+
void RtpVideoReceiver::SetCompleteFrameCallback(emscripten::val callback) {
53+
complete_frame_callback_ = callback;
54+
}
55+
2256
} // namespace wasm
2357
} // namespace owt

‎talk/owt/sdk/wasm/rtp_video_receiver.h

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,31 @@
55
#ifndef OWT_WASM_RTPVIDEORECEIVER_H_
66
#define OWT_WASM_RTPVIDEORECEIVER_H_
77

8+
#include <emscripten/bind.h>
89
#include "third_party/webrtc/video/rtp_video_stream_receiver2.h"
910

1011
namespace owt {
1112
namespace wasm {
12-
class RtpVideoReceiver {
13+
class RtpVideoReceiver
14+
: public webrtc::RtpVideoStreamReceiver2::OnCompleteFrameCallback {
1315
public:
14-
explicit RtpVideoReceiver(
15-
std::unique_ptr<webrtc::RtpVideoStreamReceiver2> receiver);
16+
explicit RtpVideoReceiver(webrtc::Transport* transport,
17+
const webrtc::VideoReceiveStream::Config* config,
18+
webrtc::ReceiveStatistics* rtp_receive_statistics,
19+
webrtc::ProcessThread* process_thread,
20+
webrtc::NackSender* nack_sender);
1621
virtual ~RtpVideoReceiver() {}
17-
virtual bool OnRtpPacket(std::vector<uint8_t> packet);
22+
virtual bool OnRtpPacket(uintptr_t packet_ptr, size_t packet_size);
23+
24+
// Overrides webrtc::RtpVideoStreamReceiver2::OnCompleteFrameCallback.
25+
void OnCompleteFrame(
26+
std::unique_ptr<webrtc::video_coding::EncodedFrame> frame) override;
27+
28+
void SetCompleteFrameCallback(emscripten::val callback);
1829

1930
private:
2031
std::unique_ptr<webrtc::RtpVideoStreamReceiver2> receiver_;
32+
emscripten::val complete_frame_callback_;
2133
};
2234

2335
} // namespace wasm

‎talk/owt/sdk/wasm/web_transport_session.cc

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
namespace owt {
88
namespace wasm {
9-
9+
1010
bool WebTransportSession::SendRtp(const uint8_t* packet,
1111
size_t length,
1212
const webrtc::PacketOptions& options) {
@@ -16,5 +16,9 @@ bool WebTransportSession::SendRtcp(const uint8_t* packet, size_t length) {
1616
return false;
1717
}
1818

19+
void WebTransportSession::SendNack(
20+
const std::vector<uint16_t>& sequence_numbers,
21+
bool buffering_allowed) {}
22+
1923
} // namespace wasm
2024
} // namespace owt

‎talk/owt/sdk/wasm/web_transport_session.h

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,13 @@
66
#define OWT_WASM_WEBTRANSPORTSESSION_H_
77

88
#include "third_party/webrtc/api/call/transport.h"
9+
#include "third_party/webrtc/modules/include/module_common_types.h"
910

1011
namespace owt {
1112
namespace wasm {
12-
class WebTransportSession : public webrtc::Transport {
13+
14+
class WebTransportSession : public webrtc::Transport,
15+
public webrtc::NackSender {
1316
public:
1417
explicit WebTransportSession() {}
1518
virtual ~WebTransportSession() {}
@@ -19,7 +22,12 @@ class WebTransportSession : public webrtc::Transport {
1922
size_t length,
2023
const webrtc::PacketOptions& options) override;
2124
bool SendRtcp(const uint8_t* packet, size_t length) override;
25+
26+
// Overrides webrtc::NackSender.
27+
void SendNack(const std::vector<uint16_t>& sequence_numbers,
28+
bool buffering_allowed) override;
2229
};
30+
2331
} // namespace wasm
2432
} // namespace owt
2533

0 commit comments

Comments
 (0)