Skip to content

Commit f9eeaaf

Browse files
committed
Add RTCEncodedAudioFrame and RTCEncodedVideoFrame constructors
rdar://149541424 https://bugs.webkit.org/show_bug.cgi?id=291740 Reviewed by Jean-Yves Avenard. We add support for RTCEncodedAudioFrame and RTCEncodedVideoFrame constructors. To match the spec, we do not process RTCEncodedAudioFrame and RTCEncodedVideoFrame when enqueued on a writable if they were not created by the corresponding readable. We use libwebrtc routines to update metadata as part of RTCEncodedAudioFrame and RTCEncodedVideoFrame constructors. We only partially support metadata changes, based on libwebrtc support and skip other potential changes. * LayoutTests/imported/w3c/web-platform-tests/interfaces/webrtc-encoded-transform.idl: * LayoutTests/imported/w3c/web-platform-tests/webrtc-encoded-transform/encoded-frame-clone.https-expected.txt: Added. * LayoutTests/imported/w3c/web-platform-tests/webrtc-encoded-transform/encoded-frame-clone.https.html: Added. * LayoutTests/imported/w3c/web-platform-tests/webrtc-encoded-transform/encoded-frame-constructors.https-expected.txt: Added. * LayoutTests/imported/w3c/web-platform-tests/webrtc-encoded-transform/encoded-frame-constructors.https.html: Added. * LayoutTests/imported/w3c/web-platform-tests/webrtc-encoded-transform/encoded-frame-worker.js: Added. (onrtctransform.process): (onrtctransform): * LayoutTests/imported/w3c/web-platform-tests/webrtc-encoded-transform/idlharness.https.window-expected.txt: * LayoutTests/imported/w3c/web-platform-tests/webrtc-encoded-transform/routines.js: (async createConnectionWithTransform): * Source/ThirdParty/libwebrtc/Configurations/libwebrtc.exp: * Source/WebCore/Headers.cmake: * Source/WebCore/Modules/mediastream/RTCEncodedAudioFrame.cpp: (WebCore::RTCEncodedAudioFrame::create): * Source/WebCore/Modules/mediastream/RTCEncodedAudioFrame.h: * Source/WebCore/Modules/mediastream/RTCEncodedAudioFrame.idl: * Source/WebCore/Modules/mediastream/RTCEncodedVideoFrame.cpp: (WebCore::RTCEncodedVideoFrame::create): * Source/WebCore/Modules/mediastream/RTCEncodedVideoFrame.h: * Source/WebCore/Modules/mediastream/RTCEncodedVideoFrame.idl: * Source/WebCore/Modules/mediastream/RTCRtpScriptTransformer.cpp: (WebCore::RTCRtpScriptTransformer::writable): (WebCore::RTCRtpScriptTransformer::start): * Source/WebCore/Modules/mediastream/RTCRtpTransformableFrame.h: (WebCore::RTCRtpTransformableFrame::isFromTransformer const): (WebCore::RTCRtpTransformableFrame::setTransformer): * Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCRtpTransformableFrame.cpp: (WebCore::LibWebRTCRtpTransformableFrame::setOptions): * Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCRtpTransformableFrame.h: * Source/WebCore/WebCore.xcodeproj/project.pbxproj: Canonical link: https://commits.webkit.org/293951@main
1 parent 8867ffb commit f9eeaaf

22 files changed

+515
-23
lines changed

LayoutTests/imported/w3c/web-platform-tests/interfaces/webrtc-encoded-transform.idl

Lines changed: 35 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -74,14 +74,23 @@ dictionary RTCEncodedVideoFrameMetadata {
7474
octet payloadType;
7575
sequence<unsigned long> contributingSources;
7676
long long timestamp; // microseconds
77+
unsigned long rtpTimestamp;
78+
DOMHighResTimeStamp receiveTime;
79+
DOMHighResTimeStamp captureTime;
80+
DOMHighResTimeStamp senderCaptureTimeOffset;
81+
DOMString mimeType;
82+
};
83+
84+
dictionary RTCEncodedVideoFrameOptions {
85+
RTCEncodedVideoFrameMetadata metadata;
7786
};
7887

7988
// New interfaces to define encoded video and audio frames. Will eventually
8089
// re-use or extend the equivalent defined in WebCodecs.
81-
[Exposed=(Window,DedicatedWorker)]
90+
[Exposed=(Window,DedicatedWorker), Serializable]
8291
interface RTCEncodedVideoFrame {
92+
constructor(RTCEncodedVideoFrame originalFrame, optional RTCEncodedVideoFrameOptions options = {});
8393
readonly attribute RTCEncodedVideoFrameType type;
84-
readonly attribute unsigned long timestamp;
8594
attribute ArrayBuffer data;
8695
RTCEncodedVideoFrameMetadata getMetadata();
8796
};
@@ -91,11 +100,20 @@ dictionary RTCEncodedAudioFrameMetadata {
91100
octet payloadType;
92101
sequence<unsigned long> contributingSources;
93102
short sequenceNumber;
103+
unsigned long rtpTimestamp;
104+
DOMHighResTimeStamp receiveTime;
105+
DOMHighResTimeStamp captureTime;
106+
DOMHighResTimeStamp senderCaptureTimeOffset;
107+
DOMString mimeType;
94108
};
95109

96-
[Exposed=(Window,DedicatedWorker)]
110+
dictionary RTCEncodedAudioFrameOptions {
111+
RTCEncodedAudioFrameMetadata metadata;
112+
};
113+
114+
[Exposed=(Window,DedicatedWorker), Serializable]
97115
interface RTCEncodedAudioFrame {
98-
readonly attribute unsigned long timestamp;
116+
constructor(RTCEncodedAudioFrame originalFrame, optional RTCEncodedAudioFrameOptions options = {});
99117
attribute ArrayBuffer data;
100118
RTCEncodedAudioFrameMetadata getMetadata();
101119
};
@@ -110,19 +128,29 @@ partial interface DedicatedWorkerGlobalScope {
110128
};
111129

112130
[Exposed=DedicatedWorker]
113-
interface RTCRtpScriptTransformer {
131+
interface RTCRtpScriptTransformer : EventTarget {
132+
// Attributes and methods related to the transformer source
114133
readonly attribute ReadableStream readable;
115-
readonly attribute WritableStream writable;
116-
readonly attribute any options;
117134
Promise<unsigned long long> generateKeyFrame(optional DOMString rid);
118135
Promise<undefined> sendKeyFrameRequest();
136+
// Attributes and methods related to the transformer sink
137+
readonly attribute WritableStream writable;
138+
attribute EventHandler onkeyframerequest;
139+
// Attributes for configuring the Javascript code
140+
readonly attribute any options;
119141
};
120142

121143
[Exposed=Window]
122144
interface RTCRtpScriptTransform {
123145
constructor(Worker worker, optional any options, optional sequence<object> transfer);
124146
};
125147

148+
[Exposed=DedicatedWorker]
149+
interface KeyFrameRequestEvent : Event {
150+
constructor(DOMString type, optional DOMString rid);
151+
readonly attribute DOMString? rid;
152+
};
153+
126154
partial interface RTCRtpSender {
127155
Promise<undefined> generateKeyFrame(optional sequence <DOMString> rids);
128156
};
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
2+
3+
PASS Clone frames in sender RTCEncodedVideoFrame
4+
PASS Clone frames in receiver RTCEncodedVideoFrame
5+
PASS Clone frames in sender RTCEncodedAudioFrame
6+
PASS Clone frames in receiver RTCEncodedAudioFrame
7+
Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
<!doctype html>
2+
<html>
3+
<head>
4+
<meta charset="utf-8">
5+
<meta name="timeout" content="long">
6+
<script src="/resources/testharness.js"></script>
7+
<script src="/resources/testharnessreport.js"></script>
8+
</head>
9+
<body>
10+
<video id="video1" autoplay></video>
11+
<video id="video2" autoplay></video>
12+
<script src ="routines.js"></script>
13+
<script>
14+
async function getInboundRTPStats(receiver)
15+
{
16+
const report = await receiver.getStats();
17+
var stats;
18+
report.forEach((statItem) => {
19+
if (statItem.type === "inbound-rtp") {
20+
stats = statItem;
21+
}
22+
});
23+
return stats;
24+
}
25+
26+
async function getDecodedFrameCount(receiver)
27+
{
28+
const stats = await getInboundRTPStats(receiver);
29+
if (!stats)
30+
return 0;
31+
return stats.kind === "video" ? stats.framesDecoded : stats.jitterBufferEmittedCount;
32+
}
33+
34+
async function checkDecodedFrameCountIsIncreasing(test, receiver, testName)
35+
{
36+
const frameCount = await getDecodedFrameCount(receiver);
37+
let counter = 0;
38+
do {
39+
await waitFor(test, 200);
40+
const newFrameCount = await getDecodedFrameCount(receiver);
41+
if (newFrameCount !== frameCount)
42+
break;
43+
} while (++counter < 20);
44+
45+
assert_less_than(counter, 20, "checkDecodedFrameCountIsIncreasing " + (testName ? testName : ""));
46+
}
47+
48+
async function checkDecodedFrameCountIsFreezing(test, receiver, testName)
49+
{
50+
let frameCount = await getDecodedFrameCount(receiver);
51+
let counter = 0;
52+
do {
53+
await waitFor(test, 200);
54+
const newFrameCount = await getDecodedFrameCount(receiver);
55+
if (newFrameCount === frameCount)
56+
break;
57+
frameCount = newFrameCount;
58+
} while (++counter < 20);
59+
60+
assert_less_than(counter, 20, "checkDecodedFrameCountIsFreezing " + (testName ? testName : ""));
61+
}
62+
63+
async function checkUsingClone(test, port, receiver)
64+
{
65+
await checkDecodedFrameCountIsIncreasing(test, receiver, "initial");
66+
67+
port.postMessage("startCloningFrame");
68+
await checkDecodedFrameCountIsFreezing(test, receiver);
69+
70+
port.postMessage("stopCloningFrame");
71+
await checkDecodedFrameCountIsIncreasing(test, receiver, "final");
72+
}
73+
74+
promise_test(async (test) => {
75+
const {sender, receiver, senderPc, receiverPc} = await createConnectionWithTransform(test, 'encoded-frame-worker.js', {video: true});
76+
return checkUsingClone(test, sender.transform.port, receiver);
77+
}, "Clone frames in sender RTCEncodedVideoFrame");
78+
79+
promise_test(async (test) => {
80+
const {sender, receiver, senderPc, receiverPc} = await createConnectionWithTransform(test, 'encoded-frame-worker.js', {video: true});
81+
return checkUsingClone(test, receiver.transform.port, receiver);
82+
}, "Clone frames in receiver RTCEncodedVideoFrame");
83+
84+
promise_test(async (test) => {
85+
const {sender, receiver, senderPc, receiverPc} = await createConnectionWithTransform(test, 'encoded-frame-worker.js', {audio: true});
86+
return checkUsingClone(test, sender.transform.port, receiver);
87+
}, "Clone frames in sender RTCEncodedAudioFrame");
88+
89+
promise_test(async (test) => {
90+
const {sender, receiver, senderPc, receiverPc} = await createConnectionWithTransform(test, 'encoded-frame-worker.js', {audio: true});
91+
return checkUsingClone(test, receiver.transform.port, receiver);
92+
}, "Clone frames in receiver RTCEncodedAudioFrame");
93+
</script>
94+
</body>
95+
</html>
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
2+
3+
PASS Create sender RTCEncodedVideoFrame
4+
PASS Create receiver RTCEncodedVideoFrame
5+
PASS Create sender RTCEncodedAudioFrame
6+
PASS Create receiver RTCEncodedAudioFrame
7+
PASS Sender RTCEncodedVideoFrame - clone
8+
PASS Sender RTCEncodedVideoFrame - clone - rtpTimestamp
9+
PASS Sender RTCEncodedVideoFrame - clone - synchronizationSource
10+
PASS Sender RTCEncodedVideoFrame - clone - frameId
11+
PASS Receiver RTCEncodedVideoFrame - clone
12+
PASS Receiver RTCEncodedVideoFrame - clone - rtpTimestamp
13+
PASS Receiver RTCEncodedVideoFrame - clone - synchronizationSource
14+
PASS Receiver RTCEncodedVideoFrame - clone - frameId
15+
PASS Sender RTCEncodedAudioFrame - clone
16+
PASS Sender RTCEncodedAudioFrame - clone - rtpTimestamp
17+
PASS Receiver RTCEncodedAudioFrame - clone
18+
PASS Receiver RTCEncodedAudioFrame - clone - rtpTimestamp
19+
Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
<!doctype html>
2+
<html>
3+
<head>
4+
<meta charset="utf-8">
5+
<meta name="timeout" content="long">
6+
<script src="/resources/testharness.js"></script>
7+
<script src="/resources/testharnessreport.js"></script>
8+
</head>
9+
<body>
10+
<video id="video1" autoplay></video>
11+
<video id="video2" autoplay></video>
12+
<script src ="routines.js"></script>
13+
<script>
14+
function validateFrame(frame1, frame2, excludedProperties = [])
15+
{
16+
assert_equals(frame1.timestamp, frame2.timestamp);
17+
if (frame1.data) {
18+
assert_array_equals(new Uint8Array(frame1.data), new Uint8Array(frame2.data));
19+
} else
20+
assert_equals(frame2.data, null);
21+
22+
const metadata1 = frame1.getMetadata();
23+
const metadata2 = frame1.getMetadata();
24+
25+
assert_array_equals(Object.getOwnPropertyNames(metadata1), Object.getOwnPropertyNames(metadata2));
26+
for (const property in metadata1) {
27+
if (excludedProperties.indexOf(property) !== -1)
28+
continue;
29+
if (property === "contributingSources" || property === "dependencies")
30+
assert_array_equals(metadata1[property], metadata2[property], property);
31+
else
32+
assert_equals(metadata1[property], metadata2[property], property);
33+
}
34+
}
35+
36+
function validateAudioFrame(frame1, frame2, excludedProperties)
37+
{
38+
assert_equals(frame1.type, frame2.type, excludedProperties);
39+
}
40+
41+
function validateVideoFrame(frame1, frame2, excludedProperties)
42+
{
43+
assert_equals(frame1.type, frame2.type);
44+
validateFrame(frame1, frame2, excludedProperties);
45+
}
46+
47+
async function getAndValidateAudioFrame(t, port, testName)
48+
{
49+
port.postMessage("getFrame");
50+
const frame = await new Promise((resolve, reject) => {
51+
port.onmessage = e => resolve(e.data);
52+
t.step_timeout(() => reject("getFrame timed out"), 1000);
53+
});
54+
55+
assert_true(frame instanceof RTCEncodedAudioFrame);
56+
57+
let clone = new RTCEncodedAudioFrame(frame);
58+
test(() => {
59+
validateAudioFrame(frame, clone);
60+
}, testName + " - clone");
61+
62+
clone = new RTCEncodedAudioFrame(frame, { metadata : { rtpTimestamp : 10 } });
63+
test(() => {
64+
assert_equals(clone.getMetadata().rtpTimestamp, 10);
65+
validateAudioFrame(frame, clone, ["rtpTimestamp"]);
66+
}, testName + " - clone - rtpTimestamp");
67+
}
68+
69+
async function getAndValidateVideoFrame(t, port, testName)
70+
{
71+
port.postMessage("getFrame");
72+
const frame = await new Promise((resolve, reject) => {
73+
port.onmessage = e => resolve(e.data);
74+
t.step_timeout(() => reject("getFrame timed out"), 1000);
75+
});
76+
77+
assert_true(frame instanceof RTCEncodedVideoFrame);
78+
79+
let clone = new RTCEncodedVideoFrame(frame);
80+
81+
test(() => {
82+
validateVideoFrame(frame, clone);
83+
}, testName + " - clone");
84+
85+
clone = new RTCEncodedVideoFrame(frame, { metadata : { rtpTimestamp : 10 } });
86+
test(() => {
87+
assert_equals(clone.getMetadata().rtpTimestamp, 10);
88+
validateAudioFrame(frame, clone, ["rtpTimestamp"]);
89+
}, testName + " - clone - rtpTimestamp");
90+
91+
clone = new RTCEncodedVideoFrame(frame, { metadata : { synchronizationSource : 10 } });
92+
test(() => {
93+
assert_equals(clone.getMetadata().synchronizationSource, 10);
94+
validateAudioFrame(frame, clone, ["synchronizationSource"]);
95+
}, testName + " - clone - synchronizationSource");
96+
97+
clone = new RTCEncodedVideoFrame(frame, { metadata : { frameId : 10 } });
98+
test(() => {
99+
assert_equals(clone.getMetadata().frameId, 10);
100+
validateAudioFrame(frame, clone, ["frameId"]);
101+
}, testName + " - clone - frameId");
102+
}
103+
104+
async function getInboundRTPStats(receiver)
105+
{
106+
const report = await receiver.getStats();
107+
var stats;
108+
report.forEach((statItem) => {
109+
if (statItem.type === "inbound-rtp") {
110+
stats = statItem;
111+
}
112+
});
113+
return stats;
114+
}
115+
116+
async function getDecodedFrameCount(receiver)
117+
{
118+
const stats = await getInboundRTPStats(receiver);
119+
return stats ? stats.framesDecoded : 0;
120+
}
121+
122+
async function checkDecodedFrameCountIsIncreasing(test, receiver, testName)
123+
{
124+
const frameCount = await getDecodedFrameCount(receiver);
125+
let counter = 0;
126+
do {
127+
await waitFor(test, 200);
128+
const newFrameCount = await getDecodedFrameCount(receiver);
129+
if (newFrameCount !== frameCount)
130+
break;
131+
} while (++counter < 20);
132+
133+
assert_less_than(counter, 20, "checkDecodedFrameCountIsIncreasing " + (testName ? testName : ""));
134+
}
135+
136+
async function checkDecodedFrameCountIsFreezing(test, receiver, testName)
137+
{
138+
let frameCount = await getDecodedFrameCount(receiver);
139+
let counter = 0;
140+
do {
141+
await waitFor(test, 200);
142+
const newFrameCount = await getDecodedFrameCount(receiver);
143+
if (newFrameCount === frameCount)
144+
break;
145+
frameCount = newFrameCount;
146+
} while (++counter < 20);
147+
148+
assert_less_than(counter, 20, "checkDecodedFrameCountIsFreezing " + (testName ? testName : ""));
149+
}
150+
151+
promise_test(async (test) => {
152+
const {sender, receiver, senderPc, receiverPc} = await createConnectionWithTransform(test, 'encoded-frame-worker.js', {video: true});
153+
return getAndValidateVideoFrame(test, sender.transform.port, "Sender RTCEncodedVideoFrame");
154+
}, "Create sender RTCEncodedVideoFrame");
155+
156+
promise_test(async (test) => {
157+
const {sender, receiver, senderPc, receiverPc} = await createConnectionWithTransform(test, 'encoded-frame-worker.js', {video: true});
158+
return getAndValidateVideoFrame(test, receiver.transform.port, "Receiver RTCEncodedVideoFrame");
159+
}, "Create receiver RTCEncodedVideoFrame");
160+
161+
promise_test(async (test) => {
162+
const {sender, receiver, senderPc, receiverPc} = await createConnectionWithTransform(test, 'encoded-frame-worker.js', {audio: true});
163+
return getAndValidateAudioFrame(test, sender.transform.port, "Sender RTCEncodedAudioFrame");
164+
}, "Create sender RTCEncodedAudioFrame");
165+
166+
promise_test(async (test) => {
167+
const {sender, receiver, senderPc, receiverPc} = await createConnectionWithTransform(test, 'encoded-frame-worker.js', {audio: true});
168+
return getAndValidateAudioFrame(test, receiver.transform.port, "Receiver RTCEncodedAudioFrame");
169+
}, "Create receiver RTCEncodedAudioFrame");
170+
</script>
171+
</body>
172+
</html>

0 commit comments

Comments
 (0)