Skip to content

Commit d1ad3a5

Browse files
DjuffinDjuffin
authored
Merge pull request #709 from aboba/main
Add g2g latency chart and decoder hw acceleration control
2 parents 1476a30 + e6a3654 commit d1ad3a5

File tree

3 files changed

+152
-36
lines changed

3 files changed

+152
-36
lines changed

‎samples/encode-decode-worker/index.html

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
<meta id="theme-color" name="theme-color" content="#ffffff">
1212
<base target="_blank">
1313

14-
<title>WebCodecs in Worker</title>
14+
<title>WebCodecs in Worker + RVFC</title>
1515
<link rel="stylesheet" href="css/main.css">
1616

1717
<style>
@@ -57,7 +57,7 @@
5757

5858
<body>
5959
<div>
60-
<h2>WebCodecs in Worker</h2>
60+
<h2>WebCodecs in Worker + RVFC</h2>
6161
</div>
6262

6363
<textarea id="vLog" style="width: 640px; height: 360px"></textarea>
@@ -89,13 +89,23 @@ <h2>WebCodecs in Worker</h2>
8989
<label for="AV1">AV1</label><br>
9090
</div>
9191

92-
<div id="hwButtons">
93-
<p>Hardware Acceleration Preference:</p>
94-
<input type="radio" id="hw" name="hwAccel" value="prefer-hardware" onchange="getHwValue(this)">
92+
<div id="encHwButtons">
93+
<p>Encoder Hardware Acceleration Preference:</p>
94+
<input type="radio" id="hw" name="encHwAccel" value="prefer-hardware" onchange="getEncHwValue(this)">
9595
<label for="hw">Prefer Hardware</label><br>
96-
<input type="radio" id="sw" name="hwAccel" value="prefer-software" onchange="getHwValue(this)">
96+
<input type="radio" id="sw" name="encHwAccel" value="prefer-software" onchange="getEncHwValue(this)">
9797
<label for="sw">Prefer Software</label><br>
98-
<input type="radio" id="no-pref" name="hwAccel" value="no-preference" checked="checked" onchange="getHwValue(this)">
98+
<input type="radio" id="no-pref" name="encHwAccel" value="no-preference" checked="checked" onchange="getEncHwValue(this)">
99+
<label for="no-pref">No Preference</label><br>
100+
</div>
101+
102+
<div id="decHwButtons">
103+
<p>Decoder Hardware Acceleration Preference:</p>
104+
<input type="radio" id="hw" name="decHwAccel" value="prefer-hardware" onchange="getDecHwValue(this)">
105+
<label for="hw">Prefer Hardware</label><br>
106+
<input type="radio" id="sw" name="decHwAccel" value="prefer-software" onchange="getDecHwValue(this)">
107+
<label for="sw">Prefer Software</label><br>
108+
<input type="radio" id="no-pref" name="decHwAccel" value="no-preference" checked="checked" onchange="getDecHwValue(this)">
99109
<label for="no-pref">No Preference</label><br>
100110
</div>
101111

@@ -143,6 +153,8 @@ <h2>WebCodecs in Worker</h2>
143153
<label for="eightK">8K</label><br>
144154
</div>
145155

156+
<div id="chart2_div" style="width: 900px; height: 500px;"></div>
157+
146158
<div class="select">
147159
<label for="videoSource">Video source: </label><select id="videoSource"></select>
148160
</div>
@@ -154,6 +166,7 @@ <h2>WebCodecs in Worker</h2>
154166
<br/></br>
155167
<button id="connect">Start</button>
156168
<button id="stop">Stop</button>
169+
<script src="https://www.gstatic.com/charts/loader.js"></script>
157170
<script src="js/main.js"></script>
158171
</body>
159172
</html>

‎samples/encode-decode-worker/js/main.js

Lines changed: 116 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,19 +6,28 @@ let stopped = false;
66
let preferredCodec ="VP8";
77
let mode = "L1T3";
88
let latencyPref = "realtime", bitPref = "variable";
9-
let hw = "no-preference";
9+
let encHw = "no-preference", decHw = "no-preference";
1010
let streamWorker;
1111
let inputStream, outputStream;
12-
let videoSource;
12+
let metrics = {
13+
all: [],
14+
};
15+
let e2e = {
16+
all: [],
17+
};
18+
1319
const rate = document.querySelector('#rate');
1420
const connectButton = document.querySelector('#connect');
1521
const stopButton = document.querySelector('#stop');
1622
const codecButtons = document.querySelector('#codecButtons');
1723
const resButtons = document.querySelector('#resButtons');
1824
const modeButtons = document.querySelector('#modeButtons');
19-
const hwButtons = document.querySelector('#hwButtons');
25+
const decHwButtons = document.querySelector('#decHwButtons');
26+
const encHwButtons = document.querySelector('#encHwButtons');
27+
const chart2_div = document.getElementById('chart2_div');
2028
const videoSelect = document.querySelector('select#videoSource');
2129
const selectors = [videoSelect];
30+
chart2_div.style.display = "none";
2231
connectButton.disabled = false;
2332
stopButton.disabled = true;
2433

@@ -36,6 +45,34 @@ const eightKConstraints = {video: {width: {min: 7680}, height: {min: 4320}}};
3645

3746
let constraints = qvgaConstraints;
3847

48+
function metrics_update(data) {
49+
metrics.all.push(data);
50+
}
51+
52+
function metrics_report() {
53+
metrics.all.sort((a, b) => {
54+
return (100000 * (a.mediaTime - b.mediaTime) + a.output - b.output);
55+
});
56+
const len = metrics.all.length;
57+
let j = 0;
58+
for (let i = 0; i < len ; i++ ) {
59+
if (metrics.all[i].output == 1) {
60+
const frameno = metrics.all[i].presentedFrames;
61+
const g2g = metrics.all[i].expectedDisplayTime - metrics.all[i-1].captureTime;
62+
const mediaTime = metrics.all[i].mediaTime;
63+
const captureTime = metrics.all[i-1].captureTime;
64+
const expectedDisplayTime = metrics.all[i].expectedDisplayTime;
65+
const delay = metrics.all[i].expectedDisplayTime - metrics.all[i-1].expectedDisplayTime;
66+
const data = [frameno, g2g];
67+
e2e.all.push(data);
68+
}
69+
}
70+
// addToEventLog('Data dump: ' + JSON.stringify(e2e.all));
71+
return {
72+
count: e2e.all.length
73+
};
74+
}
75+
3976
function addToEventLog(text, severity = 'info') {
4077
let log = document.querySelector('textarea');
4178
log.value += 'log-' + severity + ': ' + text + '\n';
@@ -132,15 +169,21 @@ function getModeValue(radio) {
132169
addToEventLog('Mode selected: ' + mode);
133170
}
134171

135-
function getHwValue(radio) {
136-
hw = radio.value;
137-
addToEventLog('Hardware Acceleration preference: ' + hw);
172+
function getDecHwValue(radio) {
173+
decHw = radio.value;
174+
addToEventLog('Decoder Hardware Acceleration preference: ' + decHw);
175+
}
176+
177+
function getEncHwValue(radio) {
178+
encHw = radio.value;
179+
addToEventLog('Encoder Hardware Acceleration preference: ' + encHw);
138180
}
139181

140182
function stop() {
141183
stopped = true;
142184
stopButton.disabled = true;
143185
connectButton.disabled = true;
186+
chart2_div.style.display = "initial";
144187
streamWorker.postMessage({ type: "stop" });
145188
try {
146189
inputStream.cancel();
@@ -159,13 +202,15 @@ function stop() {
159202
document.addEventListener('DOMContentLoaded', async function(event) {
160203
if (stopped) return;
161204
addToEventLog('DOM Content Loaded');
162-
163-
if (typeof MediaStreamTrackProcessor === 'undefined' ||
164-
typeof MediaStreamTrackGenerator === 'undefined') {
165-
addToEventLog('Your browser does not support the experimental Mediacapture-transform API.\n' +
166-
'Please launch with the --enable-blink-features=WebCodecs,MediaStreamInsertableStreams flag','fatal');
205+
206+
// Need to support standard mediacapture-transform implementations
207+
208+
if (typeof MediaStreamTrackProcessor === 'undefined' ||
209+
typeof MediaStreamTrackGenerator === 'undefined') {
210+
addToEventLog('Your browser does not support the MSTP and MSTG APIs.', 'fatal');
167211
return;
168-
}
212+
}
213+
169214
try {
170215
gotDevices(await navigator.mediaDevices.enumerateDevices());
171216
} catch (e) {
@@ -179,9 +224,32 @@ document.addEventListener('DOMContentLoaded', async function(event) {
179224
// Create a new worker.
180225
streamWorker = new Worker("js/stream_worker.js");
181226
addToEventLog('Worker created.');
182-
// Print messages from the worker in the text area.
227+
183228
streamWorker.addEventListener('message', function(e) {
184-
addToEventLog('Worker msg: ' + e.data.text, e.data.severity);
229+
if (e.data.severity != 'chart'){
230+
addToEventLog('Worker msg: ' + e.data.text, e.data.severity);
231+
} else {
232+
// draw the glass-glass latency chart
233+
metrics_report();
234+
google.charts.load('current', {'packages':['corechart']});
235+
google.charts.setOnLoadCallback(() => {
236+
let data = new google.visualization.DataTable();
237+
// addToEventLog('Data dump: ' + JSON.stringify(e2e.all));
238+
data.addColumn('number', 'Frame Number');
239+
data.addColumn('number', 'Glass-Glass Latency (ms)');
240+
data.addRows(e2e.all);
241+
let options = {
242+
width: 900,
243+
height: 500,
244+
title: 'Glass-Glass Latency (ms) versus Frame Number',
245+
haxis: {title: 'Frame Number'},
246+
vaxis: {title: 'Glass-Glass Latency'},
247+
legend: 'none'
248+
};
249+
let chart = new google.visualization.ScatterChart(chart2_div);
250+
chart.draw(data, options);
251+
});
252+
}
185253
}, false);
186254

187255
stopButton.onclick = () => {
@@ -192,7 +260,8 @@ document.addEventListener('DOMContentLoaded', async function(event) {
192260
connectButton.onclick = () => {
193261
connectButton.disabled = true;
194262
stopButton.disabled = false;
195-
hwButtons.style.display = "none";
263+
decHwButtons.style.display = "none";
264+
encHwButtons.style.display = "none";
196265
prefButtons.style.display = "none";
197266
bitButtons.style.display = "none";
198267
codecButtons.style.display = "none";
@@ -226,6 +295,36 @@ document.addEventListener('DOMContentLoaded', async function(event) {
226295
outputStream = generator.writable;
227296
document.getElementById('outputVideo').srcObject = new MediaStream([generator]);
228297

298+
// Initialize variables
299+
let paint_count = 0;
300+
let start_time = 0.0;
301+
302+
const recordOutputFrames = (now, metadata) => {
303+
metadata.output = 1.;
304+
metadata.time = now;
305+
if( start_time == 0.0 ) start_time = now;
306+
let elapsed = (now - start_time)/1000.;
307+
let fps = (++paint_count / elapsed).toFixed(3);
308+
metadata.fps = fps;
309+
metrics_update(metadata);
310+
outputVideo.requestVideoFrameCallback(recordOutputFrames);
311+
};
312+
313+
outputVideo.requestVideoFrameCallback(recordOutputFrames);
314+
315+
const recordInputFrames = (now, metadata) => {
316+
metadata.output = 0;
317+
metadata.time = now;
318+
if( start_time == 0.0 ) start_time = now;
319+
let elapsed = (now - start_time)/1000.;
320+
let fps = (++paint_count / elapsed).toFixed(3);
321+
metadata.fps = fps;
322+
metrics_update(metadata);
323+
inputVideo.requestVideoFrameCallback(recordInputFrames);
324+
};
325+
326+
inputVideo.requestVideoFrameCallback(recordInputFrames);
327+
229328
//Create video Encoder configuration
230329
const vConfig = {
231330
keyInterval: keygap,
@@ -244,7 +343,8 @@ document.addEventListener('DOMContentLoaded', async function(event) {
244343
codec: preferredCodec,
245344
width: ts.width/vConfig.resolutionScale,
246345
height: ts.height/vConfig.resolutionScale,
247-
hardwareAcceleration: hw,
346+
hardwareAcceleration: encHw,
347+
decHwAcceleration: decHw,
248348
bitrate: rate,
249349
framerate: ts.frameRate/vConfig.framerateScale,
250350
keyInterval: vConfig.keyInterval,

‎samples/encode-decode-worker/js/stream_worker.js

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -130,10 +130,10 @@ class pipeline {
130130
this.decoder.configure(decoderSupport.config);
131131
self.postMessage({text: 'Decoder successfully configured:\n' + JSON.stringify(decoderSupport.config)});
132132
} else {
133-
self.postMessage({severity: 'fatal', text: 'Config not supported:\n' + JSON.stringify(decoderSupport.config)});
133+
self.postMessage({severity: 'fatal', text: 'Decoder Config not supported:\n' + JSON.stringify(decoderSupport.config)});
134134
}
135135
} catch (e) {
136-
self.postMessage({severity: 'fatal', text: `Configuration error: ${e.message}`});
136+
self.postMessage({severity: 'fatal', text: `Decoder Configuration error: ${e.message}`});
137137
}
138138
} else {
139139
try {
@@ -161,6 +161,7 @@ class pipeline {
161161
this.encoder = encoder = new VideoEncoder({
162162
output: (chunk, cfg) => {
163163
if (cfg.decoderConfig) {
164+
cfg.decoderConfig.hardwareAcceleration = config.decHwAcceleration;
164165
const decoderConfig = JSON.stringify(cfg.decoderConfig);
165166
self.postMessage({text: 'Configuration: ' + decoderConfig});
166167
const configChunk =
@@ -197,15 +198,15 @@ class pipeline {
197198
}
198199
});
199200
try {
200-
const encoderSupport = await VideoEncoder.isConfigSupported(config);
201-
if (encoderSupport.supported) {
201+
const encoderSupport = await VideoEncoder.isConfigSupported(config);
202+
if (encoderSupport.supported) {
202203
this.encoder.configure(encoderSupport.config);
203204
self.postMessage({text: 'Encoder successfully configured:\n' + JSON.stringify(encoderSupport.config)});
204205
} else {
205206
self.postMessage({severity: 'fatal', text: 'Config not supported:\n' + JSON.stringify(encoderSupport.config)});
206207
}
207208
} catch (e) {
208-
self.postMessage({severity: 'fatal', text: `Configuration error: ${e.message}`});
209+
self.postMessage({severity: 'fatal', text: `Configuration error: ${e.message}`});
209210
}
210211
},
211212
transform(frame, controller) {
@@ -229,16 +230,18 @@ class pipeline {
229230
}
230231

231232
stop() {
232-
const encqueue_stats = encqueue_report();
233-
const decqueue_stats = decqueue_report();
234-
self.postMessage({text: 'Encoder Queue report: ' + JSON.stringify(encqueue_stats)});
235-
self.postMessage({text: 'Decoder Queue report: ' + JSON.stringify(decqueue_stats)});
236-
if (stopped) return;
237-
stopped = true;
238-
this.stopped = true;
239-
self.postMessage({text: 'stop() called'});
240233
if (encoder.state != "closed") encoder.close();
241234
if (decoder.state != "closed") decoder.close();
235+
stopped = true;
236+
this.stopped = true;
237+
const len = encqueue_aggregate.all.length;
238+
if (len > 1) {
239+
const encqueue_stats = encqueue_report();
240+
const decqueue_stats = decqueue_report();
241+
self.postMessage({severity: 'chart'});
242+
self.postMessage({text: 'Encoder Queue report: ' + JSON.stringify(encqueue_stats)});
243+
self.postMessage({text: 'Decoder Queue report: ' + JSON.stringify(decqueue_stats)});
244+
}
242245
self.postMessage({text: 'stop(): frame, encoder and decoder closed'});
243246
return;
244247
}

0 commit comments

Comments
 (0)