Skip to content

Commit 5b037a6

Browse files
BrazolBrazol
authored
fix(llc): rewrite of the rtc video layers creation logic - fix for video quality issues (#754)
* rewrite of the rtc video layers creation logic * typo fix
1 parent a1bdb6b commit 5b037a6

File tree

4 files changed

+155
-104
lines changed

4 files changed

+155
-104
lines changed

‎packages/stream_video/lib/src/webrtc/codecs_helper.dart‎

Lines changed: 90 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -1,97 +1,114 @@
1-
import 'dart:math' as math;
1+
// import 'dart:math' as math;
22

3+
import 'dart:math';
4+
5+
import 'package:collection/collection.dart';
36
import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc;
47

5-
import '../logger/stream_log.dart';
68
import 'model/rtc_video_dimension.dart';
79
import 'model/rtc_video_parameters.dart';
810

9-
// 16:9 default
10-
const _defaultSimulcast_16x9 = {
11-
'f': RtcVideoParametersPresets.h720_16x9,
12-
'h': RtcVideoParametersPresets.h360_16x9,
13-
'q': RtcVideoParametersPresets.h180_16x9,
14-
};
15-
16-
// 4:3 default
17-
const _defaultSimulcast_4x3 = {
18-
'f': RtcVideoParametersPresets.h720_4x3,
19-
'h': RtcVideoParametersPresets.h360_4x3,
20-
'q': RtcVideoParametersPresets.h180_4x3,
21-
};
22-
23-
List<rtc.RTCRtpEncoding> computeVideoEncodings({
24-
required RtcVideoDimension dimension,
25-
required bool isScreenShare,
11+
List<rtc.RTCRtpEncoding> findOptimalVideoLayers({
12+
required RtcVideoDimension dimensions,
13+
RtcVideoParameters targetResolution = RtcVideoParametersPresets.h720_16x9,
2614
}) {
27-
final presets = _presetsForDimension(
28-
isScreenShare: isScreenShare,
29-
dimension: dimension,
15+
final optimalVideoLayers = <rtc.RTCRtpEncoding>[];
16+
17+
final maxBitrate = getComputedMaxBitrate(
18+
targetResolution,
19+
dimensions.width,
20+
dimensions.height,
3021
);
31-
presets.forEach((rid, preset) {
32-
streamLog.v(
33-
'SV:RtcManager',
34-
() => '[publishVideoTrack] #$rid; preset: $preset',
22+
23+
var downscaleFactor = 1;
24+
for (final rid in ['f', 'h', 'q'].reversed) {
25+
optimalVideoLayers.insert(
26+
0,
27+
rtc.RTCRtpEncoding(
28+
rid: rid,
29+
scaleResolutionDownBy: downscaleFactor.toDouble(),
30+
maxFramerate: 30,
31+
maxBitrate: (maxBitrate / downscaleFactor).round(),
32+
),
3533
);
36-
});
3734

38-
return encodingsFromPresets(dimension, presets: presets);
35+
downscaleFactor *= 2;
36+
}
37+
38+
return withSimulcastConstraints(
39+
dimensions: dimensions,
40+
optimalVideoLayers: optimalVideoLayers,
41+
);
3942
}
4043

41-
Map<String, RtcVideoParameters> _presetsForDimension({
42-
required bool isScreenShare,
43-
required RtcVideoDimension dimension,
44-
}) {
45-
if (isScreenShare) {
46-
return _defaultSimulcast_16x9;
44+
int getComputedMaxBitrate(
45+
RtcVideoParameters targetResolution,
46+
int currentWidth,
47+
int currentHeight,
48+
) {
49+
// if the current resolution is lower than the target resolution,
50+
// we want to proportionally reduce the target bitrate
51+
final targetWidth = targetResolution.dimension.width;
52+
final targetHeight = targetResolution.dimension.height;
53+
54+
if (currentWidth < targetWidth || currentHeight < targetHeight) {
55+
final currentPixels = currentWidth * currentHeight;
56+
final targetPixels = targetWidth * targetHeight;
57+
final reductionFactor = currentPixels / targetPixels;
58+
59+
return (targetResolution.encoding.maxBitrate * reductionFactor).round();
4760
}
4861

49-
final aspectRatio = dimension.aspect();
50-
streamLog.v(
51-
'SV:RtcManager',
52-
() => '[publishVideoTrack] aspectRatio: $aspectRatio',
53-
);
54-
if (_is16x9ratio(aspectRatio)) {
55-
streamLog.v(
56-
'SV:RtcManager',
57-
() => '[publishVideoTrack] defaultSimulcast_16x9',
58-
);
59-
return _defaultSimulcast_16x9;
62+
return targetResolution.encoding.maxBitrate;
63+
}
64+
65+
List<rtc.RTCRtpEncoding> withSimulcastConstraints({
66+
required RtcVideoDimension dimensions,
67+
required List<rtc.RTCRtpEncoding> optimalVideoLayers,
68+
}) {
69+
var layers = <rtc.RTCRtpEncoding>[];
70+
71+
final size = max(dimensions.width, dimensions.height);
72+
if (size <= 320) {
73+
// provide only one layer 320x240 (q), the one with the highest quality
74+
layers = optimalVideoLayers.where((layer) => layer.rid == 'f').toList();
75+
} else if (size <= 640) {
76+
// provide two layers, 160x120 (q) and 640x480 (h)
77+
layers = optimalVideoLayers.where((layer) => layer.rid != 'h').toList();
78+
} else {
79+
// provide three layers for sizes > 640x480
80+
layers = optimalVideoLayers;
6081
}
61-
streamLog.v(
62-
'SV:RtcManager',
63-
() => '[publishVideoTrack] defaultSimulcast_4x3',
64-
);
65-
return _defaultSimulcast_4x3;
82+
83+
final ridMapping = ['q', 'h', 'f'];
84+
return layers
85+
.mapIndexed(
86+
(index, layer) => rtc.RTCRtpEncoding(
87+
rid: ridMapping[index],
88+
scaleResolutionDownBy: layer.scaleResolutionDownBy,
89+
maxFramerate: layer.maxFramerate,
90+
maxBitrate: layer.maxBitrate,
91+
),
92+
)
93+
.toList();
6694
}
6795

68-
List<rtc.RTCRtpEncoding> encodingsFromPresets(
69-
RtcVideoDimension dimension, {
70-
required Map<String, RtcVideoParameters> presets,
96+
List<rtc.RTCRtpEncoding> findOptimalScreenSharingLayers({
97+
required RtcVideoDimension dimensions,
98+
RtcVideoParameters targetResolution = RtcVideoParametersPresets.h1080_16x9,
7199
}) {
72-
final result = <rtc.RTCRtpEncoding>[];
73-
74-
presets.forEach((rid, preset) {
75-
final double scaleResolutionDownBy =
76-
math.max(1, dimension.min() / preset.dimension.min());
77-
streamLog.v(
78-
'SV:RtcManager',
79-
() => '[publishVideoTrack] #$rid; scaleResolutionDownBy: '
80-
'$scaleResolutionDownBy',
81-
);
82-
result.add(
100+
final optimalVideoLayers = <rtc.RTCRtpEncoding>[];
101+
102+
for (final rid in ['f', 'h', 'q'].reversed) {
103+
optimalVideoLayers.insert(
104+
0,
83105
rtc.RTCRtpEncoding(
84106
rid: rid,
85-
scaleResolutionDownBy: scaleResolutionDownBy,
86-
maxFramerate: preset.encoding.maxFramerate,
87-
maxBitrate: preset.encoding.maxBitrate,
107+
maxFramerate: targetResolution.encoding.maxFramerate,
108+
maxBitrate: targetResolution.encoding.maxBitrate,
88109
),
89110
);
90-
});
91-
return result;
92-
}
111+
}
93112

94-
bool _is16x9ratio(double aspectRatio) {
95-
return (aspectRatio - RtcVideoDimensionHelpers.aspect_16x9).abs() <
96-
(aspectRatio - RtcVideoDimensionHelpers.aspect_4x3).abs();
113+
return optimalVideoLayers;
97114
}

‎packages/stream_video/lib/src/webrtc/media/screen_share_constraints.dart‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ class ScreenShareConstraints extends VideoConstraints {
99
this.captureScreenAudio = false,
1010
String? sourceId,
1111
super.maxFrameRate,
12-
super.params = RtcVideoParametersPresets.h720_16x9,
12+
super.params = RtcVideoParametersPresets.h1080_16x9,
1313
}) : super(deviceId: sourceId);
1414

1515
ScreenShareConstraints.from({

‎packages/stream_video/lib/src/webrtc/model/rtc_video_parameters.dart‎

Lines changed: 57 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -50,52 +50,90 @@ class RtcVideoParameters
5050
}
5151

5252
extension RtcVideoParametersPresets on RtcVideoParameters {
53+
static const int k1080pBitrate = 2700000;
54+
static const int k720pBitrate = 1250000;
55+
static const int k540pBitrate = 700000;
56+
static const int k360pBitrate = 400000;
57+
static const int k180pBitrate = 140000;
58+
5359
// 16:9 Presets
54-
static const h180_16x9 = RtcVideoParameters(
55-
dimension: RtcVideoDimensionPresets.h180_169,
60+
static const h1080_16x9 = RtcVideoParameters(
61+
dimension: RtcVideoDimensionPresets.h1080_169,
5662
encoding: RtcVideoEncoding(
57-
maxBitrate: 120 * 1000,
58-
maxFramerate: 15,
63+
maxBitrate: k1080pBitrate,
64+
maxFramerate: 30,
65+
),
66+
);
67+
68+
static const h720_16x9 = RtcVideoParameters(
69+
dimension: RtcVideoDimensionPresets.h720_169,
70+
encoding: RtcVideoEncoding(
71+
maxBitrate: k720pBitrate,
72+
maxFramerate: 30,
73+
),
74+
);
75+
76+
static const h540_16x9 = RtcVideoParameters(
77+
dimension: RtcVideoDimensionPresets.h540_169,
78+
encoding: RtcVideoEncoding(
79+
maxBitrate: k540pBitrate,
80+
maxFramerate: 30,
5981
),
6082
);
6183

6284
static const h360_16x9 = RtcVideoParameters(
6385
dimension: RtcVideoDimensionPresets.h360_169,
6486
encoding: RtcVideoEncoding(
65-
maxBitrate: 300 * 1000,
66-
maxFramerate: 20,
87+
maxBitrate: k360pBitrate,
88+
maxFramerate: 30,
6789
),
6890
);
6991

70-
static const h720_16x9 = RtcVideoParameters(
71-
dimension: RtcVideoDimensionPresets.h720_169,
92+
static const h180_16x9 = RtcVideoParameters(
93+
dimension: RtcVideoDimensionPresets.h180_169,
7294
encoding: RtcVideoEncoding(
73-
maxBitrate: 2 * 1000 * 1000,
95+
maxBitrate: k180pBitrate,
7496
maxFramerate: 30,
7597
),
7698
);
7799

78-
// 4:3 presets
79-
static const h180_4x3 = RtcVideoParameters(
80-
dimension: RtcVideoDimensionPresets.h180_43,
100+
// 4:3 Presets
101+
static const h1080_4x3 = RtcVideoParameters(
102+
dimension: RtcVideoDimensionPresets.h1080_43,
103+
encoding: RtcVideoEncoding(
104+
maxBitrate: (k1080pBitrate * 0.75) ~/ 1,
105+
maxFramerate: 30,
106+
),
107+
);
108+
109+
static const h720_4x3 = RtcVideoParameters(
110+
dimension: RtcVideoDimensionPresets.h720_43,
111+
encoding: RtcVideoEncoding(
112+
maxBitrate: (k720pBitrate * 0.75) ~/ 1,
113+
maxFramerate: 30,
114+
),
115+
);
116+
117+
static const h540_4x3 = RtcVideoParameters(
118+
dimension: RtcVideoDimensionPresets.h540_43,
81119
encoding: RtcVideoEncoding(
82-
maxBitrate: 100 * 1000,
83-
maxFramerate: 15,
120+
maxBitrate: (k540pBitrate * 0.75) ~/ 1,
121+
maxFramerate: 30,
84122
),
85123
);
86124

87125
static const h360_4x3 = RtcVideoParameters(
88126
dimension: RtcVideoDimensionPresets.h360_43,
89127
encoding: RtcVideoEncoding(
90-
maxBitrate: 225 * 1000,
91-
maxFramerate: 20,
128+
maxBitrate: (k360pBitrate * 0.75) ~/ 1,
129+
maxFramerate: 30,
92130
),
93131
);
94132

95-
static const h720_4x3 = RtcVideoParameters(
96-
dimension: RtcVideoDimensionPresets.h720_43,
133+
static const h180_4x3 = RtcVideoParameters(
134+
dimension: RtcVideoDimensionPresets.h180_43,
97135
encoding: RtcVideoEncoding(
98-
maxBitrate: 1 * 500 * 1000,
136+
maxBitrate: (k180pBitrate * 0.75) ~/ 1,
99137
maxFramerate: 30,
100138
),
101139
);

‎packages/stream_video/lib/src/webrtc/rtc_manager.dart‎

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -411,26 +411,22 @@ extension PublisherRtcManager on RtcManager {
411411
if (track.trackType == SfuTrackType.screenShare) {
412412
final physicalSize =
413413
WidgetsBinding.instance.platformDispatcher.views.first.physicalSize;
414+
414415
final screenDimension = RtcVideoDimension(
415416
width: physicalSize.width.toInt(),
416417
height: physicalSize.height.toInt(),
417418
);
418419

419420
_logger.v(() => '[publishVideoTrack] screenDimension: $screenDimension');
420421

421-
// Simulcast is not supported for screen share tracks, but all three are required for Android to work.
422-
encodings = codecs.encodingsFromPresets(
423-
screenDimension,
424-
presets: {
425-
'f': track.mediaConstraints.params,
426-
'h': track.mediaConstraints.params,
427-
'q': track.mediaConstraints.params,
428-
},
422+
encodings = codecs.findOptimalScreenSharingLayers(
423+
dimensions: screenDimension,
424+
targetResolution: track.mediaConstraints.params,
429425
);
430426
} else {
431-
encodings = codecs.computeVideoEncodings(
432-
dimension: dimension,
433-
isScreenShare: track.trackType == SfuTrackType.screenShare,
427+
encodings = codecs.findOptimalVideoLayers(
428+
dimensions: dimension,
429+
targetResolution: track.mediaConstraints.params,
434430
);
435431
}
436432

0 commit comments

Comments
 (0)