diff --git a/Packages/StreamVideo/Runtime/Core/LowLevelClient/RtcSession.cs b/Packages/StreamVideo/Runtime/Core/LowLevelClient/RtcSession.cs index e864a084..6faea31f 100644 --- a/Packages/StreamVideo/Runtime/Core/LowLevelClient/RtcSession.cs +++ b/Packages/StreamVideo/Runtime/Core/LowLevelClient/RtcSession.cs @@ -1391,6 +1391,11 @@ var mediaStreamRecord private IEnumerable GetPublisherTracks(string sdp) { + if (Publisher == null) + { + throw new ArgumentNullException($"{nameof(Publisher)} is null in {nameof(GetPublisherTracks)}"); + } + var transceivers = Publisher.GetTransceivers().ToArray(); //StreamTodo: investigate why this return no results @@ -1436,8 +1441,7 @@ private IEnumerable GetPublisherTracks(string sdp) if (t.Sender.Track.Kind == TrackKind.Video) { - var videoLayers = GetPublisherVideoLayers(Publisher.VideoSender.GetParameters().encodings, - _publisherVideoSettings); + var videoLayers = GetPublisherVideoLayers(Publisher.VideoSender.GetParameters().encodings); trackInfo.Layers.AddRange(videoLayers); #if STREAM_DEBUG_ENABLED @@ -1472,8 +1476,7 @@ private string ReplaceVp8PayloadType(string sdpOffer) return sdpOffer; } - private IEnumerable GetPublisherVideoLayers(IEnumerable encodings, - PublisherVideoSettings videoSettings) + private IEnumerable GetPublisherVideoLayers(IEnumerable encodings) { #if STREAM_DEBUG_ENABLED var sb = new System.Text.StringBuilder(); @@ -1482,7 +1485,7 @@ private IEnumerable GetPublisherVideoLayers(IEnumerable half: 352×288 -> quarter: 176×144 <- We want the smallest resolution to be above 96x96 + public static VideoResolution MinimumSafeTargetResolution => new VideoResolution(704, 576); + public StreamPeerConnection(ILogs logs, StreamPeerType peerType, IEnumerable iceServers, IMediaInputProvider mediaInputProvider, IStreamAudioConfig audioConfig, PublisherVideoSettings publisherVideoSettings, Tracer tracer) @@ -223,6 +226,18 @@ public void Update() } public Task GetStatsReportAsync() => _peerConnection.GetStatsAsync(); + + public PublisherVideoSettings GetLatestVideoSettings() + { + if (_publisherVideoSettings == null) + { + throw new ArgumentNullException($"{nameof(_publisherVideoSettings)} is null in {nameof(GetLatestVideoSettings)}"); + } + + _publisherVideoSettings.MaxResolution = PublisherTargetResolution; + _publisherVideoSettings.FrameRate = PublisherTargetFrameRate; + return _publisherVideoSettings; + } public void Dispose() { @@ -275,6 +290,43 @@ public void Dispose() private const string VideoCodecKeyH264 = "h264"; private const string VideoCodecKeyVP8 = "vp8"; private const string AudioCodecKeyRed = "red"; + + private VideoResolution PublisherTargetResolution + { + get + { + if (_mediaInputProvider.VideoInput != null) + { + var preferred = new VideoResolution(_mediaInputProvider.VideoInput.width, + _mediaInputProvider.VideoInput.height); + + // Requesting too small resolution can cause crashes in the Android video encoder + // The target resolution is used to calculate 3 layers of video encoding (full, half, quarter) + // For very small values of quarter resolution (not sure exact but ~100x100) the encoder crashes + + if (preferred.Width < MinimumSafeTargetResolution.Width || + preferred.Height < MinimumSafeTargetResolution.Height) + { + return MinimumSafeTargetResolution; + } + } + + return _publisherVideoSettings.MaxResolution; + } + } + + private uint PublisherTargetFrameRate + { + get + { + if (_mediaInputProvider.VideoInput != null) + { + return (uint)_mediaInputProvider.VideoInput.requestedFPS; + } + + return 30; + } + } private readonly RTCPeerConnection _peerConnection; @@ -282,7 +334,9 @@ public void Dispose() private readonly StreamPeerType _peerType; private readonly IMediaInputProvider _mediaInputProvider; private readonly IStreamAudioConfig _audioConfig; - private readonly PublisherVideoSettings _publisherVideoSettings; + + //StreamTOdo: Finish implementation. This currently is not exposed to the user + it doesn't take into account VideoInput resolution. We need publisher resolution to match with WebCamTexture size + private readonly PublisherVideoSettings _publisherVideoSettings; private readonly Tracer _tracer; private readonly List _pendingIceCandidates = new List(); @@ -360,6 +414,7 @@ private void OnTrack(RTCTrackEvent trackEvent) var streamIds = trackEvent.Streams != null && trackEvent.Streams.Any() ? string.Join(",", trackEvent.Streams.Select(s => s.Id)) : ""; + var isEnabled = trackEvent.Track.Enabled; if (!string.IsNullOrEmpty(streamIds)) { @@ -587,6 +642,10 @@ private static IEnumerable GetVideoEncodingParameters( break; case TrackKind.Video: + + //StreamTodo: construct fewer layer when the target resolution is small. Android video encoder crashes when requesting very small resolution + // We're currently forcing the smallest safe resolution that the user can request so that the quarter layer doesn't reach too small resolution + // But we should allow setting small resolution and just produce fewer layers in that case var fullQuality = new RTCRtpEncodingParameters { @@ -631,17 +690,6 @@ private static IEnumerable GetVideoEncodingParameters( } } - private VideoResolution GetPublisherResolution() - { - if (_mediaInputProvider.VideoInput != null) - { - var maxResolution = _publisherVideoSettings.MaxResolution; - return new VideoResolution(maxResolution.Width, maxResolution.Height); - } - - return VideoResolution.Res_1080p; - } - private VideoStreamTrack CreatePublisherVideoTrack() { if (_mediaInputProvider.VideoInput == null) @@ -653,7 +701,7 @@ private VideoStreamTrack CreatePublisherVideoTrack() var gfxType = SystemInfo.graphicsDeviceType; var format = WebRTC.GetSupportedRenderTextureFormat(gfxType); - var res = GetPublisherResolution(); + var res = PublisherTargetResolution; _publisherVideoTrackTexture = new RenderTexture((int)res.Width, (int)res.Height, 0, format); #if STREAM_DEBUG_ENABLED