Merge pull request #176 from GetStream/feature/uni-130-user-defined-publisher-resolution-is-not-respected

Feature/uni 130 user defined publisher resolution is not respected

Author: sierpinskid

Packages/StreamVideo/Runtime/Core/LowLevelClient/RtcSession.cs

@@ -1391,6 +1391,11 @@ var mediaStreamRecord
 
         private IEnumerable<TrackInfo> GetPublisherTracks(string sdp)
         {
+            if (Publisher == null)
+            {
+                throw new ArgumentNullException($"{nameof(Publisher)} is null in {nameof(GetPublisherTracks)}");
+            }
+
             var transceivers = Publisher.GetTransceivers().ToArray();
 
             //StreamTodo: investigate why this return no results
@@ -1436,8 +1441,7 @@ private IEnumerable<TrackInfo> GetPublisherTracks(string sdp)
 
                 if (t.Sender.Track.Kind == TrackKind.Video)
                 {
-                    var videoLayers = GetPublisherVideoLayers(Publisher.VideoSender.GetParameters().encodings,
-                        _publisherVideoSettings);
+                    var videoLayers = GetPublisherVideoLayers(Publisher.VideoSender.GetParameters().encodings);
                     trackInfo.Layers.AddRange(videoLayers);
 
 #if STREAM_DEBUG_ENABLED
@@ -1472,8 +1476,7 @@ private string ReplaceVp8PayloadType(string sdpOffer)
             return sdpOffer;
         }
 
-        private IEnumerable<VideoLayer> GetPublisherVideoLayers(IEnumerable<RTCRtpEncodingParameters> encodings,
-            PublisherVideoSettings videoSettings)
+        private IEnumerable<VideoLayer> GetPublisherVideoLayers(IEnumerable<RTCRtpEncodingParameters> encodings)
         {
 #if STREAM_DEBUG_ENABLED
             var sb = new System.Text.StringBuilder();
@@ -1482,7 +1485,7 @@ private IEnumerable<VideoLayer> GetPublisherVideoLayers(IEnumerable<RTCRtpEncodi
             foreach (var encoding in encodings)
             {
                 var scaleBy = encoding.scaleResolutionDownBy ?? 1.0;
-                var resolution = videoSettings.MaxResolution;
+                var resolution = Publisher.GetLatestVideoSettings().MaxResolution;
                 var width = (uint)(resolution.Width / scaleBy);
                 var height = (uint)(resolution.Height / scaleBy);
 

Packages/StreamVideo/Runtime/Core/LowLevelClient/StreamPeerConnection.cs

@@ -82,6 +82,9 @@ private set
 
         public RTCRtpSender VideoSender { get; private set; }
 
+        // Full: 704×576  -> half: 352×288 -> quarter: 176×144 <- We want the smallest resolution to be above 96x96
+        public static VideoResolution MinimumSafeTargetResolution => new VideoResolution(704, 576);
+
         public StreamPeerConnection(ILogs logs, StreamPeerType peerType, IEnumerable<ICEServer> iceServers,
             IMediaInputProvider mediaInputProvider, IStreamAudioConfig audioConfig,
             PublisherVideoSettings publisherVideoSettings, Tracer tracer)
@@ -223,6 +226,18 @@ public void Update()
         }
 
         public Task<RTCStatsReport> GetStatsReportAsync() => _peerConnection.GetStatsAsync();
+        
+        public PublisherVideoSettings GetLatestVideoSettings()
+        {
+            if (_publisherVideoSettings == null)
+            {
+                throw new ArgumentNullException($"{nameof(_publisherVideoSettings)} is null in {nameof(GetLatestVideoSettings)}");
+            }
+
+            _publisherVideoSettings.MaxResolution = PublisherTargetResolution;
+            _publisherVideoSettings.FrameRate = PublisherTargetFrameRate;
+            return _publisherVideoSettings;
+        }
 
         public void Dispose()
         {
@@ -275,14 +290,53 @@ public void Dispose()
         private const string VideoCodecKeyH264 = "h264";
         private const string VideoCodecKeyVP8 = "vp8";
         private const string AudioCodecKeyRed = "red";
+        
+        private VideoResolution PublisherTargetResolution
+        {
+            get
+            {
+                if (_mediaInputProvider.VideoInput != null)
+                {
+                    var preferred = new VideoResolution(_mediaInputProvider.VideoInput.width,
+                        _mediaInputProvider.VideoInput.height);
+                    
+                    // Requesting too small resolution can cause crashes in the Android video encoder
+                    // The target resolution is used to calculate 3 layers of video encoding (full, half, quarter)
+                    // For very small values of quarter resolution (not sure exact but ~100x100) the encoder crashes 
+
+                    if (preferred.Width < MinimumSafeTargetResolution.Width ||
+                        preferred.Height < MinimumSafeTargetResolution.Height)
+                    {
+                        return MinimumSafeTargetResolution;
+                    }
+                }
+
+                return _publisherVideoSettings.MaxResolution;
+            }
+        }
+
+        private uint PublisherTargetFrameRate
+        {
+            get
+            {
+                if (_mediaInputProvider.VideoInput != null)
+                {
+                    return (uint)_mediaInputProvider.VideoInput.requestedFPS;
+                }
+
+                return 30;
+            }
+        }
 
         private readonly RTCPeerConnection _peerConnection;
 
         private readonly ILogs _logs;
         private readonly StreamPeerType _peerType;
         private readonly IMediaInputProvider _mediaInputProvider;
         private readonly IStreamAudioConfig _audioConfig;
-        private readonly PublisherVideoSettings _publisherVideoSettings;
+        
+        //StreamTOdo: Finish implementation. This currently is not exposed to the user + it doesn't take into account VideoInput resolution. We need publisher resolution to match with WebCamTexture size
+        private readonly PublisherVideoSettings _publisherVideoSettings; 
         private readonly Tracer _tracer;
 
         private readonly List<RTCIceCandidate> _pendingIceCandidates = new List<RTCIceCandidate>();
@@ -360,6 +414,7 @@ private void OnTrack(RTCTrackEvent trackEvent)
             var streamIds = trackEvent.Streams != null && trackEvent.Streams.Any()
                 ? string.Join(",", trackEvent.Streams.Select(s => s.Id))
                 : "";
+            var isEnabled = trackEvent.Track.Enabled;
 
             if (!string.IsNullOrEmpty(streamIds))
             {
@@ -587,6 +642,10 @@ private static IEnumerable<RTCRtpEncodingParameters> GetVideoEncodingParameters(
 
                     break;
                 case TrackKind.Video:
+                    
+                    //StreamTodo: construct fewer layer when the target resolution is small. Android video encoder crashes when requesting very small resolution
+                    // We're currently forcing the smallest safe resolution that the user can request so that the quarter layer doesn't reach too small resolution
+                    // But we should allow setting small resolution and just produce fewer layers in that case
 
                     var fullQuality = new RTCRtpEncodingParameters
                     {
@@ -631,17 +690,6 @@ private static IEnumerable<RTCRtpEncodingParameters> GetVideoEncodingParameters(
             }
         }
 
-        private VideoResolution GetPublisherResolution()
-        {
-            if (_mediaInputProvider.VideoInput != null)
-            {
-                var maxResolution = _publisherVideoSettings.MaxResolution;
-                return new VideoResolution(maxResolution.Width, maxResolution.Height);
-            }
-
-            return VideoResolution.Res_1080p;
-        }
-
         private VideoStreamTrack CreatePublisherVideoTrack()
         {
             if (_mediaInputProvider.VideoInput == null)
@@ -653,7 +701,7 @@ private VideoStreamTrack CreatePublisherVideoTrack()
             var gfxType = SystemInfo.graphicsDeviceType;
             var format = WebRTC.GetSupportedRenderTextureFormat(gfxType);
 
-            var res = GetPublisherResolution();
+            var res = PublisherTargetResolution;
             _publisherVideoTrackTexture = new RenderTexture((int)res.Width, (int)res.Height, 0, format);
 
 #if STREAM_DEBUG_ENABLED