[AND-9] Implement preferred codec & bitrate

stream-video-android-core/src/androidTest/kotlin/io/getstream/video/android/core/AndroidDeviceTest.kt

@@ -43,7 +43,6 @@ import retrofit2.Retrofit
 import retrofit2.converter.wire.WireConverterFactory
 import stream.video.sfu.event.ChangePublishQuality
 import stream.video.sfu.event.VideoLayerSetting
-import stream.video.sfu.event.VideoMediaRequest
 import stream.video.sfu.event.VideoSender
 import stream.video.sfu.models.Participant
 import stream.video.sfu.models.TrackType
@@ -580,7 +579,6 @@ class AndroidDeviceTest : IntegrationTestBase(connectCoordinatorWS = false) {
 //        assertThat(tracks3?.size).isEqualTo(1)
 
         // test handling publish quality change
-        val mediaRequest = VideoMediaRequest()
         val layers = listOf(
             VideoLayerSetting(name = "f", active = false),
             VideoLayerSetting(name = "h", active = true),
@@ -589,7 +587,6 @@ class AndroidDeviceTest : IntegrationTestBase(connectCoordinatorWS = false) {
         val quality = ChangePublishQuality(
             video_senders = listOf(
                 VideoSender(
-                    media_request = mediaRequest,
                     layers = layers,
                 ),
             ),

stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt

@@ -38,6 +38,7 @@ import io.getstream.video.android.core.model.QueriedMembers
 import io.getstream.video.android.core.model.RejectReason
 import io.getstream.video.android.core.model.SortField
 import io.getstream.video.android.core.model.UpdateUserPermissionsData
+import io.getstream.video.android.core.model.VideoCodec
 import io.getstream.video.android.core.model.VideoTrack
 import io.getstream.video.android.core.model.toIceServer
 import io.getstream.video.android.core.socket.SocketState
@@ -206,6 +207,7 @@ public class Call(
 
     /** Session handles all real time communication for video and audio */
     internal var session: RtcSession? = null
+
     internal val mediaManager by lazy {
         if (testInstanceProvider.mediaManagerCreator != null) {
             testInstanceProvider.mediaManagerCreator!!.invoke()
@@ -1028,6 +1030,10 @@ public class Call(
     }
 
     fun cleanup() {
+        updatePublishOptions(
+            preferredVideoCodec = null,
+            preferredMaxBitrate = null,
+        ) // preferredMaxBitrate not necessarily needed
         monitor.stop()
         session?.cleanup()
         supervisorJob.cancel()
@@ -1119,6 +1125,30 @@ public class Call(
         return clientImpl.toggleAudioProcessing()
     }
 
+    /**
+     * Updates publishing options for the call.
+     *
+     * @param preferredVideoCodec The preferred codec to use for publishing video. Pass `null` to return to the default codec.
+     * @param preferredMaxBitrate The preferred maximum bitrate to use for publishing video.
+     */
+    fun updatePublishOptions(
+        preferredVideoCodec: VideoCodec? = null,
+        preferredMaxBitrate: Int? = null,
+    ) {
+        logger.d {
+            "[updatePublishOptions] #track #updatePublishOptions; preferredVideoCodec: $preferredVideoCodec, preferredMaxBitrate: $preferredMaxBitrate"
+        }
+
+        // Note that the preferredVideoCodec won't affect any transceivers that are already created.
+
+        state.videoPublishOptions = state.videoPublishOptions.copy(
+            preferredCodec = preferredVideoCodec,
+            preferredMaxBitrate = preferredMaxBitrate,
+        )
+
+        clientImpl.peerConnectionFactory.updatePublishOptions(preferredVideoCodec)
+    }
+
     @InternalStreamVideoApi
     public val debug = Debug(this)
 

stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallState.kt

@@ -42,6 +42,7 @@ import io.getstream.video.android.core.model.RTMP
 import io.getstream.video.android.core.model.Reaction
 import io.getstream.video.android.core.model.RejectReason
 import io.getstream.video.android.core.model.ScreenSharingSession
+import io.getstream.video.android.core.model.VideoPublishOptions
 import io.getstream.video.android.core.model.VisibilityOnScreenState
 import io.getstream.video.android.core.permission.PermissionRequest
 import io.getstream.video.android.core.pinning.PinType
@@ -565,6 +566,8 @@ public class CallState(
 
     internal var acceptedOnThisDevice: Boolean = false
 
+    internal var videoPublishOptions: VideoPublishOptions = VideoPublishOptions()
+
     fun handleEvent(event: VideoEvent) {
         logger.d { "Updating call state with event ${event::class.java}" }
         when (event) {

stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/call/RtcSession.kt

@@ -52,6 +52,7 @@ import io.getstream.video.android.core.model.IceCandidate
 import io.getstream.video.android.core.model.IceServer
 import io.getstream.video.android.core.model.MediaTrack
 import io.getstream.video.android.core.model.StreamPeerType
+import io.getstream.video.android.core.model.VideoCodec
 import io.getstream.video.android.core.model.VideoTrack
 import io.getstream.video.android.core.model.toPeerType
 import io.getstream.video.android.core.socket.SocketState
@@ -100,11 +101,13 @@ import org.webrtc.MediaStreamTrack
 import org.webrtc.PeerConnection
 import org.webrtc.PeerConnection.PeerConnectionState
 import org.webrtc.RTCStatsReport
+import org.webrtc.RtpParameters
 import org.webrtc.RtpParameters.Encoding
 import org.webrtc.RtpTransceiver
 import org.webrtc.SessionDescription
 import retrofit2.HttpException
 import stream.video.sfu.event.Migration
+import stream.video.sfu.event.VideoLayerSetting
 import stream.video.sfu.models.ICETrickle
 import stream.video.sfu.models.Participant
 import stream.video.sfu.models.PeerType
@@ -443,6 +446,7 @@ public class RtcSession internal constructor(
                     call.mediaManager.videoTrack,
                     listOf(buildTrackId(TrackType.TRACK_TYPE_VIDEO)),
                     isScreenShare = false,
+                    isUsingSvcCodec = call.state.videoPublishOptions.preferredCodec?.supportsSvc() == true,
                 )
                 videoTransceiverInitialized = true
             }
@@ -857,8 +861,12 @@ public class RtcSession internal constructor(
         val settings = call.state.settings.value
         val red = settings?.audio?.redundantCodingEnabled ?: true
         val opus = settings?.audio?.opusDtxEnabled ?: true
+        val preferredCodec = call.state.videoPublishOptions.preferredCodec
+        val enforceVp8 = preferredCodec == null
 
-        return mangleSdpUtil(sdp, red, opus)
+        logger.v { "[mangleSdp] #updatePublishOptions; Enforcing VP8: $enforceVp8, Using: $preferredCodec" }
+
+        return mangleSdpUtil(sdp, red, opus, enableVp8 = enforceVp8)
     }
 
     @VisibleForTesting
@@ -871,7 +879,8 @@ public class RtcSession internal constructor(
             mediaConstraints = MediaConstraints(),
             onNegotiationNeeded = ::onNegotiationNeeded,
             onIceCandidateRequest = ::sendIceCandidate,
-            maxPublishingBitrate = call.state.settings.value?.video?.targetResolution?.bitrate
+            maxPublishingBitrate = call.state.videoPublishOptions.preferredMaxBitrate
+                ?: call.state.settings.value?.video?.targetResolution?.bitrate
                 ?: 1_200_000,
         )
         logger.i { "[createPublisher] #sfu; publisher: $publisher" }
@@ -888,45 +897,177 @@ public class RtcSession internal constructor(
 
     /**
      * Change the quality of video we upload when the ChangePublishQualityEvent event is received
-     * This is used for dynsacle
+     * This is used for Dynascale
      */
     internal fun updatePublishQuality(event: ChangePublishQualityEvent) = synchronized(this) {
+        dynascaleLogger.v { "[updatePublishQuality] #sfu; Handling ChangePublishQualityEvent" }
+
         val sender = publisher?.connection?.transceivers?.firstOrNull {
             it.mediaType == MediaStreamTrack.MediaType.MEDIA_TYPE_VIDEO
         }?.sender
+        val eventLayerSettingsList = event.changePublishQuality.video_senders.firstOrNull()?.layers
 
-        if (sender == null) {
+        if (sender == null || sender.parameters == null || sender.parameters.encodings == null || eventLayerSettingsList.isNullOrEmpty()) {
             dynascaleLogger.w {
-                "Request to change publishing quality not fulfilled due to missing transceivers or sender."
+                "[updatePublishQuality] #sfu; Sender properties are null or layer settings list is empty in event."
             }
             return@synchronized
         }
 
-        val enabledRids =
-            event.changePublishQuality.video_senders.firstOrNull()?.layers?.associate {
-                it.name to it.active
+        val senderParams = sender.parameters
+        val senderCodec = senderParams.codecs.firstOrNull()
+        val isUsingSvcCodec = rtpCodecToVideoCodec(senderCodec)?.supportsSvc() ?: false
+        val updatedEncodings = mutableSetOf<Encoding>() // Used only for logging
+        var didChanges = false
+
+        for (senderEncoding in senderParams.encodings) {
+            dynascaleLogger.v { "[updatePublishQuality] #sfu; Iteration for rid: ${senderEncoding.rid}" }
+
+            // For SVC, we only have one layer (q) and often rid is omitted
+            // For non-SVC, we need to find the layer by rid (simulcast)
+            val eventLayerSettings = if (isUsingSvcCodec) {
+                eventLayerSettingsList.firstOrNull()
+            } else {
+                eventLayerSettingsList.firstOrNull { it.name == senderEncoding.rid }
+            }
+
+            // Enabled/disabled setting
+            updateActiveFlag(senderEncoding, eventLayerSettings).let {
+                didChanges = it || didChanges
+                if (it) updatedEncodings.add(senderEncoding)
             }
-        dynascaleLogger.i { "enabled rids: $enabledRids}" }
-        val params = sender.parameters
-        val updatedEncodings: MutableList<Encoding> = mutableListOf()
+
+            // Quality settings
+            updateQualitySettings(senderEncoding, eventLayerSettings).let {
+                didChanges = it || didChanges
+                if (it) updatedEncodings.add(senderEncoding)
+            }
+        }
+
+        if (didChanges) {
+            logUpdatedEncodingsRids(updatedEncodings)
+
+            val didSet = sender.setParameters(senderParams)
+            dynascaleLogger.v {
+                "[updatePublishQuality] #sfu; Did set new parameters: $didSet"
+            }
+        }
+
+        logLayerSettings(eventLayerSettingsList, sender.parameters.encodings)
+    }
+
+    private fun rtpCodecToVideoCodec(codec: RtpParameters.Codec?): VideoCodec? =
+        if (codec == null) {
+            null
+        } else {
+            runCatching { VideoCodec.valueOf(codec.name.uppercase()) }.getOrNull().also {
+                dynascaleLogger.v {
+                    "[updatePublishQuality] #sfu; Codec: ${it?.name}, supports SVC: ${it?.supportsSvc()}"
+                }
+            }
+        }
+
+    private fun updateActiveFlag(senderEncoding: Encoding, eventLayerSettings: VideoLayerSetting?): Boolean {
+        return if (senderEncoding.active != eventLayerSettings?.active) {
+            senderEncoding.active = eventLayerSettings?.active ?: false
+
+            dynascaleLogger.v {
+                "[updatePublishQuality][updateActiveFlag] #sfu; rid: ${senderEncoding.rid}; Changed active flag from ${!senderEncoding.active} to ${senderEncoding.active}"
+            }
+
+            true
+        } else {
+            false
+        }
+    }
+
+    private fun updateQualitySettings(senderEncoding: Encoding, eventLayerSettings: VideoLayerSetting?): Boolean {
         var changed = false
-        for (encoding in params.encodings) {
-            val shouldEnable = enabledRids?.get(encoding.rid) ?: false
-            if (shouldEnable && encoding.active) {
-                updatedEncodings.add(encoding)
-            } else if (!shouldEnable && !encoding.active) {
-                updatedEncodings.add(encoding)
-            } else {
-                changed = true
-                encoding.active = shouldEnable
-                updatedEncodings.add(encoding)
+
+        eventLayerSettings?.let { settings ->
+            with(settings) {
+                if (max_bitrate > 0 && max_bitrate != (senderEncoding.maxBitrateBps ?: 0)) {
+                    dynascaleLogger.v {
+                        "[updatePublishQuality][updateQualitySettings] #sfu; rid: ${senderEncoding.rid}; Changed maxBitrate from ${senderEncoding.maxBitrateBps} to $max_bitrate"
+                    }
+                    senderEncoding.maxBitrateBps = max_bitrate
+                    changed = true
+                }
+
+                if (max_framerate > 0 && max_framerate != (senderEncoding.maxFramerate ?: 0)) {
+                    dynascaleLogger.v {
+                        "[updatePublishQuality][updateQualitySettings] #sfu; rid: ${senderEncoding.rid}; Changed maxFramerate from ${senderEncoding.maxFramerate} to $max_framerate"
+                    }
+                    senderEncoding.maxFramerate = max_framerate
+                    changed = true
+                }
+
+                scale_resolution_down_by.toDouble().let { scaleResolutionDownBy ->
+                    if (scaleResolutionDownBy >= 1 &&
+                        scaleResolutionDownBy != (senderEncoding.scaleResolutionDownBy ?: 0)
+                    ) {
+                        dynascaleLogger.v {
+                            "[updatePublishQuality][updateQualitySettings] #sfu; rid: ${senderEncoding.rid}; Changed scaleResolutionDownBy from ${senderEncoding.scaleResolutionDownBy} to $scale_resolution_down_by"
+                        }
+                        senderEncoding.scaleResolutionDownBy = scaleResolutionDownBy
+                        changed = true
+                    }
+                }
+
+                if (scalability_mode.isNotBlank() && scalability_mode != (senderEncoding.scalabilityMode ?: "")) {
+                    dynascaleLogger.v {
+                        "[updatePublishQuality][updateQualitySettings] #sfu; rid: ${senderEncoding.rid}; Changed scalabilityMode from ${senderEncoding.scalabilityMode} to $scalability_mode"
+                    }
+                    senderEncoding.scalabilityMode = scalability_mode
+                    changed = true
+                }
+            }
+        }
+
+        return changed
+    }
+
+    private fun logUpdatedEncodingsRids(updatedEncodings: Set<Encoding>) {
+        dynascaleLogger.d {
+            updatedEncodings.reversed().joinToString {
+                if (it.rid != null) it.rid.toString() else ""
+            }.let {
+                "[updatePublishQuality] #sfu; Updated encodings: $it"
             }
         }
-        if (changed) {
-            dynascaleLogger.i { "Updated publish quality with encodings $updatedEncodings" }
-            params.encodings.clear()
-            params.encodings.addAll(updatedEncodings)
-            sender.parameters = params
+    }
+
+    private fun logLayerSettings(
+        eventLayerSettingsList: List<VideoLayerSetting>? = null,
+        senderEncodingList: List<Encoding>,
+    ) {
+        val eventLog = eventLayerSettingsList?.joinToString(separator = "") {
+            "\n-Name: ${it.name}\n" +
+                "active: ${it.active}\n" +
+                "maxBitrate: ${it.max_bitrate}\n" +
+                "maxFramerate: ${it.max_framerate}\n" +
+                "scaleResolutionDownBy: ${it.scale_resolution_down_by}\n" +
+                "scalabilityMode: ${it.scalability_mode.ifEmpty { "(empty)" }}"
+        }
+        val senderLog = senderEncodingList.reversed().joinToString(separator = "") {
+            "\n-Name: ${it.rid}\n" +
+                "active: ${it.active}\n" +
+                "maxBitrate: ${it.maxBitrateBps}\n" +
+                "maxFramerate: ${it.maxFramerate}\n" +
+                "scaleResolutionDownBy: ${it.scaleResolutionDownBy}\n" +
+                "scalabilityMode: ${it.scalabilityMode}"
+        }
+
+        dynascaleLogger.v {
+            "[updatePublishQuality] #sfu;\n" +
+                if (eventLog == null) {
+                    ""
+                } else {
+                    "Event layers:" +
+                        "$eventLog\n\n"
+                } +
+                "Sender encodings:" +
+                senderLog
         }
     }
 
@@ -1415,7 +1556,7 @@ public class RtcSession internal constructor(
                         "video capture needs to be enabled before adding the local track",
                     )
                 }
-                createVideoLayers(transceiver, captureResolution)
+                createVideoLayers(captureResolution)
             } else if (trackType == TrackType.TRACK_TYPE_SCREEN_SHARE) {
                 createScreenShareLayers(transceiver)
             } else {
@@ -1498,15 +1639,15 @@ public class RtcSession internal constructor(
         return media.mid.toString()
     }
 
-    private fun createVideoLayers(
-        transceiver: RtpTransceiver,
-        captureResolution: CaptureFormat,
-    ): List<VideoLayer> {
-        // we tell the Sfu which resolutions we're sending
-        return transceiver.sender.parameters.encodings.map {
+    private fun createVideoLayers(captureResolution: CaptureFormat): List<VideoLayer> {
+        // We tell the SFU which resolutions we're sending.
+        // Even if we use SVC, we still generate three layers [f, h, q]
+        // because we need to announce them to the SFU via the SetPublisher request,
+        // so we use the simulcast encodings here.
+        return publisher?.simulcastEncodings?.map {
             val scaleBy = it.scaleResolutionDownBy ?: 1.0
-            val width = captureResolution.width.div(scaleBy) ?: 0
-            val height = captureResolution.height.div(scaleBy) ?: 0
+            val width = captureResolution.width.div(scaleBy)
+            val height = captureResolution.height.div(scaleBy)
             val quality = ridToVideoQuality(it.rid)
 
             // We need to divide by 1000 because the the FramerateRange is multiplied
@@ -1523,7 +1664,7 @@ public class RtcSession internal constructor(
                 fps = fps,
                 quality = quality,
             )
-        }
+        } ?: emptyList()
     }
 
     private fun createScreenShareLayers(transceiver: RtpTransceiver): List<VideoLayer> {