Fix svc mode for chrome v113, change default svc mode to L3T3_KEY (livekit#720)

* Fix svc mode for chrome v113.

Fix svc mode for chrome v113.
Add closable spatial layer for svc dynacast but disable it as two
blur/frozen issues
with current server/client:
          1. chrome 113: when switching to up layer with scalability Mode change, it will generate a
          low resolution frame and recover very quickly, but noticable
          2. livekit sfu: additional pli request cause video frozen for a few frames, also noticable

Author: cnderrauber

.changeset/spicy-cats-shop.md

@@ -0,0 +1,5 @@
+---
+'livekit-client': patch
+---
+
+Fix svc mode for chrome v113

src/room/participant/LocalParticipant.ts

@@ -625,8 +625,8 @@ export default class LocalParticipant extends Participant {
       // for svc codecs, disable simulcast and use vp8 for backup codec
       if (track instanceof LocalVideoTrack) {
         if (isSVCCodec(opts.videoCodec)) {
-          // set scalabilityMode to 'L3T3' by default
-          opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3';
+          // set scalabilityMode to 'L3T3_KEY' by default
+          opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3_KEY';
         }
 
         // set up backup
@@ -656,7 +656,7 @@ export default class LocalParticipant extends Participant {
         dims.height,
         opts,
       );
-      req.layers = videoLayersFromEncodings(req.width, req.height, simEncodings ?? encodings);
+      req.layers = videoLayersFromEncodings(req.width, req.height, encodings);
     } else if (track.kind === Track.Kind.Audio) {
       encodings = [
         {

src/room/participant/publishUtils.ts

@@ -128,17 +128,15 @@ export function computeVideoEncodings(
     // svc use first encoding as the original, so we sort encoding from high to low
     switch (scalabilityMode) {
       case 'L3T3':
-        for (let i = 0; i < 3; i += 1) {
-          encodings.push({
-            rid: videoRids[2 - i],
-            scaleResolutionDownBy: 2 ** i,
-            maxBitrate: videoEncoding.maxBitrate / 3 ** i,
-            /* @ts-ignore */
-            maxFramerate: original.encoding.maxFramerate,
-            /* @ts-ignore */
-            scalabilityMode: 'L3T3',
-          });
-        }
+      case 'L3T3_KEY':
+        encodings.push({
+          rid: videoRids[2],
+          maxBitrate: videoEncoding.maxBitrate,
+          /* @ts-ignore */
+          maxFramerate: original.encoding.maxFramerate,
+          /* @ts-ignore */
+          scalabilityMode: scalabilityMode,
+        });
         log.debug('encodings', encodings);
         return encodings;
 
@@ -368,3 +366,34 @@ export function sortPresets(presets: Array<VideoPreset> | undefined) {
     return 0;
   });
 }
+
+/** @internal */
+export class ScalabilityMode {
+  spatial: number;
+
+  temporal: number;
+
+  suffix: undefined | 'h' | '_KEY' | '_KEY_SHIFT';
+
+  constructor(scalabilityMode: string) {
+    const results = scalabilityMode.match(/^L(\d)T(\d)(h|_KEY|_KEY_SHIFT){0,1}$/);
+    if (!results) {
+      throw new Error('invalid scalability mode');
+    }
+
+    this.spatial = parseInt(results[1]);
+    this.temporal = parseInt(results[2]);
+    if (results.length > 3) {
+      switch (results[3]) {
+        case 'h':
+        case '_KEY':
+        case '_KEY_SHIFT':
+          this.suffix = results[3];
+      }
+    }
+  }
+
+  toString(): string {
+    return `L${this.spatial}T${this.temporal}${this.suffix ?? ''}`;
+  }
+}

src/room/track/LocalVideoTrack.ts

@@ -2,6 +2,7 @@ import type { SignalClient } from '../../api/SignalClient';
 import log from '../../logger';
 import { VideoLayer, VideoQuality } from '../../proto/livekit_models';
 import type { SubscribedCodec, SubscribedQuality } from '../../proto/livekit_rtc';
+import { ScalabilityMode } from '../participant/publishUtils';
 import { computeBitrate, monitorFrequency } from '../stats';
 import type { VideoSenderStats } from '../stats';
 import { Mutex, isFireFox, isMobile, isWeb } from '../utils';
@@ -349,45 +350,88 @@ async function setPublishingLayersForSender(
     }
 
     let hasChanged = false;
-    encodings.forEach((encoding, idx) => {
-      let rid = encoding.rid ?? '';
-      if (rid === '') {
-        rid = 'q';
-      }
-      const quality = videoQualityForRid(rid);
-      const subscribedQuality = qualities.find((q) => q.quality === quality);
-      if (!subscribedQuality) {
-        return;
-      }
-      if (encoding.active !== subscribedQuality.enabled) {
+
+    /* @ts-ignore */
+    if (encodings.length === 1 && encodings[0].scalabilityMode) {
+      // svc dynacast encodings
+      const encoding = encodings[0];
+      /* @ts-ignore */
+      // const mode = new ScalabilityMode(encoding.scalabilityMode);
+      let maxQuality = VideoQuality.OFF;
+      qualities.forEach((q) => {
+        if (q.enabled && (maxQuality === VideoQuality.OFF || q.quality > maxQuality)) {
+          maxQuality = q.quality;
+        }
+      });
+
+      if (maxQuality === VideoQuality.OFF) {
+        if (encoding.active) {
+          encoding.active = false;
+          hasChanged = true;
+        }
+      } else if (!encoding.active /* || mode.spatial !== maxQuality + 1*/) {
         hasChanged = true;
-        encoding.active = subscribedQuality.enabled;
-        log.debug(
-          `setting layer ${subscribedQuality.quality} to ${
-            encoding.active ? 'enabled' : 'disabled'
-          }`,
-        );
-
-        // FireFox does not support setting encoding.active to false, so we
-        // have a workaround of lowering its bitrate and resolution to the min.
-        if (isFireFox()) {
-          if (subscribedQuality.enabled) {
-            encoding.scaleResolutionDownBy = senderEncodings[idx].scaleResolutionDownBy;
-            encoding.maxBitrate = senderEncodings[idx].maxBitrate;
-            /* @ts-ignore */
-            encoding.maxFrameRate = senderEncodings[idx].maxFrameRate;
-          } else {
-            encoding.scaleResolutionDownBy = 4;
-            encoding.maxBitrate = 10;
-            /* @ts-ignore */
-            encoding.maxFrameRate = 2;
-          }
+        encoding.active = true;
+        /* disable closable spatial layer as it has video blur/frozen issue with current server/client
+          1. chrome 113: when switching to up layer with scalability Mode change, it will generate a 
+          low resolution frame and recover very quickly, but noticable
+          2. livekit sfu: additional pli request cause video frozen for a few frames, also noticable
+        @ts-ignore
+        const originalMode = new ScalabilityMode(senderEncodings[0].scalabilityMode)
+        mode.spatial = maxQuality + 1;
+        mode.suffix = originalMode.suffix;
+        if (mode.spatial === 1) {
+          // no suffix for L1Tx
+          mode.suffix = undefined;
         }
+        @ts-ignore
+        encoding.scalabilityMode = mode.toString();
+        encoding.scaleResolutionDownBy = 2 ** (2 - maxQuality);
+      */
       }
-    });
+    } else {
+      // simulcast dynacast encodings
+      encodings.forEach((encoding, idx) => {
+        let rid = encoding.rid ?? '';
+        if (rid === '') {
+          rid = 'q';
+        }
+        const quality = videoQualityForRid(rid);
+        const subscribedQuality = qualities.find((q) => q.quality === quality);
+        if (!subscribedQuality) {
+          return;
+        }
+        if (encoding.active !== subscribedQuality.enabled) {
+          hasChanged = true;
+          encoding.active = subscribedQuality.enabled;
+          log.debug(
+            `setting layer ${subscribedQuality.quality} to ${
+              encoding.active ? 'enabled' : 'disabled'
+            }`,
+          );
+
+          // FireFox does not support setting encoding.active to false, so we
+          // have a workaround of lowering its bitrate and resolution to the min.
+          if (isFireFox()) {
+            if (subscribedQuality.enabled) {
+              encoding.scaleResolutionDownBy = senderEncodings[idx].scaleResolutionDownBy;
+              encoding.maxBitrate = senderEncodings[idx].maxBitrate;
+              /* @ts-ignore */
+              encoding.maxFrameRate = senderEncodings[idx].maxFrameRate;
+            } else {
+              encoding.scaleResolutionDownBy = 4;
+              encoding.maxBitrate = 10;
+              /* @ts-ignore */
+              encoding.maxFrameRate = 2;
+            }
+          }
+        }
+      });
+    }
 
     if (hasChanged) {
       params.encodings = encodings;
+      log.debug(`setting encodings`, params.encodings);
       await sender.setParameters(params);
     }
   } finally {
@@ -425,6 +469,25 @@ export function videoLayersFromEncodings(
       },
     ];
   }
+
+  /* @ts-ignore */
+  if (encodings.length === 1 && encodings[0].scalabilityMode) {
+    // svc layers
+    /* @ts-ignore */
+    const sm = new ScalabilityMode(encodings[0].scalabilityMode);
+    const layers = [];
+    for (let i = 0; i < sm.spatial; i += 1) {
+      layers.push({
+        quality: VideoQuality.HIGH - i,
+        width: width / 2 ** i,
+        height: height / 2 ** i,
+        bitrate: encodings[0].maxBitrate ? encodings[0].maxBitrate / 3 ** i : 0,
+        ssrc: 0,
+      });
+    }
+    return layers;
+  }
+
   return encodings.map((encoding) => {
     const scale = encoding.scaleResolutionDownBy ?? 1;
     let quality = videoQualityForRid(encoding.rid ?? '');

src/room/track/options.ts

@@ -284,7 +284,7 @@ export function isCodecEqual(c1: string | undefined, c2: string | undefined): bo
 /**
  * scalability modes for svc, only supprot l3t3 now.
  */
-export type ScalabilityMode = 'L3T3';
+export type ScalabilityMode = 'L3T3' | 'L3T3_KEY';
 
 export namespace AudioPresets {
   export const telephone: AudioPreset = {