Implemented smooth audio switching

api-extractor/report/hls.js.api.md

@@ -177,6 +177,8 @@ export type AudioSelectionOption = {
     audioCodec?: string;
     groupId?: string;
     default?: boolean;
+    nextAudioTrackSwitchingSafetyDelay?: number;
+    nextAudioTrackBufferFlushDelay?: number;
 };
 
 // Warning: (ae-missing-release-tag) "AudioStreamController" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -194,6 +196,10 @@ export class AudioStreamController extends BaseStreamController implements Netwo
     _handleFragmentLoadProgress(data: FragLoadedData): void;
     // (undocumented)
     protected loadFragment(frag: Fragment, track: Level, targetBufferTime: number): void;
+    get nextAudioTrack(): number;
+    // Warning: (ae-setter-with-docs) The doc comment for the property "nextAudioTrack" must appear on the getter, not the setter.
+    set nextAudioTrack(audioTrackId: number);
+    nextAudioTrackSwitch(): void;
     // (undocumented)
     protected onError(event: Events.ERROR, data: ErrorData): void;
     // (undocumented)
@@ -235,6 +241,9 @@ export class AudioTrackController extends BasePlaylistController {
     // (undocumented)
     protected loadPlaylist(hlsUrlParameters?: HlsUrlParameters): void;
     // (undocumented)
+    get nextAudioTrack(): number;
+    set nextAudioTrack(newId: number);
+    // (undocumented)
     protected onAudioTrackLoaded(event: Events.AUDIO_TRACK_LOADED, data: AudioTrackLoadedData): void;
     // (undocumented)
     protected onError(event: Events.ERROR, data: ErrorData): void;
@@ -274,6 +283,8 @@ export interface AudioTrackSwitchedData extends MediaPlaylist {
 //
 // @public (undocumented)
 export interface AudioTrackSwitchingData extends MediaPlaylist {
+    // (undocumented)
+    flushImmediate?: boolean;
 }
 
 // Warning: (ae-missing-release-tag) "AudioTrackUpdatedData" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -379,10 +390,17 @@ export class BaseSegment {
 // @public (undocumented)
 export class BaseStreamController extends TaskLoop implements NetworkComponentAPI {
     constructor(hls: Hls, fragmentTracker: FragmentTracker, keyLoader: KeyLoader, logPrefix: string, playlistType: PlaylistLevelType);
+    protected abortCurrentFrag(): void;
     // (undocumented)
     protected afterBufferFlushed(media: Bufferable, bufferType: SourceBufferName, playlistType: PlaylistLevelType): void;
     // (undocumented)
     protected alignPlaylists(details: LevelDetails, previousDetails: LevelDetails | undefined, switchDetails: LevelDetails | undefined): number;
+    // Warning: (ae-forgotten-export) The symbol "AlternateAudio" needs to be exported by the entry point hls.d.ts
+    //
+    // (undocumented)
+    protected altAudio: AlternateAudio;
+    // (undocumented)
+    protected backtrackFragment: Fragment | null;
     // (undocumented)
     protected bitrateTest: boolean;
     // (undocumented)
@@ -391,15 +409,24 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP
     protected buffering: boolean;
     // (undocumented)
     get bufferingEnabled(): boolean;
+    protected calculateOptimalSwitchPoint(nextLevel: Level, bufferInfo: BufferInfo, levelDetails: LevelDetails | undefined, type: PlaylistLevelType): {
+        fetchdelay: number;
+        okToFlushForwardBuffer: boolean;
+    };
+    // (undocumented)
+    protected checkFragmentChanged(type?: PlaylistLevelType): void;
     // (undocumented)
     protected checkLiveUpdate(details: LevelDetails): void;
     // (undocumented)
     protected checkRetryDate(): void;
+    protected cleanupBackBuffer(type: PlaylistLevelType): void;
     // (undocumented)
     protected clearTrackerIfNeeded(frag: Fragment): void;
     // (undocumented)
     protected config: HlsConfig;
     // (undocumented)
+    protected couldBacktrack: boolean;
+    // (undocumented)
     protected decrypter: Decrypter;
     // (undocumented)
     protected _doFragLoad(frag: Fragment, level: Level, targetBufferTime?: number | null, progressCallback?: FragmentLoadProgressCallback): Promise<PartsLoadedData | FragLoadedData | null>;
@@ -411,6 +438,7 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP
     protected flushBufferGap(frag: Fragment): void;
     // (undocumented)
     protected flushMainBuffer(startOffset: number, endOffset: number, type?: SourceBufferName | null): void;
+    protected followingBufferedFrag(frag: Fragment | null, type?: PlaylistLevelType): Fragment | null;
     // (undocumented)
     protected fragBufferedComplete(frag: Fragment, part: Part | null): void;
     // (undocumented)
@@ -422,9 +450,12 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP
     // (undocumented)
     protected fragmentTracker: FragmentTracker;
     // (undocumented)
+    protected fragPlaying: Fragment | null;
+    // (undocumented)
     protected fragPrevious: MediaFragment | null;
     // (undocumented)
     protected getAppendedFrag(position: number, playlistType?: PlaylistLevelType): Fragment | null;
+    protected getBufferedFrag(position: number, type?: PlaylistLevelType): Fragment | null;
     // (undocumented)
     protected getCurrentContext(chunkMeta: ChunkMetadata): {
         frag: MediaFragment;
@@ -444,6 +475,8 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP
     // (undocumented)
     protected getLoadPosition(): number;
     // (undocumented)
+    getMainFwdBufferInfo(): BufferInfo | null;
+    // (undocumented)
     protected getMaxBufferLength(levelBitrate?: number): number;
     // (undocumented)
     protected getNextFragment(pos: number, levelDetails: LevelDetails): Fragment | null;
@@ -486,7 +519,10 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP
     // (undocumented)
     protected mediaBuffer: Bufferable | null;
     // (undocumented)
+    nextLevelSwitch(type: PlaylistLevelType): void;
+    // (undocumented)
     protected nextLoadPosition: number;
+    nextTrackSwitch(): void;
     // (undocumented)
     protected onError(event: Events.ERROR, data: ErrorData): void;
     // (undocumented)
@@ -539,6 +575,7 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP
     resumeBuffering(): void;
     // (undocumented)
     protected retryDate: number;
+    protected scheduleTrackSwitch(bufferInfo: BufferInfo, fetchdelay: number, okToFlushForwardBuffer: boolean, type: PlaylistLevelType): void;
     // (undocumented)
     protected setStartPosition(details: LevelDetails, sliding: number): void;
     // (undocumented)
@@ -2116,6 +2153,9 @@ class Hls implements HlsEventEmitter {
     // (undocumented)
     static get MetadataSchema(): typeof MetadataSchema;
     get minAutoLevel(): number;
+    get nextAudioTrack(): number;
+    // Warning: (ae-setter-with-docs) The doc comment for the property "nextAudioTrack" must appear on the getter, not the setter.
+    set nextAudioTrack(audioTrackId: number);
     get nextAutoLevel(): number;
     // Warning: (ae-setter-with-docs) The doc comment for the property "nextAutoLevel" must appear on the getter, not the setter.
     set nextAutoLevel(nextLevel: number);
@@ -4539,6 +4579,8 @@ export interface SteeringManifestLoadedData {
 export class StreamController extends BaseStreamController implements NetworkComponentAPI {
     constructor(hls: Hls, fragmentTracker: FragmentTracker, keyLoader: KeyLoader);
     // (undocumented)
+    protected abortCurrentFrag(): void;
+    // (undocumented)
     get currentFrag(): Fragment | null;
     // (undocumented)
     get currentLevel(): number;
@@ -4551,8 +4593,6 @@ export class StreamController extends BaseStreamController implements NetworkCom
     // (undocumented)
     get forceStartLoad(): boolean;
     // (undocumented)
-    getMainFwdBufferInfo(): BufferInfo | null;
-    // (undocumented)
     protected _handleFragmentLoadProgress(data: FragLoadedData): void;
     // (undocumented)
     get hasEnoughToStart(): boolean;
@@ -4563,7 +4603,7 @@ export class StreamController extends BaseStreamController implements NetworkCom
     // (undocumented)
     get maxBufferLength(): number;
     // (undocumented)
-    get nextBufferedFrag(): MediaFragment | null;
+    get nextBufferedFrag(): Fragment | null;
     // (undocumented)
     get nextLevel(): number;
     nextLevelSwitch(): void;

demo/index-light.html

@@ -454,6 +454,12 @@ <h3>
             <td>Current audio-track:</td>
             <td><div id="audioTrackControl" style="display: inline"></div></td>
           </tr>
+          <tr>
+            <td>Next audio-track:</td>
+            <td>
+              <div id="nextAudioTrackControl" style="display: inline"></div>
+            </td>
+          </tr>
           <tr>
             <td>Language / Name:</td>
             <td>

demo/index.html

@@ -453,6 +453,12 @@ <h3>
             <td>Current audio-track:</td>
             <td><div id="audioTrackControl" style="display: inline"></div></td>
           </tr>
+          <tr>
+            <td>Next audio-track:</td>
+            <td>
+              <div id="nextAudioTrackControl" style="display: inline"></div>
+            </td>
+          </tr>
           <tr>
             <td>Language / Name:</td>
             <td>

demo/main.js

@@ -1393,9 +1393,12 @@ function updateAudioTrackInfo() {
   const buttonEnabled = 'btn-primary" ';
   const buttonDisabled = 'btn-success" ';
   let html1 = '';
+  let html2 = '';
   const audioTrackId = hls.audioTrack;
+  const nextAudioTrackId = hls.nextAudioTrack;
   const len = hls.audioTracks.length;
   const track = hls.audioTracks[audioTrackId];
+  const nextTrack = hls.audioTracks[nextAudioTrackId];
 
   for (let i = 0; i < len; i++) {
     html1 += buttonTemplate;
@@ -1413,10 +1416,30 @@ function updateAudioTrackInfo() {
       '</button>';
   }
 
+  for (let i = 0; i < len; i++) {
+    html2 += buttonTemplate;
+    if (nextAudioTrackId === i) {
+      html2 += buttonEnabled;
+    } else {
+      html2 += buttonDisabled;
+    }
+
+    html2 +=
+      'onclick="hls.nextAudioTrack=' +
+      i +
+      '">' +
+      hls.audioTracks[i].name +
+      '</button>';
+  }
+
   $('#audioTrackLabel').text(
     track ? track.lang || track.name : 'None selected'
   );
   $('#audioTrackControl').html(html1);
+  $('#audioTrackLabel').text(
+    nextTrack ? nextTrack.lang || nextTrack.name : 'None selected'
+  );
+  $('#nextAudioTrackControl').html(html2);
 }
 
 function codecs2label(levelCodecs) {

docs/API.md

@@ -171,6 +171,7 @@ See [API Reference](https://hlsjs-dev.video-dev.org/api-docs/) for a complete li
   - [`hls.allAudioTracks`](#hlsallaudiotracks)
   - [`hls.audioTracks`](#hlsaudiotracks)
   - [`hls.audioTrack`](#hlsaudiotrack)
+  - [`hls.nextAudioTrack`](#hlsnextaudiotrack)
 - [Subtitle Tracks Control API](#subtitle-tracks-control-api)
   - [`hls.setSubtitleOption(subtitleOption)`](#hlssetsubtitleoptionsubtitleoption)
   - [`hls.allSubtitleTracks`](#hlsallsubtitletracks)
@@ -1985,6 +1986,10 @@ get : array of supported audio tracks in the active audio group ID
 
 get/set : index of selected audio track in `hls.audioTracks`
 
+### `hls.nextAudioTrack`
+
+get/set : index of the next audio track that will be selected, allowing for seamless audio track switching
+
 ## Subtitle Tracks Control API
 
 ### `hls.setSubtitleOption(subtitleOption)`
@@ -2321,7 +2326,7 @@ Full list of Events is available below:
 - `Hls.Events.AUDIO_TRACKS_UPDATED` - fired to notify that audio track lists has been updated
   - data: { audioTracks : audioTracks }
 - `Hls.Events.AUDIO_TRACK_SWITCHING` - fired when an audio track switching is requested
-  - data: { id : audio track id, type : playlist type ('AUDIO' | 'main'), url : audio track URL }
+  - data: { id : audio track id, type : playlist type ('AUDIO' | 'main'), url : audio track URL, flushImmediate: boolean indicating whether audio buffer should be flushed immediately when switching }
 - `Hls.Events.AUDIO_TRACK_SWITCHED` - fired when an audio track switch actually occurs
   - data: { id : audio track id }
 - `Hls.Events.AUDIO_TRACK_LOADING` - fired when an audio track loading starts

src/controller/audio-stream-controller.ts

@@ -66,11 +66,13 @@ class AudioStreamController
   private bufferedTrack: MediaPlaylist | null = null;
   private switchingTrack: MediaPlaylist | null = null;
   private trackId: number = -1;
+  private nextTrackId: number = -1;
   private waitingData: WaitingForPTSData | null = null;
   private mainDetails: LevelDetails | null = null;
   private flushing: boolean = false;
   private bufferFlushed: boolean = false;
   private cachedTrackLoadedData: TrackLoadedData | null = null;
+  private pendingAudioTrackSwitch: boolean = false;
 
   constructor(
     hls: Hls,
@@ -307,6 +309,7 @@ class AudioStreamController
     }
 
     this.lastCurrentTime = media.currentTime;
+    this.checkFragmentChanged(PlaylistLevelType.AUDIO);
   }
 
   private doTickIdle() {
@@ -372,7 +375,10 @@ class AudioStreamController
     const fragments = trackDetails.fragments;
     const start = fragments[0].start;
     const loadPosition = this.getLoadPosition();
-    const targetBufferTime = this.flushing ? loadPosition : bufferInfo.end;
+    const targetBufferTime =
+      this.flushing || this.pendingAudioTrackSwitch
+        ? loadPosition
+        : bufferInfo.end;
 
     if (this.switchingTrack && media) {
       const pos = loadPosition;
@@ -493,6 +499,7 @@ class AudioStreamController
         // switching to audio track, start timer if not already started
         this.setInterval(TICK_INTERVAL);
         this.state = State.IDLE;
+        this.pendingAudioTrackSwitch = !data.flushImmediate;
         this.tick();
       }
     } else {
@@ -1022,8 +1029,8 @@ class AudioStreamController
     }
   }
 
-  private flushAudioIfNeeded(switchingTrack: MediaPlaylist) {
-    if (this.media && this.bufferedTrack) {
+  private flushAudioIfNeeded(switchingTrack: AudioTrackSwitchingData) {
+    if (this.media && this.bufferedTrack && switchingTrack.flushImmediate) {
       const { name, lang, assocLang, characteristics, audioCodec, channels } =
         this.bufferedTrack;
       if (
@@ -1033,7 +1040,10 @@ class AudioStreamController
           audioMatchPredicate,
         )
       ) {
-        if (useAlternateAudio(switchingTrack.url, this.hls)) {
+        if (
+          (useAlternateAudio(switchingTrack.url, this.hls),
+          switchingTrack.flushImmediate)
+        ) {
           this.log('Switching audio track : flushing all audio');
           super.flushMainBuffer(0, Number.POSITIVE_INFINITY, 'audio');
           this.bufferedTrack = null;
@@ -1045,12 +1055,55 @@ class AudioStreamController
     }
   }
 
-  private completeAudioSwitch(switchingTrack: MediaPlaylist) {
+  private completeAudioSwitch(switchingTrack: AudioTrackSwitchingData) {
     const { hls } = this;
-    this.flushAudioIfNeeded(switchingTrack);
+    const { flushImmediate } = switchingTrack;
+    if (!flushImmediate) {
+      const { config } = this;
+      const bufferFlushDelay =
+        config.audioPreference?.nextAudioTrackBufferFlushDelay || 0.25;
+      const startOffset = Math.max(
+        this.getLoadPosition() + bufferFlushDelay,
+        this.fragPrevious?.start || 0,
+      );
+      super.flushMainBuffer(
+        startOffset,
+        Number.POSITIVE_INFINITY,
+        PlaylistLevelType.AUDIO,
+      );
+    }
+
     this.bufferedTrack = switchingTrack;
-    this.switchingTrack = null;
-    hls.trigger(Events.AUDIO_TRACK_SWITCHED, { ...switchingTrack });
+    this.pendingAudioTrackSwitch = false;
+    if (flushImmediate) {
+      this.switchingTrack = null;
+      hls.trigger(Events.AUDIO_TRACK_SWITCHED, switchingTrack);
+    }
+  }
+
+  /**
+   * Index of next audio track loaded as scheduled by audio stream controller.
+   */
+  get nextAudioTrack(): number {
+    return this.nextTrackId;
+  }
+
+  /**
+   * Set next audio track index for seamless audio track switching.
+   * This schedules audio track switching without interrupting playback.
+   */
+  set nextAudioTrack(audioTrackId: number) {
+    this.nextTrackId = audioTrackId;
+  }
+
+  /**
+   * try to switch ASAP without breaking audio playback:
+   * in order to ensure smooth but quick audio track switching,
+   * we need to find the next flushable buffer range
+   * we should take into account new segment fetch time
+   */
+  public nextAudioTrackSwitch(): void {
+    super.nextLevelSwitch(PlaylistLevelType.AUDIO);
   }
 }
 export default AudioStreamController;