diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md index 0dd5b746a23..2460027289e 100644 --- a/api-extractor/report/hls.js.api.md +++ b/api-extractor/report/hls.js.api.md @@ -185,8 +185,12 @@ export type AudioSelectionOption = { export class AudioStreamController extends BaseStreamController implements NetworkComponentAPI { constructor(hls: Hls, fragmentTracker: FragmentTracker, keyLoader: KeyLoader); // (undocumented) + protected checkFragmentChanged(): boolean; + // (undocumented) doTick(): void; // (undocumented) + protected getBufferOutput(): Bufferable | null; + // (undocumented) protected getLoadPosition(): number; // (undocumented) protected _handleFragmentLoadComplete(fragLoadedData: FragLoadedData): void; @@ -194,6 +198,7 @@ export class AudioStreamController extends BaseStreamController implements Netwo _handleFragmentLoadProgress(data: FragLoadedData): void; // (undocumented) protected loadFragment(frag: Fragment, track: Level, targetBufferTime: number): void; + get nextAudioTrack(): number; // (undocumented) protected onError(event: Events.ERROR, data: ErrorData): void; // (undocumented) @@ -235,6 +240,9 @@ export class AudioTrackController extends BasePlaylistController { // (undocumented) protected loadPlaylist(hlsUrlParameters?: HlsUrlParameters): void; // (undocumented) + get nextAudioTrack(): number; + set nextAudioTrack(newId: number); + // (undocumented) protected onAudioTrackLoaded(event: Events.AUDIO_TRACK_LOADED, data: AudioTrackLoadedData): void; // (undocumented) protected onError(event: Events.ERROR, data: ErrorData): void; @@ -274,6 +282,8 @@ export interface AudioTrackSwitchedData extends MediaPlaylist { // // @public (undocumented) export interface AudioTrackSwitchingData extends MediaPlaylist { + // (undocumented) + flushImmediate?: boolean; } // Warning: (ae-missing-release-tag) "AudioTrackUpdatedData" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -379,10 +389,14 @@ export class BaseSegment { // @public (undocumented) export class BaseStreamController extends TaskLoop implements NetworkComponentAPI { constructor(hls: Hls, fragmentTracker: FragmentTracker, keyLoader: KeyLoader, logPrefix: string, playlistType: PlaylistLevelType); + protected abortCurrentFrag(): void; // (undocumented) - protected afterBufferFlushed(media: Bufferable, bufferType: SourceBufferName, playlistType: PlaylistLevelType): void; + protected afterBufferFlushed(media: Bufferable, bufferType: SourceBufferName): void; // (undocumented) protected alignPlaylists(details: LevelDetails, previousDetails: LevelDetails | undefined, switchDetails: LevelDetails | undefined): number; + protected get backtrackFragment(): Fragment | undefined; + // Warning: (ae-setter-with-docs) The doc comment for the property "backtrackFragment" must appear on the getter, not the setter. + protected set backtrackFragment(_value: Fragment | undefined); // (undocumented) protected bitrateTest: boolean; // (undocumented) @@ -391,14 +405,24 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP protected buffering: boolean; // (undocumented) get bufferingEnabled(): boolean; + protected calculateOptimalSwitchPoint(nextLevel: Level, bufferInfo: BufferInfo): { + fetchdelay: number; + okToFlushForwardBuffer: boolean; + }; + // (undocumented) + protected checkFragmentChanged(): boolean; // (undocumented) protected checkLiveUpdate(details: LevelDetails): void; // (undocumented) protected checkRetryDate(): void; + protected cleanupBackBuffer(): void; // (undocumented) protected clearTrackerIfNeeded(frag: Fragment): void; // (undocumented) protected config: HlsConfig; + protected get couldBacktrack(): boolean; + // Warning: (ae-setter-with-docs) The doc comment for the property "couldBacktrack" must appear on the getter, not the setter. + protected set couldBacktrack(_value: boolean); // (undocumented) protected decrypter: Decrypter; // (undocumented) @@ -411,6 +435,7 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP protected flushBufferGap(frag: Fragment): void; // (undocumented) protected flushMainBuffer(startOffset: number, endOffset: number, type?: SourceBufferName | null): void; + protected followingBufferedFrag(frag: Fragment | null): Fragment | null; // (undocumented) protected fragBufferedComplete(frag: Fragment, part: Part | null): void; // (undocumented) @@ -422,9 +447,14 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP // (undocumented) protected fragmentTracker: FragmentTracker; // (undocumented) + protected fragPlaying: Fragment | null; + // (undocumented) protected fragPrevious: MediaFragment | null; // (undocumented) - protected getAppendedFrag(position: number, playlistType?: PlaylistLevelType): Fragment | null; + protected getAppendedFrag(position: number): Fragment | null; + protected getBufferedFrag(position: number): Fragment | null; + // (undocumented) + protected getBufferOutput(): Bufferable | null; // (undocumented) protected getCurrentContext(chunkMeta: ChunkMetadata): { frag: MediaFragment; @@ -485,6 +515,7 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP protected media: HTMLMediaElement | null; // (undocumented) protected mediaBuffer: Bufferable | null; + nextLevelSwitch(): void; // (undocumented) protected nextLoadPosition: number; // (undocumented) @@ -539,6 +570,7 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP resumeBuffering(): void; // (undocumented) protected retryDate: number; + protected scheduleTrackSwitch(bufferInfo: BufferInfo, fetchdelay: number, okToFlushForwardBuffer: boolean): void; // (undocumented) protected setStartPosition(details: LevelDetails, sliding: number): void; // (undocumented) @@ -2116,6 +2148,9 @@ class Hls implements HlsEventEmitter { // (undocumented) static get MetadataSchema(): typeof MetadataSchema; get minAutoLevel(): number; + get nextAudioTrack(): number; + // Warning: (ae-setter-with-docs) The doc comment for the property "nextAudioTrack" must appear on the getter, not the setter. + set nextAudioTrack(audioTrackId: number); get nextAutoLevel(): number; // Warning: (ae-setter-with-docs) The doc comment for the property "nextAutoLevel" must appear on the getter, not the setter. set nextAutoLevel(nextLevel: number); @@ -4539,6 +4574,16 @@ export interface SteeringManifestLoadedData { export class StreamController extends BaseStreamController implements NetworkComponentAPI { constructor(hls: Hls, fragmentTracker: FragmentTracker, keyLoader: KeyLoader); // (undocumented) + protected abortCurrentFrag(): void; + protected get backtrackFragment(): Fragment | undefined; + // Warning: (ae-setter-with-docs) The doc comment for the property "backtrackFragment" must appear on the getter, not the setter. + protected set backtrackFragment(value: Fragment | undefined); + // (undocumented) + protected checkFragmentChanged(): boolean; + protected get couldBacktrack(): boolean; + // Warning: (ae-setter-with-docs) The doc comment for the property "couldBacktrack" must appear on the getter, not the setter. + protected set couldBacktrack(value: boolean); + // (undocumented) get currentFrag(): Fragment | null; // (undocumented) get currentLevel(): number; @@ -4550,6 +4595,7 @@ export class StreamController extends BaseStreamController implements NetworkCom protected flushMainBuffer(startOffset: number, endOffset: number): void; // (undocumented) get forceStartLoad(): boolean; + protected getBufferOutput(): Bufferable | null; // (undocumented) getMainFwdBufferInfo(): BufferInfo | null; // (undocumented) @@ -4563,10 +4609,9 @@ export class StreamController extends BaseStreamController implements NetworkCom // (undocumented) get maxBufferLength(): number; // (undocumented) - get nextBufferedFrag(): MediaFragment | null; + get nextBufferedFrag(): Fragment | null; // (undocumented) get nextLevel(): number; - nextLevelSwitch(): void; // (undocumented) protected onError(event: Events.ERROR, data: ErrorData): void; // (undocumented) @@ -4609,6 +4654,7 @@ export type StreamControllerConfig = { testBandwidth: boolean; liveSyncMode?: 'edge' | 'buffered'; startOnSegmentBoundary: boolean; + nextAudioTrackBufferFlushForwardOffset: number; }; // Warning: (ae-missing-release-tag) "SubtitleFragProcessedData" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) diff --git a/demo/index-light.html b/demo/index-light.html index 9e7cdd00e5b..7341e079ce9 100644 --- a/demo/index-light.html +++ b/demo/index-light.html @@ -454,6 +454,12 @@

Current audio-track:
+ + Next audio-track: + +
+ + Language / Name: diff --git a/demo/index.html b/demo/index.html index 34c420219cc..4a9a7b3c0b2 100644 --- a/demo/index.html +++ b/demo/index.html @@ -453,6 +453,12 @@

Current audio-track:
+ + Next audio-track: + +
+ + Language / Name: diff --git a/demo/main.js b/demo/main.js index 4ae20f18d6a..b63fca983a8 100644 --- a/demo/main.js +++ b/demo/main.js @@ -1393,9 +1393,12 @@ function updateAudioTrackInfo() { const buttonEnabled = 'btn-primary" '; const buttonDisabled = 'btn-success" '; let html1 = ''; + let html2 = ''; const audioTrackId = hls.audioTrack; + const nextAudioTrackId = hls.nextAudioTrack; const len = hls.audioTracks.length; const track = hls.audioTracks[audioTrackId]; + const nextTrack = hls.audioTracks[nextAudioTrackId]; for (let i = 0; i < len; i++) { html1 += buttonTemplate; @@ -1413,10 +1416,30 @@ function updateAudioTrackInfo() { ''; } + for (let i = 0; i < len; i++) { + html2 += buttonTemplate; + if (nextAudioTrackId === i) { + html2 += buttonEnabled; + } else { + html2 += buttonDisabled; + } + + html2 += + 'onclick="hls.nextAudioTrack=' + + i + + '">' + + hls.audioTracks[i].name + + ''; + } + $('#audioTrackLabel').text( track ? track.lang || track.name : 'None selected' ); $('#audioTrackControl').html(html1); + $('#audioTrackLabel').text( + nextTrack ? nextTrack.lang || nextTrack.name : 'None selected' + ); + $('#nextAudioTrackControl').html(html2); } function codecs2label(levelCodecs) { diff --git a/docs/API.md b/docs/API.md index 2d6f2761a81..eb0be90ba08 100644 --- a/docs/API.md +++ b/docs/API.md @@ -178,6 +178,7 @@ See [API Reference](https://hlsjs-dev.video-dev.org/api-docs/) for a complete li - [`hls.allAudioTracks`](#hlsallaudiotracks) - [`hls.audioTracks`](#hlsaudiotracks) - [`hls.audioTrack`](#hlsaudiotrack) + - [`hls.nextAudioTrack`](#hlsnextaudiotrack) - [Subtitle Tracks Control API](#subtitle-tracks-control-api) - [`hls.setSubtitleOption(subtitleOption)`](#hlssetsubtitleoptionsubtitleoption) - [`hls.allSubtitleTracks`](#hlsallsubtitletracks) @@ -2046,6 +2047,10 @@ get : array of supported audio tracks in the active audio group ID get/set : index of selected audio track in `hls.audioTracks` +### `hls.nextAudioTrack` + +get/set : index of the next audio track that will be selected, allowing for seamless audio track switching + ## Subtitle Tracks Control API ### `hls.setSubtitleOption(subtitleOption)` @@ -2382,7 +2387,7 @@ Full list of Events is available below: - `Hls.Events.AUDIO_TRACKS_UPDATED` - fired to notify that audio track lists has been updated - data: { audioTracks : audioTracks } - `Hls.Events.AUDIO_TRACK_SWITCHING` - fired when an audio track switching is requested - - data: { id : audio track id, type : playlist type ('AUDIO' | 'main'), url : audio track URL } + - data: { id : audio track id, type : playlist type ('AUDIO' | 'main'), url : audio track URL, flushImmediate: boolean indicating whether audio buffer should be flushed immediately when switching } - `Hls.Events.AUDIO_TRACK_SWITCHED` - fired when an audio track switch actually occurs - data: { id : audio track id } - `Hls.Events.AUDIO_TRACK_LOADING` - fired when an audio track loading starts diff --git a/src/config.ts b/src/config.ts index 7ba0901fb2f..534ff163aa9 100644 --- a/src/config.ts +++ b/src/config.ts @@ -222,6 +222,7 @@ export type StreamControllerConfig = { testBandwidth: boolean; liveSyncMode?: 'edge' | 'buffered'; startOnSegmentBoundary: boolean; + nextAudioTrackBufferFlushForwardOffset: number; }; export type GapControllerConfig = { @@ -376,6 +377,7 @@ export const hlsDefaultConfig: HlsConfig = { backBufferLength: Infinity, // used by buffer-controller frontBufferFlushThreshold: Infinity, startOnSegmentBoundary: false, // used by stream-controller + nextAudioTrackBufferFlushForwardOffset: 0.25, // used by stream-controller maxBufferSize: 60 * 1000 * 1000, // used by stream-controller maxFragLookUpTolerance: 0.25, // used by stream-controller maxBufferHole: 0.1, // used by stream-controller and gap-controller diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts index 11aa46f877f..8ea97f8e4b6 100644 --- a/src/controller/audio-stream-controller.ts +++ b/src/controller/audio-stream-controller.ts @@ -19,6 +19,7 @@ import { useAlternateAudio, } from '../utils/rendition-helper'; import type { FragmentTracker } from './fragment-tracker'; +import type { Bufferable } from '../hls'; import type Hls from '../hls'; import type { Fragment, MediaFragment, Part } from '../loader/fragment'; import type KeyLoader from '../loader/key-loader'; @@ -64,8 +65,9 @@ class AudioStreamController private mainFragLoading: FragLoadingData | null = null; private audioOnly: boolean = false; private bufferedTrack: MediaPlaylist | null = null; - private switchingTrack: MediaPlaylist | null = null; + private switchingTrack: AudioTrackSwitchingData | null = null; private trackId: number = -1; + private nextTrackId: number = -1; private waitingData: WaitingForPTSData | null = null; private mainDetails: LevelDetails | null = null; private flushing: boolean = false; @@ -177,7 +179,11 @@ class AudioStreamController } protected getLoadPosition(): number { - if (!this.startFragRequested && this.nextLoadPosition >= 0) { + if ( + !this.startFragRequested && + this.nextLoadPosition >= 0 && + this.switchingTrack?.flushImmediate !== false + ) { return this.nextLoadPosition; } return super.getLoadPosition(); @@ -307,6 +313,7 @@ class AudioStreamController } this.lastCurrentTime = media.currentTime; + this.checkFragmentChanged(); } private doTickIdle() { @@ -345,11 +352,7 @@ class AudioStreamController const bufferable = this.mediaBuffer ? this.mediaBuffer : this.media; if (this.bufferFlushed && bufferable) { this.bufferFlushed = false; - this.afterBufferFlushed( - bufferable, - ElementaryStreamTypes.AUDIO, - PlaylistLevelType.AUDIO, - ); + this.afterBufferFlushed(bufferable, ElementaryStreamTypes.AUDIO); } const bufferInfo = this.getFwdBufferInfo( @@ -372,7 +375,11 @@ class AudioStreamController const fragments = trackDetails.fragments; const start = fragments[0].start; const loadPosition = this.getLoadPosition(); - const targetBufferTime = this.flushing ? loadPosition : bufferInfo.end; + const targetBufferTime = + this.flushing || + (this.switchingTrack && !this.switchingTrack.flushImmediate) + ? loadPosition + : bufferInfo.end; if (this.switchingTrack && media) { const pos = loadPosition; @@ -488,6 +495,10 @@ class AudioStreamController if (altAudio) { this.switchingTrack = data; // main audio track are handled by stream-controller, just do something if switching to alt audio track + if (!data.flushImmediate) { + this.nextTrackId = data.id; + this.nextLevelSwitch(); + } this.flushAudioIfNeeded(data); if (this.state !== State.STOPPED) { // switching to audio track, start timer if not already started @@ -758,8 +769,6 @@ class AudioStreamController const track = this.switchingTrack; if (track) { this.bufferedTrack = track; - this.switchingTrack = null; - this.hls.trigger(Events.AUDIO_TRACK_SWITCHED, { ...track }); } } this.fragBufferedComplete(frag, part); @@ -768,6 +777,28 @@ class AudioStreamController } } + protected getBufferOutput(): Bufferable | null { + return this.mediaBuffer ? this.mediaBuffer : this.media; + } + + protected checkFragmentChanged() { + const previousFrag = this.fragPlaying; + const fragChanged = super.checkFragmentChanged(); + if (!fragChanged) { + return false; + } + + const fragPlaying = this.fragPlaying; + const fragPreviousLevel = previousFrag?.level; + if (!fragPlaying || fragPlaying.level !== fragPreviousLevel) { + this.cleanupBackBuffer(); + if (this.switchingTrack) { + this.completeAudioSwitch(this.switchingTrack); + } + } + return true; + } + protected onError(event: Events.ERROR, data: ErrorData) { if (data.fatal) { this.state = State.ERROR; @@ -842,7 +873,7 @@ class AudioStreamController } const mediaBuffer = this.mediaBuffer || this.media; if (mediaBuffer) { - this.afterBufferFlushed(mediaBuffer, type, PlaylistLevelType.AUDIO); + this.afterBufferFlushed(mediaBuffer, type); this.tick(); } } @@ -870,8 +901,18 @@ class AudioStreamController } this.state = State.PARSING; - if (this.switchingTrack && audio) { - this.completeAudioSwitch(this.switchingTrack); + if (audio && this.switchingTrack && !this.switchingTrack.flushImmediate) { + const { config } = this; + const bufferFlushDelay = config.nextAudioTrackBufferFlushForwardOffset; + const startOffset = Math.max( + this.getLoadPosition() + bufferFlushDelay, + frag.start, + ); + super.flushMainBuffer( + startOffset, + Number.POSITIVE_INFINITY, + PlaylistLevelType.AUDIO, + ); } if (initSegment?.tracks) { @@ -1022,8 +1063,8 @@ class AudioStreamController } } - private flushAudioIfNeeded(switchingTrack: MediaPlaylist) { - if (this.media && this.bufferedTrack) { + private flushAudioIfNeeded(switchingTrack: AudioTrackSwitchingData) { + if (this.media && this.bufferedTrack && switchingTrack.flushImmediate) { const { name, lang, assocLang, characteristics, audioCodec, channels } = this.bufferedTrack; if ( @@ -1045,12 +1086,18 @@ class AudioStreamController } } - private completeAudioSwitch(switchingTrack: MediaPlaylist) { + private completeAudioSwitch(switchingTrack: AudioTrackSwitchingData) { const { hls } = this; - this.flushAudioIfNeeded(switchingTrack); this.bufferedTrack = switchingTrack; this.switchingTrack = null; hls.trigger(Events.AUDIO_TRACK_SWITCHED, { ...switchingTrack }); } + + /** + * Index of next audio track loaded as scheduled by audio stream controller. + */ + get nextAudioTrack(): number { + return this.nextTrackId; + } } export default AudioStreamController; diff --git a/src/controller/audio-track-controller.ts b/src/controller/audio-track-controller.ts index 381dea97268..6e4473ea178 100644 --- a/src/controller/audio-track-controller.ts +++ b/src/controller/audio-track-controller.ts @@ -247,6 +247,16 @@ class AudioTrackController extends BasePlaylistController { this.setAudioTrack(newId); } + get nextAudioTrack(): number { + return this.trackId; + } + + set nextAudioTrack(newId: number) { + // If audio track is selected from API then don't choose from the manifest default track + this.selectDefaultTrack = false; + this.setAudioTrack(newId, false); + } + public setAudioOption( audioOption: MediaPlaylist | AudioSelectionOption | undefined, ): MediaPlaylist | null { @@ -310,7 +320,7 @@ class AudioTrackController extends BasePlaylistController { return null; } - private setAudioTrack(newId: number): void { + private setAudioTrack(newId: number, flushImmediate: boolean = true): void { const tracks = this.tracksInGroup; // check if level idx is valid @@ -331,7 +341,10 @@ class AudioTrackController extends BasePlaylistController { ); this.trackId = newId; this.currentTrack = track; - this.hls.trigger(Events.AUDIO_TRACK_SWITCHING, { ...track }); + this.hls.trigger(Events.AUDIO_TRACK_SWITCHING, { + ...track, + flushImmediate, + }); // Do not reload track unless live if (trackLoaded) { return; diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts index bbedf5d4871..a6b32c53aa5 100644 --- a/src/controller/base-stream-controller.ts +++ b/src/controller/base-stream-controller.ts @@ -39,6 +39,7 @@ import { getPartWith, updateFragPTSDTS, } from '../utils/level-helper'; +import { estimatedAudioBitrate } from '../utils/mediacapabilities-helper'; import { appendUint8Array } from '../utils/mp4-tools'; import TimeRanges from '../utils/time-ranges'; import type { FragmentTracker } from './fragment-tracker'; @@ -100,6 +101,7 @@ export default class BaseStreamController protected fragPrevious: MediaFragment | null = null; protected fragCurrent: Fragment | null = null; + protected fragPlaying: Fragment | null = null; protected fragmentTracker: FragmentTracker; protected transmuxer: TransmuxerInterface | null = null; protected _state: (typeof State)[keyof typeof State] = State.STOPPED; @@ -198,6 +200,34 @@ export default class BaseStreamController return this.buffering; } + /** + * Get backtrack fragment. Returns null in base class. + * Override in stream-controller to return actual backtrack fragment. + */ + protected get backtrackFragment(): Fragment | undefined { + return undefined; + } + + /** + * Set backtrack fragment. No-op in base class. + * Override in stream-controller to set actual backtrack fragment. + */ + protected set backtrackFragment(_value: Fragment | undefined) {} + + /** + * Get could backtrack flag. Returns false in base class. + * Override in stream-controller to return actual value. + */ + protected get couldBacktrack(): boolean { + return false; + } + + /** + * Set could backtrack flag. No-op in base class. + * Override in stream-controller to set actual value. + */ + protected set couldBacktrack(_value: boolean) {} + public pauseBuffering() { this.buffering = false; } @@ -304,6 +334,7 @@ export default class BaseStreamController event: Events.MEDIA_DETACHING, data: MediaDetachingData, ) { + this.fragPlaying = null; const transferringMedia = !!data.transferMedia; const media = this.media; if (media === null) { @@ -335,7 +366,11 @@ export default class BaseStreamController protected onManifestLoading() { this.initPTS = []; - this.levels = this.levelLastLoaded = this.fragCurrent = null; + this.fragPlaying = + this.levels = + this.levelLastLoaded = + this.fragCurrent = + null; this.lastCurrentTime = this.startPosition = 0; this.startFragRequested = false; } @@ -573,9 +608,7 @@ export default class BaseStreamController bufferedInfo ? bufferedInfo.len : this.config.maxBufferLength, ); // If backtracking, always remove from the tracker without reducing max buffer length - const backtrackFragment = (this as any).backtrackFragment as - | Fragment - | undefined; + const backtrackFragment = this.backtrackFragment as Fragment | undefined; const backtracked = backtrackFragment ? (frag.sn as number) - (backtrackFragment.sn as number) : 0; @@ -1320,12 +1353,9 @@ export default class BaseStreamController return false; } - protected getAppendedFrag( - position: number, - playlistType: PlaylistLevelType = PlaylistLevelType.MAIN, - ): Fragment | null { + protected getAppendedFrag(position: number): Fragment | null { const fragOrPart = (this.fragmentTracker as any) - ? this.fragmentTracker.getAppendedFrag(position, playlistType) + ? this.fragmentTracker.getAppendedFrag(position, this.playlistType) : null; if (fragOrPart && 'fragment' in fragOrPart) { return fragOrPart.fragment; @@ -2038,7 +2068,6 @@ export default class BaseStreamController protected afterBufferFlushed( media: Bufferable, bufferType: SourceBufferName, - playlistType: PlaylistLevelType, ) { if (!media) { return; @@ -2049,7 +2078,7 @@ export default class BaseStreamController this.fragmentTracker.detectEvictedFragments( bufferType, bufferedTimeRanges, - playlistType, + this.playlistType, ); if (this.state === State.ENDED) { this.resetLoadingState(); @@ -2260,6 +2289,235 @@ export default class BaseStreamController get state(): (typeof State)[keyof typeof State] { return this._state; } + + /** + * Calculate optimal switch point by considering fetch delays and buffer info + * to avoid causing playback interruption + */ + protected calculateOptimalSwitchPoint( + nextLevel: Level, + bufferInfo: BufferInfo, + ): { fetchdelay: number; okToFlushForwardBuffer: boolean } { + let fetchdelay = 0; + const { hls, media, config, levels, playlistType } = this; + const levelDetails = this.getLevelDetails(); + if (media && !media.paused && levels) { + const maxBitrate = + playlistType === PlaylistLevelType.AUDIO + ? estimatedAudioBitrate(nextLevel.audioCodec, 128000) + : nextLevel.maxBitrate; + // add a safety delay of 1s + const ttfbSec = 1 + hls.ttfbEstimate / 1000; + const bandwidth = hls.bandwidthEstimate * config.abrBandWidthUpFactor; + const fragDuration = + (levelDetails && + (this.loadingParts + ? levelDetails.partTarget + : levelDetails.averagetargetduration)) || + this.fragCurrent?.duration || + 6; + fetchdelay = ttfbSec + (maxBitrate * fragDuration) / bandwidth; + if (!nextLevel.details) { + fetchdelay += ttfbSec; + } + } + + const currentTime = this.media?.currentTime || this.getLoadPosition(); + // Do not flush in live stream with low buffer + const okToFlushForwardBuffer = + !levelDetails?.live || bufferInfo.end - currentTime > fetchdelay * 1.5; + + return { fetchdelay, okToFlushForwardBuffer }; + } + + /** + * Generic track switching scheduler that prevents buffering interruptions + * by finding optimal flush points in the buffer + * This method can be overridden by subclasses with specific implementation details + */ + protected scheduleTrackSwitch( + bufferInfo: BufferInfo, + fetchdelay: number, + okToFlushForwardBuffer: boolean, + ): void { + const { media, playlistType } = this; + if (!media || !bufferInfo) { + return; + } + + // find buffer range that will be reached once new fragment will be fetched + const bufferedFrag = okToFlushForwardBuffer + ? this.getBufferedFrag(this.getLoadPosition() + fetchdelay) + : null; + + if (bufferedFrag) { + // we can flush buffer range following this one without stalling playback + const nextBufferedFrag = this.followingBufferedFrag(bufferedFrag); + if (nextBufferedFrag) { + // if we are here, we can also cancel any loading/demuxing in progress, as they are useless + this.abortCurrentFrag(); + // start flush position is in next buffered frag. Leave some padding for non-independent segments and smoother playback. + const maxStart = nextBufferedFrag.maxStartPTS + ? nextBufferedFrag.maxStartPTS + : nextBufferedFrag.start; + const fragDuration = nextBufferedFrag.duration; + const startPts = Math.max( + bufferedFrag.end, + maxStart + + Math.min( + Math.max( + fragDuration - this.config.maxFragLookUpTolerance, + fragDuration * (this.couldBacktrack ? 0.5 : 0.125), + ), + fragDuration * (this.couldBacktrack ? 0.75 : 0.25), + ), + ); + const bufferType = + playlistType === PlaylistLevelType.MAIN ? null : 'audio'; + // Flush forward buffer from next buffered frag start to infinity + this.flushMainBuffer(startPts, Number.POSITIVE_INFINITY, bufferType); + // Flush back buffer (excluding current fragment) + this.cleanupBackBuffer(); + } + } + } + + /** + * Handle back-buffer cleanup during track switching + */ + protected cleanupBackBuffer(): void { + const { media, playlistType } = this; + if (!media) { + return; + } + + // remove back-buffer + const fragPlayingCurrent = this.getAppendedFrag(this.getLoadPosition()); + if (fragPlayingCurrent && fragPlayingCurrent.start > 1) { + const isAudio = playlistType === PlaylistLevelType.AUDIO; + // flush buffer preceding current fragment (flush until current fragment start offset) + // minus 1s to avoid video freezing, that could happen if we flush keyframe of current video ... + this.flushMainBuffer( + 0, + fragPlayingCurrent.start - (isAudio ? 0 : 1), + isAudio ? 'audio' : null, + ); + } + } + + /** + * Gets buffered fragment at the specified position + */ + protected getBufferedFrag(position: number): Fragment | null { + return this.fragmentTracker.getBufferedFrag(position, this.playlistType); + } + + /** + * Gets the next buffered fragment following the given fragment + */ + protected followingBufferedFrag(frag: Fragment | null): Fragment | null { + if (frag) { + // try to get range of next fragment (500ms after this range) + return this.getBufferedFrag(frag.end + 0.5); + } + return null; + } + + /** + * Aborts the current fragment loading and resets state + * Can be overridden by subclasses for specific behavior + */ + protected abortCurrentFrag(): void { + const fragCurrent = this.fragCurrent; + this.fragCurrent = null; + if (fragCurrent) { + fragCurrent.abortRequests(); + this.fragmentTracker.removeFragment(fragCurrent); + } + switch (this.state) { + case State.KEY_LOADING: + case State.FRAG_LOADING: + case State.FRAG_LOADING_WAITING_RETRY: + case State.PARSING: + case State.PARSED: + this.state = State.IDLE; + break; + } + this.nextLoadPosition = this.getLoadPosition(); + } + + protected checkFragmentChanged(): boolean { + const video = this.media; + let fragPlayingCurrent: Fragment | null = null; + if (video && video.readyState > 1 && video.seeking === false) { + const currentTime = video.currentTime; + /* if video element is in seeked state, currentTime can only increase. + (assuming that playback rate is positive ...) + As sometimes currentTime jumps back to zero after a + media decode error, check this, to avoid seeking back to + wrong position after a media decode error + */ + + if (BufferHelper.isBuffered(video, currentTime)) { + fragPlayingCurrent = this.getAppendedFrag(currentTime); + } else if (BufferHelper.isBuffered(video, currentTime + 0.1)) { + /* ensure that FRAG_CHANGED event is triggered at startup, + when first video frame is displayed and playback is paused. + add a tolerance of 100ms, in case current position is not buffered, + check if current pos+100ms is buffered and use that buffer range + for FRAG_CHANGED event reporting */ + fragPlayingCurrent = this.getAppendedFrag(currentTime + 0.1); + } + if (fragPlayingCurrent) { + this.backtrackFragment = undefined; + const fragPlaying = this.fragPlaying; + const fragCurrentLevel = fragPlayingCurrent.level; + if ( + !fragPlaying || + fragPlayingCurrent.sn !== fragPlaying.sn || + fragPlaying.level !== fragCurrentLevel + ) { + this.fragPlaying = fragPlayingCurrent; + return true; + } + } + } + return false; + } + + protected getBufferOutput(): Bufferable | null { + return null; + } + + /** + * try to switch ASAP without breaking video playback: + * in order to ensure smooth but quick level switching, + * we need to find the next flushable buffer range + * we should take into account new segment fetch time + */ + public nextLevelSwitch() { + const { levels, media, hls, config, playlistType } = this; + // ensure that media is defined and that metadata are available (to retrieve currentTime) + if (media?.readyState && levels && hls && config) { + const bufferOutput = this.getBufferOutput(); + const bufferInfo = this.getFwdBufferInfo(bufferOutput, playlistType); + if (!bufferInfo) { + return; + } + + const nextLevelId = + playlistType === PlaylistLevelType.AUDIO + ? hls.nextAudioTrack + : hls.nextLoadLevel; + const nextLevel = levels[nextLevelId]; + + const { fetchdelay, okToFlushForwardBuffer } = + this.calculateOptimalSwitchPoint(nextLevel, bufferInfo); + + this.scheduleTrackSwitch(bufferInfo, fetchdelay, okToFlushForwardBuffer); + } + this.tickImmediate(); + } } function interstitialsEnabled(config: HlsConfig): boolean { diff --git a/src/controller/stream-controller.ts b/src/controller/stream-controller.ts index ded3caf1adb..e231e0be320 100644 --- a/src/controller/stream-controller.ts +++ b/src/controller/stream-controller.ts @@ -49,16 +49,16 @@ import type { import type { Level } from '../types/level'; import type { Track, TrackSet } from '../types/track'; import type { TransmuxerResult } from '../types/transmuxer'; -import type { BufferInfo } from '../utils/buffer-helper'; +import type { Bufferable, BufferInfo } from '../utils/buffer-helper'; -const TICK_INTERVAL = 100; // how often to tick in ms - -const enum AlternateAudio { +export const enum AlternateAudio { DISABLED = 0, SWITCHING, SWITCHED, } +const TICK_INTERVAL = 100; // how often to tick in ms + export default class StreamController extends BaseStreamController implements NetworkComponentAPI @@ -69,9 +69,8 @@ export default class StreamController private _hasEnoughToStart: boolean = false; private altAudio: AlternateAudio = AlternateAudio.DISABLED; private audioOnly: boolean = false; - private fragPlaying: Fragment | null = null; - private couldBacktrack: boolean = false; - private backtrackFragment: Fragment | null = null; + private _couldBacktrack: boolean = false; + private _backtrackFragment: Fragment | undefined = undefined; private audioCodecSwitch: boolean = false; private videoBuffer: ExtendedSourceBuffer | null = null; @@ -317,7 +316,7 @@ export default class StreamController this.backtrackFragment && this.backtrackFragment.start > bufferInfo.end ) { - this.backtrackFragment = null; + this.backtrackFragment = undefined; } const targetBufferTime = this.backtrackFragment ? this.backtrackFragment.start @@ -339,7 +338,7 @@ export default class StreamController this.fragmentTracker.removeFragment(backtrackFrag); } } else if (this.backtrackFragment && bufferInfo.len) { - this.backtrackFragment = null; + this.backtrackFragment = undefined; } // Avoid loop loading by using nextLoadPosition set for backtracking and skipping consecutive GAP tags if (frag && this.isLoopLoading(frag, targetBufferTime)) { @@ -355,7 +354,7 @@ export default class StreamController ? this.videoBuffer : this.mediaBuffer) || this.media; if (mediaBuffer) { - this.afterBufferFlushed(mediaBuffer, type, PlaylistLevelType.MAIN); + this.afterBufferFlushed(mediaBuffer, type); } } frag = this.getNextFragmentLoopLoading( @@ -402,21 +401,6 @@ export default class StreamController } } - private getBufferedFrag(position: number) { - return this.fragmentTracker.getBufferedFrag( - position, - PlaylistLevelType.MAIN, - ); - } - - private followingBufferedFrag(frag: Fragment | null) { - if (frag) { - // try to get range of next fragment (500ms after this range) - return this.getBufferedFrag(frag.end + 0.5); - } - return null; - } - /* on immediate level switch : - pause playback if playing @@ -429,106 +413,70 @@ export default class StreamController } /** - * try to switch ASAP without breaking video playback: - * in order to ensure smooth but quick level switching, - * we need to find the next flushable buffer range - * we should take into account new segment fetch time + * Get the buffer output to use for buffer calculations. + * Override to use altAudio logic in stream-controller. */ - public nextLevelSwitch() { - const { levels, media, hls, config } = this; - // ensure that media is defined and that metadata are available (to retrieve currentTime) - if (media?.readyState && levels && hls && config) { - const bufferInfo = this.getMainFwdBufferInfo(); - if (!bufferInfo) { - return; - } - const levelDetails = this.getLevelDetails(); - - let fetchdelay = 0; - if (!media.paused) { - // add a safety delay of 1s - const ttfbSec = 1 + hls.ttfbEstimate / 1000; - const bandwidth = hls.bandwidthEstimate * config.abrBandWidthUpFactor; - const nextLevelId = hls.nextLoadLevel; - const nextLevel = levels[nextLevelId]; - const fragDuration = - (levelDetails && - (this.loadingParts - ? levelDetails.partTarget - : levelDetails.averagetargetduration)) || - this.fragCurrent?.duration || - 6; - fetchdelay = - ttfbSec + (nextLevel.maxBitrate * fragDuration) / bandwidth; - if (!nextLevel.details) { - fetchdelay += ttfbSec; - } - } - - // Do not flush in live stream with low buffer - - const okToFlushForwardBuffer = - !levelDetails?.live || - (bufferInfo.len || 0) > levelDetails.targetduration * 2; - - // find buffer range that will be reached once new fragment will be fetched - const bufferedFrag = okToFlushForwardBuffer - ? this.getBufferedFrag(media.currentTime + fetchdelay) - : null; - if (bufferedFrag) { - // we can flush buffer range following this one without stalling playback - const nextBufferedFrag = this.followingBufferedFrag(bufferedFrag); - if (nextBufferedFrag) { - // if we are here, we can also cancel any loading/demuxing in progress, as they are useless - this.abortCurrentFrag(); - // start flush position is in next buffered frag. Leave some padding for non-independent segments and smoother playback. - const maxStart = nextBufferedFrag.maxStartPTS - ? nextBufferedFrag.maxStartPTS - : nextBufferedFrag.start; - const fragDuration = nextBufferedFrag.duration; - const startPts = Math.max( - bufferedFrag.end, - maxStart + - Math.min( - Math.max( - fragDuration - this.config.maxFragLookUpTolerance, - fragDuration * (this.couldBacktrack ? 0.5 : 0.125), - ), - fragDuration * (this.couldBacktrack ? 0.75 : 0.25), - ), - ); - this.flushMainBuffer(startPts, Number.POSITIVE_INFINITY); - } - } - // remove back-buffer - const fragPlayingCurrent = this.getAppendedFrag(media.currentTime); - if (fragPlayingCurrent && fragPlayingCurrent.start > 1) { - // flush buffer preceding current fragment (flush until current fragment start offset) - // minus 1s to avoid video freezing, that could happen if we flush keyframe of current video ... - this.flushMainBuffer(0, fragPlayingCurrent.start - 1); - } + protected getBufferOutput(): Bufferable | null { + if (this.mediaBuffer && this.altAudio === AlternateAudio.SWITCHED) { + return this.mediaBuffer; } - this.tickImmediate(); + return this.media; } - private abortCurrentFrag() { - const fragCurrent = this.fragCurrent; - this.fragCurrent = null; - this.backtrackFragment = null; - if (fragCurrent) { - fragCurrent.abortRequests(); - this.fragmentTracker.removeFragment(fragCurrent); + protected checkFragmentChanged(): boolean { + const previousFrag = this.fragPlaying; + const fragChanged = super.checkFragmentChanged(); + if (!fragChanged) { + return false; } - switch (this.state) { - case State.KEY_LOADING: - case State.FRAG_LOADING: - case State.FRAG_LOADING_WAITING_RETRY: - case State.PARSING: - case State.PARSED: - this.state = State.IDLE; - break; + + const fragPlaying = this.fragPlaying; + if (fragPlaying && previousFrag) { + const fragCurrentLevel = fragPlaying.level; + this.hls.trigger(Events.FRAG_CHANGED, { frag: fragPlaying }); + if ( + !fragPlaying || + fragPlaying.level !== (fragCurrentLevel as number | undefined) + ) { + this.hls.trigger(Events.LEVEL_SWITCHED, { + level: fragCurrentLevel, + }); + } } - this.nextLoadPosition = this.getLoadPosition(); + return true; + } + + /** + * Get backtrack fragment. Override to return actual backtrack fragment. + */ + protected get backtrackFragment(): Fragment | undefined { + return this._backtrackFragment; + } + + /** + * Set backtrack fragment. Override to set actual backtrack fragment. + */ + protected set backtrackFragment(value: Fragment | undefined) { + this._backtrackFragment = value; + } + + /** + * Get could backtrack flag. Override to return actual value. + */ + protected get couldBacktrack(): boolean { + return this._couldBacktrack; + } + + /** + * Set could backtrack flag. Override to set actual value. + */ + protected set couldBacktrack(value: boolean) { + this._couldBacktrack = value; + } + + protected abortCurrentFrag(): void { + this.backtrackFragment = undefined; + super.abortCurrentFrag(); } protected flushMainBuffer(startOffset: number, endOffset: number) { @@ -559,7 +507,6 @@ export default class StreamController removeEventListener(media, 'seeked', this.onMediaSeeked); } this.videoBuffer = null; - this.fragPlaying = null; super.onMediaDetaching(event, data); const transferringMedia = !!data.transferMedia; if (transferringMedia) { @@ -611,7 +558,7 @@ export default class StreamController this.log('Trigger BUFFER_RESET'); this.hls.trigger(Events.BUFFER_RESET, undefined); this.couldBacktrack = false; - this.fragPlaying = this.backtrackFragment = null; + this.backtrackFragment = undefined; this.altAudio = AlternateAudio.DISABLED; this.audioOnly = false; } @@ -1099,7 +1046,7 @@ export default class StreamController ? this.videoBuffer : this.mediaBuffer) || this.media; if (mediaBuffer) { - this.afterBufferFlushed(mediaBuffer, type, PlaylistLevelType.MAIN); + this.afterBufferFlushed(mediaBuffer, type); this.tick(); } } @@ -1566,10 +1513,7 @@ export default class StreamController public getMainFwdBufferInfo(): BufferInfo | null { // Observe video SourceBuffer (this.mediaBuffer) only when alt-audio is used, otherwise observe combined media buffer - const bufferOutput = - this.mediaBuffer && this.altAudio === AlternateAudio.SWITCHED - ? this.mediaBuffer - : this.media; + const bufferOutput = this.getBufferOutput(); return this.getFwdBufferInfo(bufferOutput, PlaylistLevelType.MAIN); } @@ -1594,52 +1538,6 @@ export default class StreamController this.state = State.IDLE; } - private checkFragmentChanged() { - const video = this.media; - let fragPlayingCurrent: Fragment | null = null; - if (video && video.readyState > 1 && video.seeking === false) { - const currentTime = video.currentTime; - /* if video element is in seeked state, currentTime can only increase. - (assuming that playback rate is positive ...) - As sometimes currentTime jumps back to zero after a - media decode error, check this, to avoid seeking back to - wrong position after a media decode error - */ - - if (BufferHelper.isBuffered(video, currentTime)) { - fragPlayingCurrent = this.getAppendedFrag(currentTime); - } else if (BufferHelper.isBuffered(video, currentTime + 0.1)) { - /* ensure that FRAG_CHANGED event is triggered at startup, - when first video frame is displayed and playback is paused. - add a tolerance of 100ms, in case current position is not buffered, - check if current pos+100ms is buffered and use that buffer range - for FRAG_CHANGED event reporting */ - fragPlayingCurrent = this.getAppendedFrag(currentTime + 0.1); - } - if (fragPlayingCurrent) { - this.backtrackFragment = null; - const fragPlaying = this.fragPlaying; - const fragCurrentLevel = fragPlayingCurrent.level; - if ( - !fragPlaying || - fragPlayingCurrent.sn !== fragPlaying.sn || - fragPlaying.level !== fragCurrentLevel - ) { - this.fragPlaying = fragPlayingCurrent; - this.hls.trigger(Events.FRAG_CHANGED, { frag: fragPlayingCurrent }); - if ( - !fragPlaying || - fragPlaying.level !== (fragCurrentLevel as number | undefined) - ) { - this.hls.trigger(Events.LEVEL_SWITCHED, { - level: fragCurrentLevel, - }); - } - } - } - } - } - get nextLevel(): number { const frag = this.nextBufferedFrag; if (frag) { diff --git a/src/hls.ts b/src/hls.ts index cf61a67b01a..8bbec17580c 100644 --- a/src/hls.ts +++ b/src/hls.ts @@ -1068,6 +1068,28 @@ export default class Hls implements HlsEventEmitter { } } + /** + * Index of next audio track as scheduled by audio stream controller. + */ + get nextAudioTrack(): number { + return this.audioStreamController?.nextAudioTrack ?? -1; + } + + /** + * Set audio track index for next loaded data. + * This will switch the audio track asap, without interrupting playback. + * May abort current loading of data, and flush parts of buffer(outside + * currently played fragment region). Audio Track Switched event will be + * delayed until the currently playing fragment is of the next audio track. + * @param audioTrackId - Pass -1 for automatic level selection + */ + set nextAudioTrack(audioTrackId: number) { + const { audioTrackController } = this; + if (audioTrackController) { + audioTrackController.nextAudioTrack = audioTrackId; + } + } + /** * get the complete list of subtitle tracks across all media groups */ diff --git a/src/types/events.ts b/src/types/events.ts index 186da9ae0f0..7cc5d00980f 100644 --- a/src/types/events.ts +++ b/src/types/events.ts @@ -253,7 +253,9 @@ export interface LevelPTSUpdatedData { end: number; } -export interface AudioTrackSwitchingData extends MediaPlaylist {} +export interface AudioTrackSwitchingData extends MediaPlaylist { + flushImmediate?: boolean; +} export interface AudioTrackSwitchedData extends MediaPlaylist {} diff --git a/src/utils/mediacapabilities-helper.ts b/src/utils/mediacapabilities-helper.ts index 280a30c9561..b3806ef75cf 100644 --- a/src/utils/mediacapabilities-helper.ts +++ b/src/utils/mediacapabilities-helper.ts @@ -261,8 +261,8 @@ function makeAudioConfigurations( return []; } -function estimatedAudioBitrate( - audioCodec: string, +export function estimatedAudioBitrate( + audioCodec: string | undefined, levelBitrate: number, ): number { if (levelBitrate <= 1) {