From f0068c0374065b00694b6ab04d72b29b34962cdd Mon Sep 17 00:00:00 2001 From: Kyle Seager Date: Mon, 13 Oct 2025 16:34:02 -0600 Subject: [PATCH 01/11] Implemented smooth audio switching --- demo/index-light.html | 6 + demo/index.html | 6 + demo/main.js | 23 ++ src/controller/audio-stream-controller.ts | 68 +++++- src/controller/audio-track-controller.ts | 17 +- src/controller/base-stream-controller.ts | 277 +++++++++++++++++++++- src/controller/stream-controller.ts | 179 +------------- src/hls.ts | 25 ++ src/types/events.ts | 4 +- src/types/media-playlist.ts | 1 + 10 files changed, 420 insertions(+), 186 deletions(-) diff --git a/demo/index-light.html b/demo/index-light.html index 9e7cdd00e5b..7341e079ce9 100644 --- a/demo/index-light.html +++ b/demo/index-light.html @@ -454,6 +454,12 @@

Current audio-track:
+ + Next audio-track: + +
+ + Language / Name: diff --git a/demo/index.html b/demo/index.html index 34c420219cc..4a9a7b3c0b2 100644 --- a/demo/index.html +++ b/demo/index.html @@ -453,6 +453,12 @@

Current audio-track:
+ + Next audio-track: + +
+ + Language / Name: diff --git a/demo/main.js b/demo/main.js index 4ae20f18d6a..b63fca983a8 100644 --- a/demo/main.js +++ b/demo/main.js @@ -1393,9 +1393,12 @@ function updateAudioTrackInfo() { const buttonEnabled = 'btn-primary" '; const buttonDisabled = 'btn-success" '; let html1 = ''; + let html2 = ''; const audioTrackId = hls.audioTrack; + const nextAudioTrackId = hls.nextAudioTrack; const len = hls.audioTracks.length; const track = hls.audioTracks[audioTrackId]; + const nextTrack = hls.audioTracks[nextAudioTrackId]; for (let i = 0; i < len; i++) { html1 += buttonTemplate; @@ -1413,10 +1416,30 @@ function updateAudioTrackInfo() { ''; } + for (let i = 0; i < len; i++) { + html2 += buttonTemplate; + if (nextAudioTrackId === i) { + html2 += buttonEnabled; + } else { + html2 += buttonDisabled; + } + + html2 += + 'onclick="hls.nextAudioTrack=' + + i + + '">' + + hls.audioTracks[i].name + + ''; + } + $('#audioTrackLabel').text( track ? track.lang || track.name : 'None selected' ); $('#audioTrackControl').html(html1); + $('#audioTrackLabel').text( + nextTrack ? nextTrack.lang || nextTrack.name : 'None selected' + ); + $('#nextAudioTrackControl').html(html2); } function codecs2label(levelCodecs) { diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts index d4cc58c5c70..aa15091ed95 100644 --- a/src/controller/audio-stream-controller.ts +++ b/src/controller/audio-stream-controller.ts @@ -66,11 +66,13 @@ class AudioStreamController private bufferedTrack: MediaPlaylist | null = null; private switchingTrack: MediaPlaylist | null = null; private trackId: number = -1; + private nextTrackId: number = -1; private waitingData: WaitingForPTSData | null = null; private mainDetails: LevelDetails | null = null; private flushing: boolean = false; private bufferFlushed: boolean = false; private cachedTrackLoadedData: TrackLoadedData | null = null; + private pendingAudioTrackSwitch: boolean = false; constructor( hls: Hls, @@ -307,6 +309,7 @@ class AudioStreamController } this.lastCurrentTime = media.currentTime; + this.checkFragmentChanged(PlaylistLevelType.AUDIO); } private doTickIdle() { @@ -328,6 +331,18 @@ class AudioStreamController ) { return; } + if ( + this.pendingAudioTrackSwitch && + this.fragPlaying && + this.switchingTrack + ) { + if (this.fragPlaying.level === this.nextTrackId) { + this.pendingAudioTrackSwitch = false; + this.cleanupBackBuffer(); + this.hls.trigger(Events.AUDIO_TRACK_SWITCHED, this.switchingTrack); + this.switchingTrack = null; + } + } const levelInfo = levels[trackId]; @@ -360,7 +375,11 @@ class AudioStreamController return; } - if (!this.switchingTrack && this._streamEnded(bufferInfo, trackDetails)) { + if ( + !this.switchingTrack && + !this.pendingAudioTrackSwitch && + this._streamEnded(bufferInfo, trackDetails) + ) { hls.trigger(Events.BUFFER_EOS, { type: 'audio' }); this.state = State.ENDED; return; @@ -758,8 +777,10 @@ class AudioStreamController const track = this.switchingTrack; if (track) { this.bufferedTrack = track; - this.switchingTrack = null; - this.hls.trigger(Events.AUDIO_TRACK_SWITCHED, { ...track }); + if (!this.pendingAudioTrackSwitch) { + this.switchingTrack = null; + this.hls.trigger(Events.AUDIO_TRACK_SWITCHED, { ...track }); + } } } this.fragBufferedComplete(frag, part); @@ -1022,7 +1043,7 @@ class AudioStreamController } } - private flushAudioIfNeeded(switchingTrack: MediaPlaylist) { + private flushAudioIfNeeded(switchingTrack: AudioTrackSwitchingData) { if (this.media && this.bufferedTrack) { const { name, lang, assocLang, characteristics, audioCodec, channels } = this.bufferedTrack; @@ -1033,7 +1054,10 @@ class AudioStreamController audioMatchPredicate, ) ) { - if (useAlternateAudio(switchingTrack.url, this.hls)) { + if ( + (useAlternateAudio(switchingTrack.url, this.hls), + switchingTrack.flushImmediate) + ) { this.log('Switching audio track : flushing all audio'); super.flushMainBuffer(0, Number.POSITIVE_INFINITY, 'audio'); this.bufferedTrack = null; @@ -1045,12 +1069,40 @@ class AudioStreamController } } - private completeAudioSwitch(switchingTrack: MediaPlaylist) { + private completeAudioSwitch(switchingTrack: AudioTrackSwitchingData) { const { hls } = this; this.flushAudioIfNeeded(switchingTrack); this.bufferedTrack = switchingTrack; - this.switchingTrack = null; - hls.trigger(Events.AUDIO_TRACK_SWITCHED, { ...switchingTrack }); + this.pendingAudioTrackSwitch = !switchingTrack.flushImmediate; + if (switchingTrack.flushImmediate) { + this.switchingTrack = null; + hls.trigger(Events.AUDIO_TRACK_SWITCHED, { ...switchingTrack }); + } + } + + /** + * Index of next audio track loaded as scheduled by audio stream controller. + */ + get nextAudioTrack(): number { + return this.nextTrackId; + } + + /** + * Set next audio track index for seamless audio track switching. + * This schedules audio track switching without interrupting playback. + */ + set nextAudioTrack(audioTrackId: number) { + this.nextTrackId = audioTrackId; + } + + /** + * try to switch ASAP without breaking audio playback: + * in order to ensure smooth but quick audio track switching, + * we need to find the next flushable buffer range + * we should take into account new segment fetch time + */ + public nextAudioTrackSwitch(): void { + super.nextLevelSwitch(PlaylistLevelType.AUDIO); } } export default AudioStreamController; diff --git a/src/controller/audio-track-controller.ts b/src/controller/audio-track-controller.ts index 433fed22b60..45a2bfbc759 100644 --- a/src/controller/audio-track-controller.ts +++ b/src/controller/audio-track-controller.ts @@ -244,6 +244,16 @@ class AudioTrackController extends BasePlaylistController { this.setAudioTrack(newId); } + get nextAudioTrack(): number { + return this.trackId; + } + + set nextAudioTrack(newId: number) { + // If audio track is selected from API then don't choose from the manifest default track + this.selectDefaultTrack = false; + this.setAudioTrack(newId, false); + } + public setAudioOption( audioOption: MediaPlaylist | AudioSelectionOption | undefined, ): MediaPlaylist | null { @@ -307,7 +317,7 @@ class AudioTrackController extends BasePlaylistController { return null; } - private setAudioTrack(newId: number): void { + private setAudioTrack(newId: number, flushImmediate: boolean = true): void { const tracks = this.tracksInGroup; // check if level idx is valid @@ -328,7 +338,10 @@ class AudioTrackController extends BasePlaylistController { ); this.trackId = newId; this.currentTrack = track; - this.hls.trigger(Events.AUDIO_TRACK_SWITCHING, { ...track }); + this.hls.trigger(Events.AUDIO_TRACK_SWITCHING, { + ...track, + flushImmediate, + }); // Do not reload track unless live if (trackLoaded) { return; diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts index f00b94a1a2a..06f9123b4bb 100644 --- a/src/controller/base-stream-controller.ts +++ b/src/controller/base-stream-controller.ts @@ -92,6 +92,12 @@ export type InFlightData = { state: (typeof State)[keyof typeof State]; }; +export const enum AlternateAudio { + DISABLED = 0, + SWITCHING, + SWITCHED, +} + export default class BaseStreamController extends TaskLoop implements NetworkComponentAPI @@ -100,7 +106,11 @@ export default class BaseStreamController protected fragPrevious: MediaFragment | null = null; protected fragCurrent: Fragment | null = null; + protected fragPlaying: Fragment | null = null; + protected backtrackFragment: Fragment | null = null; + protected altAudio: AlternateAudio = AlternateAudio.DISABLED; protected fragmentTracker: FragmentTracker; + protected couldBacktrack: boolean = false; protected transmuxer: TransmuxerInterface | null = null; protected _state: (typeof State)[keyof typeof State] = State.STOPPED; protected playlistType: PlaylistLevelType; @@ -304,6 +314,7 @@ export default class BaseStreamController event: Events.MEDIA_DETACHING, data: MediaDetachingData, ) { + this.fragPlaying = null; const transferringMedia = !!data.transferMedia; const media = this.media; if (media === null) { @@ -338,6 +349,7 @@ export default class BaseStreamController this.levels = this.levelLastLoaded = this.fragCurrent = null; this.lastCurrentTime = this.startPosition = 0; this.startFragRequested = false; + this.fragPlaying = this.backtrackFragment = null; } protected onError(event: Events.ERROR, data: ErrorData) {} @@ -567,9 +579,7 @@ export default class BaseStreamController bufferedInfo ? bufferedInfo.len : this.config.maxBufferLength, ); // If backtracking, always remove from the tracker without reducing max buffer length - const backtrackFragment = (this as any).backtrackFragment as - | Fragment - | undefined; + const backtrackFragment = this.backtrackFragment as Fragment | undefined; const backtracked = backtrackFragment ? (frag.sn as number) - (backtrackFragment.sn as number) : 0; @@ -2234,6 +2244,267 @@ export default class BaseStreamController get state(): (typeof State)[keyof typeof State] { return this._state; } + + /** + * Calculate optimal switch point by considering fetch delays and buffer info + * to avoid causing playback interruption + */ + protected calculateOptimalSwitchPoint( + nextLevel: Level, + bufferInfo: BufferInfo, + levelDetails: LevelDetails | undefined, + type: PlaylistLevelType, + ): { fetchdelay: number; okToFlushForwardBuffer: boolean } { + let fetchdelay = 0; + const { hls, media, config, levels } = this; + + if (media && !media.paused && levels) { + // add a safety delay of 1s for OR nextAudioTrackSwitchingSafetyDelay depending on playlist type + const safetyDelay = + type === PlaylistLevelType.AUDIO + ? (hls.config.audioPreference?.nextAudioTrackSwitchingSafetyDelay ?? + 1) + : 1; + const ttfbSec = safetyDelay + hls.ttfbEstimate / 1000; + const bandwidth = hls.bandwidthEstimate * config.abrBandWidthUpFactor; + const fragDuration = + (levelDetails && + (this.loadingParts + ? levelDetails.partTarget + : levelDetails.averagetargetduration)) || + this.fragCurrent?.duration || + 6; + fetchdelay = ttfbSec + (nextLevel.maxBitrate * fragDuration) / bandwidth; + if (!nextLevel.details) { + fetchdelay += ttfbSec; + } + } + + // Do not flush in live stream with low buffer + + const okToFlushForwardBuffer = + !levelDetails?.live || + (bufferInfo.len || 0) > levelDetails.targetduration * 2; + + return { fetchdelay, okToFlushForwardBuffer }; + } + + /** + * Generic track switching scheduler that prevents buffering interruptions + * by finding optimal flush points in the buffer + * This method can be overridden by subclasses with specific implementation details + */ + protected scheduleTrackSwitch( + bufferInfo: BufferInfo, + fetchdelay: number, + okToFlushForwardBuffer: boolean, + type: PlaylistLevelType, + ): void { + const { media } = this; + if (!media || !bufferInfo) { + return; + } + + // find buffer range that will be reached once new fragment will be fetched + const bufferedFrag = okToFlushForwardBuffer + ? this.getBufferedFrag(this.getLoadPosition() + fetchdelay) + : null; + + if (bufferedFrag) { + // we can flush buffer range following this one without stalling playback + const nextBufferedFrag = this.followingBufferedFrag(bufferedFrag); + if (nextBufferedFrag) { + // if we are here, we can also cancel any loading/demuxing in progress, as they are useless + this.abortCurrentFrag(); + // start flush position is in next buffered frag. Leave some padding for non-independent segments and smoother playback. + const maxStart = nextBufferedFrag.maxStartPTS + ? nextBufferedFrag.maxStartPTS + : nextBufferedFrag.start; + const fragDuration = nextBufferedFrag.duration; + const startPts = Math.max( + bufferedFrag.end, + maxStart + + Math.min( + Math.max( + fragDuration - this.config.maxFragLookUpTolerance, + fragDuration * (this.couldBacktrack ? 0.5 : 0.125), + ), + fragDuration * (this.couldBacktrack ? 0.75 : 0.25), + ), + ); + const bufferType = type === PlaylistLevelType.MAIN ? null : 'audio'; + // Flush forward buffer from next buffered frag start to infinity + this.flushMainBuffer(startPts, Number.POSITIVE_INFINITY, bufferType); + // Flush back buffer (excluding current fragment) + this.cleanupBackBuffer(); + } + } + } + + /** + * Handle back-buffer cleanup during track switching + */ + protected cleanupBackBuffer(): void { + const { media } = this; + if (!media) { + return; + } + + // remove back-buffer + const fragPlayingCurrent = this.getAppendedFrag(this.getLoadPosition()); + if (fragPlayingCurrent && fragPlayingCurrent.start > 1) { + // flush buffer preceding current fragment (flush until current fragment start offset) + // minus 1s to avoid video freezing, that could happen if we flush keyframe of current video ... + this.flushMainBuffer(0, fragPlayingCurrent.start - 1); + } + } + + /** + * Base method for track switching that uses common logic + * to prevent buffering interruptions + */ + public nextTrackSwitch(): void { + // Base implementation - to be overridden by subclasses + // This provides the common pattern that both audio and video controllers can use + } + + /** + * Gets buffered fragment at the specified position + */ + protected getBufferedFrag( + position: number, + type: PlaylistLevelType = PlaylistLevelType.MAIN, + ): Fragment | null { + return this.fragmentTracker.getBufferedFrag(position, type); + } + + /** + * Gets the next buffered fragment following the given fragment + */ + protected followingBufferedFrag( + frag: Fragment | null, + type: PlaylistLevelType = PlaylistLevelType.MAIN, + ): Fragment | null { + if (frag) { + // try to get range of next fragment (500ms after this range) + return this.getBufferedFrag(frag.end + 0.5, type); + } + return null; + } + + /** + * Aborts the current fragment loading and resets state + * Can be overridden by subclasses for specific behavior + */ + protected abortCurrentFrag(): void { + const fragCurrent = this.fragCurrent; + this.fragCurrent = null; + if (fragCurrent) { + fragCurrent.abortRequests(); + this.fragmentTracker.removeFragment(fragCurrent); + } + switch (this.state) { + case State.KEY_LOADING: + case State.FRAG_LOADING: + case State.FRAG_LOADING_WAITING_RETRY: + case State.PARSING: + case State.PARSED: + this.state = State.IDLE; + break; + } + this.nextLoadPosition = this.getLoadPosition(); + } + + protected checkFragmentChanged( + type: PlaylistLevelType = PlaylistLevelType.MAIN, + ) { + const video = this.media; + let fragPlayingCurrent: Fragment | null = null; + if (video && video.readyState > 1 && video.seeking === false) { + const currentTime = this.getLoadPosition(); + /* if video element is in seeked state, currentTime can only increase. + (assuming that playback rate is positive ...) + As sometimes currentTime jumps back to zero after a + media decode error, check this, to avoid seeking back to + wrong position after a media decode error + */ + + if (BufferHelper.isBuffered(video, currentTime)) { + fragPlayingCurrent = this.getAppendedFrag(currentTime, type); + } else if (BufferHelper.isBuffered(video, currentTime + 0.1)) { + /* ensure that FRAG_CHANGED event is triggered at startup, + when first video frame is displayed and playback is paused. + add a tolerance of 100ms, in case current position is not buffered, + check if current pos+100ms is buffered and use that buffer range + for FRAG_CHANGED event reporting */ + fragPlayingCurrent = this.getAppendedFrag(currentTime + 0.1, type); + } + if (fragPlayingCurrent) { + this.backtrackFragment = null; + const fragPlaying = this.fragPlaying; + const fragCurrentLevel = fragPlayingCurrent.level; + if ( + !fragPlaying || + fragPlayingCurrent.sn !== fragPlaying.sn || + fragPlaying.level !== fragCurrentLevel + ) { + this.fragPlaying = fragPlayingCurrent; + this.hls.trigger(Events.FRAG_CHANGED, { frag: fragPlayingCurrent }); + if (!fragPlaying || fragPlaying.level !== fragCurrentLevel) { + this.hls.trigger(Events.LEVEL_SWITCHED, { + level: fragCurrentLevel, + }); + } + } + } + } + } + + public getMainFwdBufferInfo(): BufferInfo | null { + // Observe video SourceBuffer (this.mediaBuffer) only when alt-audio is used, otherwise observe combined media buffer + const bufferOutput = + this.mediaBuffer && this.altAudio === AlternateAudio.SWITCHED + ? this.mediaBuffer + : this.media; + return this.getFwdBufferInfo(bufferOutput, PlaylistLevelType.MAIN); + } + + public nextLevelSwitch(type: PlaylistLevelType) { + const { levels, media, hls, config } = this; + // ensure that media is defined and that metadata are available (to retrieve currentTime) + if (media?.readyState && levels && hls && config) { + const bufferOutput = + this.mediaBuffer && + (this.altAudio === AlternateAudio.SWITCHED || + type === PlaylistLevelType.AUDIO) + ? this.mediaBuffer + : this.media; + const bufferInfo = this.getFwdBufferInfo(bufferOutput, type); + if (!bufferInfo) { + return; + } + const levelDetails = this.getLevelDetails(); + + const nextLevelId = hls.nextLoadLevel; + const nextLevel = levels[nextLevelId]; + + const { fetchdelay, okToFlushForwardBuffer } = + this.calculateOptimalSwitchPoint( + nextLevel, + bufferInfo, + levelDetails, + PlaylistLevelType.MAIN, + ); + + this.scheduleTrackSwitch( + bufferInfo, + fetchdelay, + okToFlushForwardBuffer, + PlaylistLevelType.MAIN, + ); + } + this.tickImmediate(); + } } function interstitialsEnabled(config: HlsConfig): boolean { diff --git a/src/controller/stream-controller.ts b/src/controller/stream-controller.ts index d7bac6ec2f4..76ed9b209cd 100644 --- a/src/controller/stream-controller.ts +++ b/src/controller/stream-controller.ts @@ -1,4 +1,7 @@ -import BaseStreamController, { State } from './base-stream-controller'; +import BaseStreamController, { + AlternateAudio, + State, +} from './base-stream-controller'; import { findFragmentByPTS } from './fragment-finders'; import { FragmentState } from './fragment-tracker'; import { MAX_START_GAP_JUMP } from './gap-controller'; @@ -49,16 +52,9 @@ import type { import type { Level } from '../types/level'; import type { Track, TrackSet } from '../types/track'; import type { TransmuxerResult } from '../types/transmuxer'; -import type { BufferInfo } from '../utils/buffer-helper'; const TICK_INTERVAL = 100; // how often to tick in ms -const enum AlternateAudio { - DISABLED = 0, - SWITCHING, - SWITCHED, -} - export default class StreamController extends BaseStreamController implements NetworkComponentAPI @@ -67,11 +63,7 @@ export default class StreamController private level: number = -1; private _forceStartLoad: boolean = false; private _hasEnoughToStart: boolean = false; - private altAudio: AlternateAudio = AlternateAudio.DISABLED; private audioOnly: boolean = false; - private fragPlaying: Fragment | null = null; - private couldBacktrack: boolean = false; - private backtrackFragment: Fragment | null = null; private audioCodecSwitch: boolean = false; private videoBuffer: ExtendedSourceBuffer | null = null; @@ -402,21 +394,6 @@ export default class StreamController } } - private getBufferedFrag(position: number) { - return this.fragmentTracker.getBufferedFrag( - position, - PlaylistLevelType.MAIN, - ); - } - - private followingBufferedFrag(frag: Fragment | null) { - if (frag) { - // try to get range of next fragment (500ms after this range) - return this.getBufferedFrag(frag.end + 0.5); - } - return null; - } - /* on immediate level switch : - pause playback if playing @@ -435,100 +412,12 @@ export default class StreamController * we should take into account new segment fetch time */ public nextLevelSwitch() { - const { levels, media, hls, config } = this; - // ensure that media is defined and that metadata are available (to retrieve currentTime) - if (media?.readyState && levels && hls && config) { - const bufferInfo = this.getMainFwdBufferInfo(); - if (!bufferInfo) { - return; - } - const levelDetails = this.getLevelDetails(); - - let fetchdelay = 0; - if (!media.paused) { - // add a safety delay of 1s - const ttfbSec = 1 + hls.ttfbEstimate / 1000; - const bandwidth = hls.bandwidthEstimate * config.abrBandWidthUpFactor; - const nextLevelId = hls.nextLoadLevel; - const nextLevel = levels[nextLevelId]; - const fragDuration = - (levelDetails && - (this.loadingParts - ? levelDetails.partTarget - : levelDetails.averagetargetduration)) || - this.fragCurrent?.duration || - 6; - fetchdelay = - ttfbSec + (nextLevel.maxBitrate * fragDuration) / bandwidth; - if (!nextLevel.details) { - fetchdelay += ttfbSec; - } - } - - // Do not flush in live stream with low buffer - - const okToFlushForwardBuffer = - !levelDetails?.live || - (bufferInfo.len || 0) > levelDetails.targetduration * 2; - - // find buffer range that will be reached once new fragment will be fetched - const bufferedFrag = okToFlushForwardBuffer - ? this.getBufferedFrag(media.currentTime + fetchdelay) - : null; - if (bufferedFrag) { - // we can flush buffer range following this one without stalling playback - const nextBufferedFrag = this.followingBufferedFrag(bufferedFrag); - if (nextBufferedFrag) { - // if we are here, we can also cancel any loading/demuxing in progress, as they are useless - this.abortCurrentFrag(); - // start flush position is in next buffered frag. Leave some padding for non-independent segments and smoother playback. - const maxStart = nextBufferedFrag.maxStartPTS - ? nextBufferedFrag.maxStartPTS - : nextBufferedFrag.start; - const fragDuration = nextBufferedFrag.duration; - const startPts = Math.max( - bufferedFrag.end, - maxStart + - Math.min( - Math.max( - fragDuration - this.config.maxFragLookUpTolerance, - fragDuration * (this.couldBacktrack ? 0.5 : 0.125), - ), - fragDuration * (this.couldBacktrack ? 0.75 : 0.25), - ), - ); - this.flushMainBuffer(startPts, Number.POSITIVE_INFINITY); - } - } - // remove back-buffer - const fragPlayingCurrent = this.getAppendedFrag(media.currentTime); - if (fragPlayingCurrent && fragPlayingCurrent.start > 1) { - // flush buffer preceding current fragment (flush until current fragment start offset) - // minus 1s to avoid video freezing, that could happen if we flush keyframe of current video ... - this.flushMainBuffer(0, fragPlayingCurrent.start - 1); - } - } - this.tickImmediate(); + super.nextLevelSwitch(PlaylistLevelType.MAIN); } - private abortCurrentFrag() { - const fragCurrent = this.fragCurrent; - this.fragCurrent = null; + protected abortCurrentFrag(): void { + super.abortCurrentFrag(); this.backtrackFragment = null; - if (fragCurrent) { - fragCurrent.abortRequests(); - this.fragmentTracker.removeFragment(fragCurrent); - } - switch (this.state) { - case State.KEY_LOADING: - case State.FRAG_LOADING: - case State.FRAG_LOADING_WAITING_RETRY: - case State.PARSING: - case State.PARSED: - this.state = State.IDLE; - break; - } - this.nextLoadPosition = this.getLoadPosition(); } protected flushMainBuffer(startOffset: number, endOffset: number) { @@ -559,7 +448,6 @@ export default class StreamController removeEventListener(media, 'seeked', this.onMediaSeeked); } this.videoBuffer = null; - this.fragPlaying = null; super.onMediaDetaching(event, data); const transferringMedia = !!data.transferMedia; if (transferringMedia) { @@ -611,7 +499,6 @@ export default class StreamController this.log('Trigger BUFFER_RESET'); this.hls.trigger(Events.BUFFER_RESET, undefined); this.couldBacktrack = false; - this.fragPlaying = this.backtrackFragment = null; this.altAudio = AlternateAudio.DISABLED; this.audioOnly = false; } @@ -1564,15 +1451,6 @@ export default class StreamController this.tickImmediate(); } - public getMainFwdBufferInfo(): BufferInfo | null { - // Observe video SourceBuffer (this.mediaBuffer) only when alt-audio is used, otherwise observe combined media buffer - const bufferOutput = - this.mediaBuffer && this.altAudio === AlternateAudio.SWITCHED - ? this.mediaBuffer - : this.media; - return this.getFwdBufferInfo(bufferOutput, PlaylistLevelType.MAIN); - } - public get maxBufferLength(): number { const { levels, level } = this; const levelInfo = levels?.[level]; @@ -1594,49 +1472,6 @@ export default class StreamController this.state = State.IDLE; } - private checkFragmentChanged() { - const video = this.media; - let fragPlayingCurrent: Fragment | null = null; - if (video && video.readyState > 1 && video.seeking === false) { - const currentTime = video.currentTime; - /* if video element is in seeked state, currentTime can only increase. - (assuming that playback rate is positive ...) - As sometimes currentTime jumps back to zero after a - media decode error, check this, to avoid seeking back to - wrong position after a media decode error - */ - - if (BufferHelper.isBuffered(video, currentTime)) { - fragPlayingCurrent = this.getAppendedFrag(currentTime); - } else if (BufferHelper.isBuffered(video, currentTime + 0.1)) { - /* ensure that FRAG_CHANGED event is triggered at startup, - when first video frame is displayed and playback is paused. - add a tolerance of 100ms, in case current position is not buffered, - check if current pos+100ms is buffered and use that buffer range - for FRAG_CHANGED event reporting */ - fragPlayingCurrent = this.getAppendedFrag(currentTime + 0.1); - } - if (fragPlayingCurrent) { - this.backtrackFragment = null; - const fragPlaying = this.fragPlaying; - const fragCurrentLevel = fragPlayingCurrent.level; - if ( - !fragPlaying || - fragPlayingCurrent.sn !== fragPlaying.sn || - fragPlaying.level !== fragCurrentLevel - ) { - this.fragPlaying = fragPlayingCurrent; - this.hls.trigger(Events.FRAG_CHANGED, { frag: fragPlayingCurrent }); - if (!fragPlaying || fragPlaying.level !== fragCurrentLevel) { - this.hls.trigger(Events.LEVEL_SWITCHED, { - level: fragCurrentLevel, - }); - } - } - } - } - } - get nextLevel(): number { const frag = this.nextBufferedFrag; if (frag) { diff --git a/src/hls.ts b/src/hls.ts index cf61a67b01a..1cc0fa06fde 100644 --- a/src/hls.ts +++ b/src/hls.ts @@ -1068,6 +1068,31 @@ export default class Hls implements HlsEventEmitter { } } + /** + * Index of next audio track as scheduled by audio stream controller. + */ + get nextAudioTrack(): number { + return this.audioStreamController?.nextAudioTrack ?? -1; + } + + /** + * Set audio track index for next loaded data. + * This will switch the audio track asap, without interrupting playback. + * May abort current loading of data, and flush parts of buffer(outside + * currently played fragment region). Audio Track Switched event will be + * delayed until the currently playing fragment is of the next audio track. + * @param audioTrackId - Pass -1 for automatic level selection + */ + set nextAudioTrack(audioTrackId: number) { + this.logger.log(`set nextAudioTrack:${audioTrackId}`); + const { audioTrackController, audioStreamController } = this; + if (audioTrackController && audioStreamController) { + audioTrackController.nextAudioTrack = + audioStreamController.nextAudioTrack = audioTrackId; + audioStreamController.nextAudioTrackSwitch(); + } + } + /** * get the complete list of subtitle tracks across all media groups */ diff --git a/src/types/events.ts b/src/types/events.ts index 186da9ae0f0..7cc5d00980f 100644 --- a/src/types/events.ts +++ b/src/types/events.ts @@ -253,7 +253,9 @@ export interface LevelPTSUpdatedData { end: number; } -export interface AudioTrackSwitchingData extends MediaPlaylist {} +export interface AudioTrackSwitchingData extends MediaPlaylist { + flushImmediate?: boolean; +} export interface AudioTrackSwitchedData extends MediaPlaylist {} diff --git a/src/types/media-playlist.ts b/src/types/media-playlist.ts index e3748b8a8f6..934d05f7f37 100644 --- a/src/types/media-playlist.ts +++ b/src/types/media-playlist.ts @@ -32,6 +32,7 @@ export type AudioSelectionOption = { audioCodec?: string; groupId?: string; default?: boolean; + nextAudioTrackSwitchingSafetyDelay?: number; }; export type SubtitleSelectionOption = { From 8a30a776edf9a80e6043bb39d11f77955b1bee47 Mon Sep 17 00:00:00 2001 From: Kyle Seager Date: Mon, 13 Oct 2025 16:44:51 -0600 Subject: [PATCH 02/11] Updated documentation --- api-extractor/report/hls.js.api.md | 45 ++++++++++++++++++++++++++++-- docs/API.md | 7 ++++- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md index 6fbeb757a1e..93431e46cad 100644 --- a/api-extractor/report/hls.js.api.md +++ b/api-extractor/report/hls.js.api.md @@ -177,6 +177,7 @@ export type AudioSelectionOption = { audioCodec?: string; groupId?: string; default?: boolean; + nextAudioTrackSwitchingSafetyDelay?: number; }; // Warning: (ae-missing-release-tag) "AudioStreamController" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -194,6 +195,10 @@ export class AudioStreamController extends BaseStreamController implements Netwo _handleFragmentLoadProgress(data: FragLoadedData): void; // (undocumented) protected loadFragment(frag: Fragment, track: Level, targetBufferTime: number): void; + get nextAudioTrack(): number; + // Warning: (ae-setter-with-docs) The doc comment for the property "nextAudioTrack" must appear on the getter, not the setter. + set nextAudioTrack(audioTrackId: number); + nextAudioTrackSwitch(): void; // (undocumented) protected onError(event: Events.ERROR, data: ErrorData): void; // (undocumented) @@ -235,6 +240,9 @@ export class AudioTrackController extends BasePlaylistController { // (undocumented) protected loadPlaylist(hlsUrlParameters?: HlsUrlParameters): void; // (undocumented) + get nextAudioTrack(): number; + set nextAudioTrack(newId: number); + // (undocumented) protected onAudioTrackLoaded(event: Events.AUDIO_TRACK_LOADED, data: AudioTrackLoadedData): void; // (undocumented) protected onError(event: Events.ERROR, data: ErrorData): void; @@ -274,6 +282,8 @@ export interface AudioTrackSwitchedData extends MediaPlaylist { // // @public (undocumented) export interface AudioTrackSwitchingData extends MediaPlaylist { + // (undocumented) + flushImmediate?: boolean; } // Warning: (ae-missing-release-tag) "AudioTrackUpdatedData" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -379,10 +389,17 @@ export class BaseSegment { // @public (undocumented) export class BaseStreamController extends TaskLoop implements NetworkComponentAPI { constructor(hls: Hls, fragmentTracker: FragmentTracker, keyLoader: KeyLoader, logPrefix: string, playlistType: PlaylistLevelType); + protected abortCurrentFrag(): void; // (undocumented) protected afterBufferFlushed(media: Bufferable, bufferType: SourceBufferName, playlistType: PlaylistLevelType): void; // (undocumented) protected alignPlaylists(details: LevelDetails, previousDetails: LevelDetails | undefined, switchDetails: LevelDetails | undefined): number; + // Warning: (ae-forgotten-export) The symbol "AlternateAudio" needs to be exported by the entry point hls.d.ts + // + // (undocumented) + protected altAudio: AlternateAudio; + // (undocumented) + protected backtrackFragment: Fragment | null; // (undocumented) protected bitrateTest: boolean; // (undocumented) @@ -391,15 +408,24 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP protected buffering: boolean; // (undocumented) get bufferingEnabled(): boolean; + protected calculateOptimalSwitchPoint(nextLevel: Level, bufferInfo: BufferInfo, levelDetails: LevelDetails | undefined, type: PlaylistLevelType): { + fetchdelay: number; + okToFlushForwardBuffer: boolean; + }; + // (undocumented) + protected checkFragmentChanged(type?: PlaylistLevelType): void; // (undocumented) protected checkLiveUpdate(details: LevelDetails): void; // (undocumented) protected checkRetryDate(): void; + protected cleanupBackBuffer(): void; // (undocumented) protected clearTrackerIfNeeded(frag: Fragment): void; // (undocumented) protected config: HlsConfig; // (undocumented) + protected couldBacktrack: boolean; + // (undocumented) protected decrypter: Decrypter; // (undocumented) protected _doFragLoad(frag: Fragment, level: Level, targetBufferTime?: number | null, progressCallback?: FragmentLoadProgressCallback): Promise; @@ -411,6 +437,7 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP protected flushBufferGap(frag: Fragment): void; // (undocumented) protected flushMainBuffer(startOffset: number, endOffset: number, type?: SourceBufferName | null): void; + protected followingBufferedFrag(frag: Fragment | null, type?: PlaylistLevelType): Fragment | null; // (undocumented) protected fragBufferedComplete(frag: Fragment, part: Part | null): void; // (undocumented) @@ -422,9 +449,12 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP // (undocumented) protected fragmentTracker: FragmentTracker; // (undocumented) + protected fragPlaying: Fragment | null; + // (undocumented) protected fragPrevious: MediaFragment | null; // (undocumented) protected getAppendedFrag(position: number, playlistType?: PlaylistLevelType): Fragment | null; + protected getBufferedFrag(position: number, type?: PlaylistLevelType): Fragment | null; // (undocumented) protected getCurrentContext(chunkMeta: ChunkMetadata): { frag: MediaFragment; @@ -444,6 +474,8 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP // (undocumented) protected getLoadPosition(): number; // (undocumented) + getMainFwdBufferInfo(): BufferInfo | null; + // (undocumented) protected getMaxBufferLength(levelBitrate?: number): number; // (undocumented) protected getNextFragment(pos: number, levelDetails: LevelDetails): Fragment | null; @@ -486,7 +518,10 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP // (undocumented) protected mediaBuffer: Bufferable | null; // (undocumented) + nextLevelSwitch(type: PlaylistLevelType): void; + // (undocumented) protected nextLoadPosition: number; + nextTrackSwitch(): void; // (undocumented) protected onError(event: Events.ERROR, data: ErrorData): void; // (undocumented) @@ -539,6 +574,7 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP resumeBuffering(): void; // (undocumented) protected retryDate: number; + protected scheduleTrackSwitch(bufferInfo: BufferInfo, fetchdelay: number, okToFlushForwardBuffer: boolean, type: PlaylistLevelType): void; // (undocumented) protected setStartPosition(details: LevelDetails, sliding: number): void; // (undocumented) @@ -2116,6 +2152,9 @@ class Hls implements HlsEventEmitter { // (undocumented) static get MetadataSchema(): typeof MetadataSchema; get minAutoLevel(): number; + get nextAudioTrack(): number; + // Warning: (ae-setter-with-docs) The doc comment for the property "nextAudioTrack" must appear on the getter, not the setter. + set nextAudioTrack(audioTrackId: number); get nextAutoLevel(): number; // Warning: (ae-setter-with-docs) The doc comment for the property "nextAutoLevel" must appear on the getter, not the setter. set nextAutoLevel(nextLevel: number); @@ -4539,6 +4578,8 @@ export interface SteeringManifestLoadedData { export class StreamController extends BaseStreamController implements NetworkComponentAPI { constructor(hls: Hls, fragmentTracker: FragmentTracker, keyLoader: KeyLoader); // (undocumented) + protected abortCurrentFrag(): void; + // (undocumented) get currentFrag(): Fragment | null; // (undocumented) get currentLevel(): number; @@ -4551,8 +4592,6 @@ export class StreamController extends BaseStreamController implements NetworkCom // (undocumented) get forceStartLoad(): boolean; // (undocumented) - getMainFwdBufferInfo(): BufferInfo | null; - // (undocumented) protected _handleFragmentLoadProgress(data: FragLoadedData): void; // (undocumented) get hasEnoughToStart(): boolean; @@ -4563,7 +4602,7 @@ export class StreamController extends BaseStreamController implements NetworkCom // (undocumented) get maxBufferLength(): number; // (undocumented) - get nextBufferedFrag(): MediaFragment | null; + get nextBufferedFrag(): Fragment | null; // (undocumented) get nextLevel(): number; nextLevelSwitch(): void; diff --git a/docs/API.md b/docs/API.md index 359c5ddbd7d..da3aa49ec95 100644 --- a/docs/API.md +++ b/docs/API.md @@ -171,6 +171,7 @@ See [API Reference](https://hlsjs-dev.video-dev.org/api-docs/) for a complete li - [`hls.allAudioTracks`](#hlsallaudiotracks) - [`hls.audioTracks`](#hlsaudiotracks) - [`hls.audioTrack`](#hlsaudiotrack) + - [`hls.nextAudioTrack`](#hlsnextaudiotrack) - [Subtitle Tracks Control API](#subtitle-tracks-control-api) - [`hls.setSubtitleOption(subtitleOption)`](#hlssetsubtitleoptionsubtitleoption) - [`hls.allSubtitleTracks`](#hlsallsubtitletracks) @@ -1985,6 +1986,10 @@ get : array of supported audio tracks in the active audio group ID get/set : index of selected audio track in `hls.audioTracks` +### `hls.nextAudioTrack` + +get/set : index of the next audio track that will be selected, allowing for seamless audio track switching + ## Subtitle Tracks Control API ### `hls.setSubtitleOption(subtitleOption)` @@ -2321,7 +2326,7 @@ Full list of Events is available below: - `Hls.Events.AUDIO_TRACKS_UPDATED` - fired to notify that audio track lists has been updated - data: { audioTracks : audioTracks } - `Hls.Events.AUDIO_TRACK_SWITCHING` - fired when an audio track switching is requested - - data: { id : audio track id, type : playlist type ('AUDIO' | 'main'), url : audio track URL } + - data: { id : audio track id, type : playlist type ('AUDIO' | 'main'), url : audio track URL, flushImmediate: boolean indicating whether audio buffer should be flushed immediately when switching } - `Hls.Events.AUDIO_TRACK_SWITCHED` - fired when an audio track switch actually occurs - data: { id : audio track id } - `Hls.Events.AUDIO_TRACK_LOADING` - fired when an audio track loading starts From a490937c0f23c1cc6484a8ab1a9dc47d908b8dbb Mon Sep 17 00:00:00 2001 From: Kyle Seager Date: Mon, 20 Oct 2025 12:14:24 -0600 Subject: [PATCH 03/11] Fixed bug where audio track switch was delayed based on how large the buffer was --- src/controller/base-stream-controller.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts index 06f9123b4bb..5709cafb495 100644 --- a/src/controller/base-stream-controller.ts +++ b/src/controller/base-stream-controller.ts @@ -2280,11 +2280,10 @@ export default class BaseStreamController } } + const currentTime = this.media?.currentTime || this.getLoadPosition(); // Do not flush in live stream with low buffer - const okToFlushForwardBuffer = - !levelDetails?.live || - (bufferInfo.len || 0) > levelDetails.targetduration * 2; + !levelDetails?.live || bufferInfo.end - currentTime > fetchdelay * 1.5; return { fetchdelay, okToFlushForwardBuffer }; } @@ -2493,14 +2492,14 @@ export default class BaseStreamController nextLevel, bufferInfo, levelDetails, - PlaylistLevelType.MAIN, + type, ); this.scheduleTrackSwitch( bufferInfo, fetchdelay, okToFlushForwardBuffer, - PlaylistLevelType.MAIN, + type, ); } this.tickImmediate(); From 947fd7530ba6cd2bf2dd1f2dd4b41ff5b3989364 Mon Sep 17 00:00:00 2001 From: Kyle Seager Date: Wed, 5 Nov 2025 15:01:39 -0700 Subject: [PATCH 04/11] Updated Smooth Audio Track Switching to address PR comments and functional bugs --- api-extractor/report/hls.js.api.md | 3 +- src/controller/audio-stream-controller.ts | 50 +++++++++++------------ src/controller/base-stream-controller.ts | 34 +++++++++++---- src/types/media-playlist.ts | 1 + 4 files changed, 54 insertions(+), 34 deletions(-) diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md index 4e066a73648..9a3b21da8c1 100644 --- a/api-extractor/report/hls.js.api.md +++ b/api-extractor/report/hls.js.api.md @@ -178,6 +178,7 @@ export type AudioSelectionOption = { groupId?: string; default?: boolean; nextAudioTrackSwitchingSafetyDelay?: number; + nextAudioTrackBufferFlushDelay?: number; }; // Warning: (ae-missing-release-tag) "AudioStreamController" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -418,7 +419,7 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP protected checkLiveUpdate(details: LevelDetails): void; // (undocumented) protected checkRetryDate(): void; - protected cleanupBackBuffer(): void; + protected cleanupBackBuffer(type: PlaylistLevelType): void; // (undocumented) protected clearTrackerIfNeeded(frag: Fragment): void; // (undocumented) diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts index 7534f7524d7..95c5e144e3b 100644 --- a/src/controller/audio-stream-controller.ts +++ b/src/controller/audio-stream-controller.ts @@ -331,18 +331,6 @@ class AudioStreamController ) { return; } - if ( - this.pendingAudioTrackSwitch && - this.fragPlaying && - this.switchingTrack - ) { - if (this.fragPlaying.level === this.nextTrackId) { - this.pendingAudioTrackSwitch = false; - this.cleanupBackBuffer(); - this.hls.trigger(Events.AUDIO_TRACK_SWITCHED, this.switchingTrack); - this.switchingTrack = null; - } - } const levelInfo = levels[trackId]; @@ -375,11 +363,7 @@ class AudioStreamController return; } - if ( - !this.switchingTrack && - !this.pendingAudioTrackSwitch && - this._streamEnded(bufferInfo, trackDetails) - ) { + if (!this.switchingTrack && this._streamEnded(bufferInfo, trackDetails)) { hls.trigger(Events.BUFFER_EOS, { type: 'audio' }); this.state = State.ENDED; return; @@ -391,7 +375,10 @@ class AudioStreamController const fragments = trackDetails.fragments; const start = fragments[0].start; const loadPosition = this.getLoadPosition(); - const targetBufferTime = this.flushing ? loadPosition : bufferInfo.end; + const targetBufferTime = + this.flushing || this.pendingAudioTrackSwitch + ? loadPosition + : bufferInfo.end; if (this.switchingTrack && media) { const pos = loadPosition; @@ -512,6 +499,7 @@ class AudioStreamController // switching to audio track, start timer if not already started this.setInterval(TICK_INTERVAL); this.state = State.IDLE; + this.pendingAudioTrackSwitch = !data.flushImmediate; this.tick(); } } else { @@ -777,10 +765,8 @@ class AudioStreamController const track = this.switchingTrack; if (track) { this.bufferedTrack = track; - if (!this.pendingAudioTrackSwitch) { - this.switchingTrack = null; - this.hls.trigger(Events.AUDIO_TRACK_SWITCHED, { ...track }); - } + this.switchingTrack = null; + this.hls.trigger(Events.AUDIO_TRACK_SWITCHED, { ...track }); } } this.fragBufferedComplete(frag, part); @@ -1044,7 +1030,7 @@ class AudioStreamController } private flushAudioIfNeeded(switchingTrack: AudioTrackSwitchingData) { - if (this.media && this.bufferedTrack) { + if (this.media && this.bufferedTrack && switchingTrack.flushImmediate) { const { name, lang, assocLang, characteristics, audioCodec, channels } = this.bufferedTrack; if ( @@ -1062,8 +1048,20 @@ class AudioStreamController super.flushMainBuffer(0, Number.POSITIVE_INFINITY, 'audio'); this.bufferedTrack = null; } else { + const { config } = this; // Main is being buffered. Set bufferedTrack so that it is flushed when switching back to alt-audio this.bufferedTrack = switchingTrack; + const bufferFlushDelay = + config.audioPreference?.nextAudioTrackBufferFlushDelay || 0.25; + const startOffset = Math.max( + this.getLoadPosition() + bufferFlushDelay, + this.fragPrevious?.start || 0, + ); + super.flushMainBuffer( + startOffset, + Number.POSITIVE_INFINITY, + PlaylistLevelType.AUDIO, + ); } } } @@ -1071,12 +1069,12 @@ class AudioStreamController private completeAudioSwitch(switchingTrack: AudioTrackSwitchingData) { const { hls } = this; - this.flushAudioIfNeeded(switchingTrack); + this.flushAudioIfNeeded({ ...switchingTrack, flushImmediate: true }); this.bufferedTrack = switchingTrack; - this.pendingAudioTrackSwitch = !switchingTrack.flushImmediate; + this.pendingAudioTrackSwitch = false; if (switchingTrack.flushImmediate) { this.switchingTrack = null; - hls.trigger(Events.AUDIO_TRACK_SWITCHED, { ...switchingTrack }); + hls.trigger(Events.AUDIO_TRACK_SWITCHED, switchingTrack); } } diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts index 8c0a231f813..759bbc39ff4 100644 --- a/src/controller/base-stream-controller.ts +++ b/src/controller/base-stream-controller.ts @@ -2309,7 +2309,8 @@ export default class BaseStreamController const currentTime = this.media?.currentTime || this.getLoadPosition(); // Do not flush in live stream with low buffer const okToFlushForwardBuffer = - !levelDetails?.live || bufferInfo.end - currentTime > fetchdelay * 1.5; + type !== PlaylistLevelType.AUDIO && + (!levelDetails?.live || bufferInfo.end - currentTime > fetchdelay * 1.5); return { fetchdelay, okToFlushForwardBuffer }; } @@ -2361,7 +2362,7 @@ export default class BaseStreamController // Flush forward buffer from next buffered frag start to infinity this.flushMainBuffer(startPts, Number.POSITIVE_INFINITY, bufferType); // Flush back buffer (excluding current fragment) - this.cleanupBackBuffer(); + this.cleanupBackBuffer(type); } } } @@ -2369,18 +2370,25 @@ export default class BaseStreamController /** * Handle back-buffer cleanup during track switching */ - protected cleanupBackBuffer(): void { + protected cleanupBackBuffer(type: PlaylistLevelType): void { const { media } = this; if (!media) { return; } // remove back-buffer - const fragPlayingCurrent = this.getAppendedFrag(this.getLoadPosition()); + const fragPlayingCurrent = this.getAppendedFrag( + this.getLoadPosition(), + type, + ); if (fragPlayingCurrent && fragPlayingCurrent.start > 1) { // flush buffer preceding current fragment (flush until current fragment start offset) // minus 1s to avoid video freezing, that could happen if we flush keyframe of current video ... - this.flushMainBuffer(0, fragPlayingCurrent.start - 1); + this.flushMainBuffer( + 0, + fragPlayingCurrent.start - 1, + type === PlaylistLevelType.AUDIO ? 'audio' : null, + ); } } @@ -2473,8 +2481,17 @@ export default class BaseStreamController fragPlayingCurrent.sn !== fragPlaying.sn || fragPlaying.level !== fragCurrentLevel ) { + if (type === PlaylistLevelType.AUDIO) { + if (fragPlaying?.level !== fragPlayingCurrent.level) { + this.flushMainBuffer( + 0, + this.getLoadPosition(), + PlaylistLevelType.AUDIO, + ); + } + } this.fragPlaying = fragPlayingCurrent; - if (type !== PlaylistLevelType.AUDIO) { + if (type === PlaylistLevelType.MAIN) { this.hls.trigger(Events.FRAG_CHANGED, { frag: fragPlayingCurrent }); if ( !fragPlaying || @@ -2515,7 +2532,10 @@ export default class BaseStreamController } const levelDetails = this.getLevelDetails(); - const nextLevelId = hls.nextLoadLevel; + const nextLevelId = + type === PlaylistLevelType.AUDIO + ? hls.nextAudioTrack + : hls.nextLoadLevel; const nextLevel = levels[nextLevelId]; const { fetchdelay, okToFlushForwardBuffer } = diff --git a/src/types/media-playlist.ts b/src/types/media-playlist.ts index 934d05f7f37..17b6d30e8e8 100644 --- a/src/types/media-playlist.ts +++ b/src/types/media-playlist.ts @@ -33,6 +33,7 @@ export type AudioSelectionOption = { groupId?: string; default?: boolean; nextAudioTrackSwitchingSafetyDelay?: number; + nextAudioTrackBufferFlushDelay?: number; }; export type SubtitleSelectionOption = { From 5f687c61665aec18ea9e7f383ec1849e0eb457c8 Mon Sep 17 00:00:00 2001 From: Kyle Seager Date: Wed, 5 Nov 2025 17:26:47 -0700 Subject: [PATCH 05/11] Moved flushing forward buffer for nextAudioTrack to happen after transmux complete, which is before buffering the new audio track's audio segment --- src/controller/audio-stream-controller.ts | 31 +++++++++++++---------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts index 95c5e144e3b..107a31efac7 100644 --- a/src/controller/audio-stream-controller.ts +++ b/src/controller/audio-stream-controller.ts @@ -1048,20 +1048,8 @@ class AudioStreamController super.flushMainBuffer(0, Number.POSITIVE_INFINITY, 'audio'); this.bufferedTrack = null; } else { - const { config } = this; // Main is being buffered. Set bufferedTrack so that it is flushed when switching back to alt-audio this.bufferedTrack = switchingTrack; - const bufferFlushDelay = - config.audioPreference?.nextAudioTrackBufferFlushDelay || 0.25; - const startOffset = Math.max( - this.getLoadPosition() + bufferFlushDelay, - this.fragPrevious?.start || 0, - ); - super.flushMainBuffer( - startOffset, - Number.POSITIVE_INFINITY, - PlaylistLevelType.AUDIO, - ); } } } @@ -1069,10 +1057,25 @@ class AudioStreamController private completeAudioSwitch(switchingTrack: AudioTrackSwitchingData) { const { hls } = this; - this.flushAudioIfNeeded({ ...switchingTrack, flushImmediate: true }); + const { flushImmediate } = switchingTrack; + if (!flushImmediate) { + const { config } = this; + const bufferFlushDelay = + config.audioPreference?.nextAudioTrackBufferFlushDelay || 0.25; + const startOffset = Math.max( + this.getLoadPosition() + bufferFlushDelay, + this.fragPrevious?.start || 0, + ); + super.flushMainBuffer( + startOffset, + Number.POSITIVE_INFINITY, + PlaylistLevelType.AUDIO, + ); + } + this.bufferedTrack = switchingTrack; this.pendingAudioTrackSwitch = false; - if (switchingTrack.flushImmediate) { + if (flushImmediate) { this.switchingTrack = null; hls.trigger(Events.AUDIO_TRACK_SWITCHED, switchingTrack); } From b3e878bf4ae44999d1b7a4c4185c6e2578c9199a Mon Sep 17 00:00:00 2001 From: Kyle Seager Date: Mon, 10 Nov 2025 16:02:25 -0700 Subject: [PATCH 06/11] Cleaned up changes & addressed PR comments --- api-extractor/report/hls.js.api.md | 5 ++-- src/config.ts | 2 ++ src/controller/audio-stream-controller.ts | 11 +++----- src/controller/base-stream-controller.ts | 32 +++++++++-------------- src/types/media-playlist.ts | 2 -- src/utils/mediacapabilities-helper.ts | 4 +-- 6 files changed, 22 insertions(+), 34 deletions(-) diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md index 9a3b21da8c1..7a8d2c248e0 100644 --- a/api-extractor/report/hls.js.api.md +++ b/api-extractor/report/hls.js.api.md @@ -177,8 +177,6 @@ export type AudioSelectionOption = { audioCodec?: string; groupId?: string; default?: boolean; - nextAudioTrackSwitchingSafetyDelay?: number; - nextAudioTrackBufferFlushDelay?: number; }; // Warning: (ae-missing-release-tag) "AudioStreamController" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -409,7 +407,7 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP protected buffering: boolean; // (undocumented) get bufferingEnabled(): boolean; - protected calculateOptimalSwitchPoint(nextLevel: Level, bufferInfo: BufferInfo, levelDetails: LevelDetails | undefined, type: PlaylistLevelType): { + protected calculateOptimalSwitchPoint(nextLevel: Level, bufferInfo: BufferInfo, type: PlaylistLevelType): { fetchdelay: number; okToFlushForwardBuffer: boolean; }; @@ -4649,6 +4647,7 @@ export type StreamControllerConfig = { testBandwidth: boolean; liveSyncMode?: 'edge' | 'buffered'; startOnSegmentBoundary: boolean; + nextAudioTrackBufferFlushForwardOffset: number; }; // Warning: (ae-missing-release-tag) "SubtitleFragProcessedData" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) diff --git a/src/config.ts b/src/config.ts index 7ba0901fb2f..534ff163aa9 100644 --- a/src/config.ts +++ b/src/config.ts @@ -222,6 +222,7 @@ export type StreamControllerConfig = { testBandwidth: boolean; liveSyncMode?: 'edge' | 'buffered'; startOnSegmentBoundary: boolean; + nextAudioTrackBufferFlushForwardOffset: number; }; export type GapControllerConfig = { @@ -376,6 +377,7 @@ export const hlsDefaultConfig: HlsConfig = { backBufferLength: Infinity, // used by buffer-controller frontBufferFlushThreshold: Infinity, startOnSegmentBoundary: false, // used by stream-controller + nextAudioTrackBufferFlushForwardOffset: 0.25, // used by stream-controller maxBufferSize: 60 * 1000 * 1000, // used by stream-controller maxFragLookUpTolerance: 0.25, // used by stream-controller maxBufferHole: 0.1, // used by stream-controller and gap-controller diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts index 107a31efac7..41606f9715c 100644 --- a/src/controller/audio-stream-controller.ts +++ b/src/controller/audio-stream-controller.ts @@ -64,7 +64,7 @@ class AudioStreamController private mainFragLoading: FragLoadingData | null = null; private audioOnly: boolean = false; private bufferedTrack: MediaPlaylist | null = null; - private switchingTrack: MediaPlaylist | null = null; + private switchingTrack: AudioTrackSwitchingData | null = null; private trackId: number = -1; private nextTrackId: number = -1; private waitingData: WaitingForPTSData | null = null; @@ -72,7 +72,6 @@ class AudioStreamController private flushing: boolean = false; private bufferFlushed: boolean = false; private cachedTrackLoadedData: TrackLoadedData | null = null; - private pendingAudioTrackSwitch: boolean = false; constructor( hls: Hls, @@ -376,7 +375,8 @@ class AudioStreamController const start = fragments[0].start; const loadPosition = this.getLoadPosition(); const targetBufferTime = - this.flushing || this.pendingAudioTrackSwitch + this.flushing || + (this.switchingTrack && !this.switchingTrack.flushImmediate) ? loadPosition : bufferInfo.end; @@ -499,7 +499,6 @@ class AudioStreamController // switching to audio track, start timer if not already started this.setInterval(TICK_INTERVAL); this.state = State.IDLE; - this.pendingAudioTrackSwitch = !data.flushImmediate; this.tick(); } } else { @@ -1060,8 +1059,7 @@ class AudioStreamController const { flushImmediate } = switchingTrack; if (!flushImmediate) { const { config } = this; - const bufferFlushDelay = - config.audioPreference?.nextAudioTrackBufferFlushDelay || 0.25; + const bufferFlushDelay = config.nextAudioTrackBufferFlushForwardOffset; const startOffset = Math.max( this.getLoadPosition() + bufferFlushDelay, this.fragPrevious?.start || 0, @@ -1074,7 +1072,6 @@ class AudioStreamController } this.bufferedTrack = switchingTrack; - this.pendingAudioTrackSwitch = false; if (flushImmediate) { this.switchingTrack = null; hls.trigger(Events.AUDIO_TRACK_SWITCHED, switchingTrack); diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts index 759bbc39ff4..79393aba1ea 100644 --- a/src/controller/base-stream-controller.ts +++ b/src/controller/base-stream-controller.ts @@ -39,6 +39,7 @@ import { getPartWith, updateFragPTSDTS, } from '../utils/level-helper'; +import { estimatedAudioBitrate } from '../utils/mediacapabilities-helper'; import { appendUint8Array } from '../utils/mp4-tools'; import TimeRanges from '../utils/time-ranges'; import type { FragmentTracker } from './fragment-tracker'; @@ -2278,20 +2279,18 @@ export default class BaseStreamController protected calculateOptimalSwitchPoint( nextLevel: Level, bufferInfo: BufferInfo, - levelDetails: LevelDetails | undefined, type: PlaylistLevelType, ): { fetchdelay: number; okToFlushForwardBuffer: boolean } { let fetchdelay = 0; const { hls, media, config, levels } = this; - + const levelDetails = this.getLevelDetails(); if (media && !media.paused && levels) { - // add a safety delay of 1s for OR nextAudioTrackSwitchingSafetyDelay depending on playlist type - const safetyDelay = + const maxBitrate = type === PlaylistLevelType.AUDIO - ? (hls.config.audioPreference?.nextAudioTrackSwitchingSafetyDelay ?? - 1) - : 1; - const ttfbSec = safetyDelay + hls.ttfbEstimate / 1000; + ? estimatedAudioBitrate(nextLevel.audioCodec, 128000) + : nextLevel.maxBitrate; + // add a safety delay of 1s + const ttfbSec = 1 + hls.ttfbEstimate / 1000; const bandwidth = hls.bandwidthEstimate * config.abrBandWidthUpFactor; const fragDuration = (levelDetails && @@ -2300,7 +2299,7 @@ export default class BaseStreamController : levelDetails.averagetargetduration)) || this.fragCurrent?.duration || 6; - fetchdelay = ttfbSec + (nextLevel.maxBitrate * fragDuration) / bandwidth; + fetchdelay = ttfbSec + (maxBitrate * fragDuration) / bandwidth; if (!nextLevel.details) { fetchdelay += ttfbSec; } @@ -2309,8 +2308,7 @@ export default class BaseStreamController const currentTime = this.media?.currentTime || this.getLoadPosition(); // Do not flush in live stream with low buffer const okToFlushForwardBuffer = - type !== PlaylistLevelType.AUDIO && - (!levelDetails?.live || bufferInfo.end - currentTime > fetchdelay * 1.5); + !levelDetails?.live || bufferInfo.end - currentTime > fetchdelay * 1.5; return { fetchdelay, okToFlushForwardBuffer }; } @@ -2333,12 +2331,12 @@ export default class BaseStreamController // find buffer range that will be reached once new fragment will be fetched const bufferedFrag = okToFlushForwardBuffer - ? this.getBufferedFrag(this.getLoadPosition() + fetchdelay) + ? this.getBufferedFrag(this.getLoadPosition() + fetchdelay, type) : null; if (bufferedFrag) { // we can flush buffer range following this one without stalling playback - const nextBufferedFrag = this.followingBufferedFrag(bufferedFrag); + const nextBufferedFrag = this.followingBufferedFrag(bufferedFrag, type); if (nextBufferedFrag) { // if we are here, we can also cancel any loading/demuxing in progress, as they are useless this.abortCurrentFrag(); @@ -2530,7 +2528,6 @@ export default class BaseStreamController if (!bufferInfo) { return; } - const levelDetails = this.getLevelDetails(); const nextLevelId = type === PlaylistLevelType.AUDIO @@ -2539,12 +2536,7 @@ export default class BaseStreamController const nextLevel = levels[nextLevelId]; const { fetchdelay, okToFlushForwardBuffer } = - this.calculateOptimalSwitchPoint( - nextLevel, - bufferInfo, - levelDetails, - type, - ); + this.calculateOptimalSwitchPoint(nextLevel, bufferInfo, type); this.scheduleTrackSwitch( bufferInfo, diff --git a/src/types/media-playlist.ts b/src/types/media-playlist.ts index 17b6d30e8e8..e3748b8a8f6 100644 --- a/src/types/media-playlist.ts +++ b/src/types/media-playlist.ts @@ -32,8 +32,6 @@ export type AudioSelectionOption = { audioCodec?: string; groupId?: string; default?: boolean; - nextAudioTrackSwitchingSafetyDelay?: number; - nextAudioTrackBufferFlushDelay?: number; }; export type SubtitleSelectionOption = { diff --git a/src/utils/mediacapabilities-helper.ts b/src/utils/mediacapabilities-helper.ts index 280a30c9561..b3806ef75cf 100644 --- a/src/utils/mediacapabilities-helper.ts +++ b/src/utils/mediacapabilities-helper.ts @@ -261,8 +261,8 @@ function makeAudioConfigurations( return []; } -function estimatedAudioBitrate( - audioCodec: string, +export function estimatedAudioBitrate( + audioCodec: string | undefined, levelBitrate: number, ): number { if (levelBitrate <= 1) { From 4d6a3a97ab7548781fd0e8ed9d221052b3312460 Mon Sep 17 00:00:00 2001 From: Kyle Seager Date: Mon, 10 Nov 2025 16:58:59 -0700 Subject: [PATCH 07/11] Cleaned up changes --- src/controller/audio-stream-controller.ts | 64 +++++++++++++---------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts index 41606f9715c..c174d013e07 100644 --- a/src/controller/audio-stream-controller.ts +++ b/src/controller/audio-stream-controller.ts @@ -494,7 +494,9 @@ class AudioStreamController if (altAudio) { this.switchingTrack = data; // main audio track are handled by stream-controller, just do something if switching to alt audio track - this.flushAudioIfNeeded(data); + if (data.flushImmediate) { + this.flushAudioIfNeeded(data); + } if (this.state !== State.STOPPED) { // switching to audio track, start timer if not already started this.setInterval(TICK_INTERVAL); @@ -764,8 +766,6 @@ class AudioStreamController const track = this.switchingTrack; if (track) { this.bufferedTrack = track; - this.switchingTrack = null; - this.hls.trigger(Events.AUDIO_TRACK_SWITCHED, { ...track }); } } this.fragBufferedComplete(frag, part); @@ -774,6 +774,22 @@ class AudioStreamController } } + protected checkFragmentChanged( + type: PlaylistLevelType = PlaylistLevelType.AUDIO, + ) { + const previousFrag = this.fragPlaying; + + super.checkFragmentChanged(type); + + const fragPlaying = this.fragPlaying; + const fragPreviousLevel = previousFrag?.level; + if (!fragPlaying || fragPlaying.level !== fragPreviousLevel) { + if (this.switchingTrack) { + this.completeAudioSwitch(this.switchingTrack); + } + } + } + protected onError(event: Events.ERROR, data: ErrorData) { if (data.fatal) { this.state = State.ERROR; @@ -876,8 +892,18 @@ class AudioStreamController } this.state = State.PARSING; - if (this.switchingTrack && audio) { - this.completeAudioSwitch(this.switchingTrack); + if (audio && this.switchingTrack && !this.switchingTrack.flushImmediate) { + const { config } = this; + const bufferFlushDelay = config.nextAudioTrackBufferFlushForwardOffset; + const startOffset = Math.max( + this.getLoadPosition() + bufferFlushDelay, + this.fragPrevious?.start || 0, + ); + super.flushMainBuffer( + startOffset, + Number.POSITIVE_INFINITY, + PlaylistLevelType.AUDIO, + ); } if (initSegment?.tracks) { @@ -1029,7 +1055,7 @@ class AudioStreamController } private flushAudioIfNeeded(switchingTrack: AudioTrackSwitchingData) { - if (this.media && this.bufferedTrack && switchingTrack.flushImmediate) { + if (this.media && this.bufferedTrack) { const { name, lang, assocLang, characteristics, audioCodec, channels } = this.bufferedTrack; if ( @@ -1039,10 +1065,7 @@ class AudioStreamController audioMatchPredicate, ) ) { - if ( - (useAlternateAudio(switchingTrack.url, this.hls), - switchingTrack.flushImmediate) - ) { + if (useAlternateAudio(switchingTrack.url, this.hls)) { this.log('Switching audio track : flushing all audio'); super.flushMainBuffer(0, Number.POSITIVE_INFINITY, 'audio'); this.bufferedTrack = null; @@ -1056,26 +1079,9 @@ class AudioStreamController private completeAudioSwitch(switchingTrack: AudioTrackSwitchingData) { const { hls } = this; - const { flushImmediate } = switchingTrack; - if (!flushImmediate) { - const { config } = this; - const bufferFlushDelay = config.nextAudioTrackBufferFlushForwardOffset; - const startOffset = Math.max( - this.getLoadPosition() + bufferFlushDelay, - this.fragPrevious?.start || 0, - ); - super.flushMainBuffer( - startOffset, - Number.POSITIVE_INFINITY, - PlaylistLevelType.AUDIO, - ); - } - this.bufferedTrack = switchingTrack; - if (flushImmediate) { - this.switchingTrack = null; - hls.trigger(Events.AUDIO_TRACK_SWITCHED, switchingTrack); - } + this.switchingTrack = null; + hls.trigger(Events.AUDIO_TRACK_SWITCHED, switchingTrack); } /** From ce81cc2dad32f5a2d4d710bf5165a78304cb7614 Mon Sep 17 00:00:00 2001 From: Kyle Seager Date: Tue, 11 Nov 2025 09:40:37 -0700 Subject: [PATCH 08/11] Addressed PR comments by cleaning up changes --- api-extractor/report/hls.js.api.md | 32 ++++++---- src/controller/audio-stream-controller.ts | 5 ++ src/controller/base-stream-controller.ts | 76 +++++++++++------------ src/controller/stream-controller.ts | 60 ++++++++++++++++-- 4 files changed, 119 insertions(+), 54 deletions(-) diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md index 7a8d2c248e0..9e1e3c589e1 100644 --- a/api-extractor/report/hls.js.api.md +++ b/api-extractor/report/hls.js.api.md @@ -185,8 +185,12 @@ export type AudioSelectionOption = { export class AudioStreamController extends BaseStreamController implements NetworkComponentAPI { constructor(hls: Hls, fragmentTracker: FragmentTracker, keyLoader: KeyLoader); // (undocumented) + protected checkFragmentChanged(type?: PlaylistLevelType): void; + // (undocumented) doTick(): void; // (undocumented) + protected getBufferOutput(): Bufferable | null; + // (undocumented) protected getLoadPosition(): number; // (undocumented) protected _handleFragmentLoadComplete(fragLoadedData: FragLoadedData): void; @@ -393,12 +397,9 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP protected afterBufferFlushed(media: Bufferable, bufferType: SourceBufferName, playlistType: PlaylistLevelType): void; // (undocumented) protected alignPlaylists(details: LevelDetails, previousDetails: LevelDetails | undefined, switchDetails: LevelDetails | undefined): number; - // Warning: (ae-forgotten-export) The symbol "AlternateAudio" needs to be exported by the entry point hls.d.ts - // - // (undocumented) - protected altAudio: AlternateAudio; - // (undocumented) - protected backtrackFragment: Fragment | null; + protected get backtrackFragment(): Fragment | null; + // Warning: (ae-setter-with-docs) The doc comment for the property "backtrackFragment" must appear on the getter, not the setter. + protected set backtrackFragment(_value: Fragment | null); // (undocumented) protected bitrateTest: boolean; // (undocumented) @@ -422,8 +423,9 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP protected clearTrackerIfNeeded(frag: Fragment): void; // (undocumented) protected config: HlsConfig; - // (undocumented) - protected couldBacktrack: boolean; + protected get couldBacktrack(): boolean; + // Warning: (ae-setter-with-docs) The doc comment for the property "couldBacktrack" must appear on the getter, not the setter. + protected set couldBacktrack(_value: boolean); // (undocumented) protected decrypter: Decrypter; // (undocumented) @@ -455,6 +457,8 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP protected getAppendedFrag(position: number, playlistType?: PlaylistLevelType): Fragment | null; protected getBufferedFrag(position: number, type?: PlaylistLevelType): Fragment | null; // (undocumented) + protected getBufferOutput(): Bufferable | null; + // (undocumented) protected getCurrentContext(chunkMeta: ChunkMetadata): { frag: MediaFragment; part: Part | null; @@ -473,8 +477,6 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP // (undocumented) protected getLoadPosition(): number; // (undocumented) - getMainFwdBufferInfo(): BufferInfo | null; - // (undocumented) protected getMaxBufferLength(levelBitrate?: number): number; // (undocumented) protected getNextFragment(pos: number, levelDetails: LevelDetails): Fragment | null; @@ -520,7 +522,6 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP nextLevelSwitch(type: PlaylistLevelType): void; // (undocumented) protected nextLoadPosition: number; - nextTrackSwitch(): void; // (undocumented) protected onError(event: Events.ERROR, data: ErrorData): void; // (undocumented) @@ -4578,6 +4579,12 @@ export class StreamController extends BaseStreamController implements NetworkCom constructor(hls: Hls, fragmentTracker: FragmentTracker, keyLoader: KeyLoader); // (undocumented) protected abortCurrentFrag(): void; + protected get backtrackFragment(): Fragment | null; + // Warning: (ae-setter-with-docs) The doc comment for the property "backtrackFragment" must appear on the getter, not the setter. + protected set backtrackFragment(value: Fragment | null); + protected get couldBacktrack(): boolean; + // Warning: (ae-setter-with-docs) The doc comment for the property "couldBacktrack" must appear on the getter, not the setter. + protected set couldBacktrack(value: boolean); // (undocumented) get currentFrag(): Fragment | null; // (undocumented) @@ -4590,6 +4597,9 @@ export class StreamController extends BaseStreamController implements NetworkCom protected flushMainBuffer(startOffset: number, endOffset: number): void; // (undocumented) get forceStartLoad(): boolean; + protected getBufferOutput(): Bufferable | null; + // (undocumented) + getMainFwdBufferInfo(): BufferInfo | null; // (undocumented) protected _handleFragmentLoadProgress(data: FragLoadedData): void; // (undocumented) diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts index c174d013e07..eb8095b3f93 100644 --- a/src/controller/audio-stream-controller.ts +++ b/src/controller/audio-stream-controller.ts @@ -19,6 +19,7 @@ import { useAlternateAudio, } from '../utils/rendition-helper'; import type { FragmentTracker } from './fragment-tracker'; +import type { Bufferable } from '../hls'; import type Hls from '../hls'; import type { Fragment, MediaFragment, Part } from '../loader/fragment'; import type KeyLoader from '../loader/key-loader'; @@ -774,6 +775,10 @@ class AudioStreamController } } + protected getBufferOutput(): Bufferable | null { + return this.mediaBuffer ? this.mediaBuffer : this.media; + } + protected checkFragmentChanged( type: PlaylistLevelType = PlaylistLevelType.AUDIO, ) { diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts index 79393aba1ea..7d1e99b0221 100644 --- a/src/controller/base-stream-controller.ts +++ b/src/controller/base-stream-controller.ts @@ -93,12 +93,6 @@ export type InFlightData = { state: (typeof State)[keyof typeof State]; }; -export const enum AlternateAudio { - DISABLED = 0, - SWITCHING, - SWITCHED, -} - export default class BaseStreamController extends TaskLoop implements NetworkComponentAPI @@ -108,10 +102,7 @@ export default class BaseStreamController protected fragPrevious: MediaFragment | null = null; protected fragCurrent: Fragment | null = null; protected fragPlaying: Fragment | null = null; - protected backtrackFragment: Fragment | null = null; - protected altAudio: AlternateAudio = AlternateAudio.DISABLED; protected fragmentTracker: FragmentTracker; - protected couldBacktrack: boolean = false; protected transmuxer: TransmuxerInterface | null = null; protected _state: (typeof State)[keyof typeof State] = State.STOPPED; protected playlistType: PlaylistLevelType; @@ -209,6 +200,34 @@ export default class BaseStreamController return this.buffering; } + /** + * Get backtrack fragment. Returns null in base class. + * Override in stream-controller to return actual backtrack fragment. + */ + protected get backtrackFragment(): Fragment | null { + return null; + } + + /** + * Set backtrack fragment. No-op in base class. + * Override in stream-controller to set actual backtrack fragment. + */ + protected set backtrackFragment(_value: Fragment | null) {} + + /** + * Get could backtrack flag. Returns false in base class. + * Override in stream-controller to return actual value. + */ + protected get couldBacktrack(): boolean { + return false; + } + + /** + * Set could backtrack flag. No-op in base class. + * Override in stream-controller to set actual value. + */ + protected set couldBacktrack(_value: boolean) {} + public pauseBuffering() { this.buffering = false; } @@ -2390,15 +2409,6 @@ export default class BaseStreamController } } - /** - * Base method for track switching that uses common logic - * to prevent buffering interruptions - */ - public nextTrackSwitch(): void { - // Base implementation - to be overridden by subclasses - // This provides the common pattern that both audio and video controllers can use - } - /** * Gets buffered fragment at the specified position */ @@ -2479,14 +2489,12 @@ export default class BaseStreamController fragPlayingCurrent.sn !== fragPlaying.sn || fragPlaying.level !== fragCurrentLevel ) { - if (type === PlaylistLevelType.AUDIO) { - if (fragPlaying?.level !== fragPlayingCurrent.level) { - this.flushMainBuffer( - 0, - this.getLoadPosition(), - PlaylistLevelType.AUDIO, - ); - } + if ( + type === PlaylistLevelType.AUDIO && + fragPlaying && + fragPlaying.level !== fragPlayingCurrent.level + ) { + this.flushMainBuffer(0, fragPlaying.end, PlaylistLevelType.AUDIO); } this.fragPlaying = fragPlayingCurrent; if (type === PlaylistLevelType.MAIN) { @@ -2505,25 +2513,15 @@ export default class BaseStreamController } } - public getMainFwdBufferInfo(): BufferInfo | null { - // Observe video SourceBuffer (this.mediaBuffer) only when alt-audio is used, otherwise observe combined media buffer - const bufferOutput = - this.mediaBuffer && this.altAudio === AlternateAudio.SWITCHED - ? this.mediaBuffer - : this.media; - return this.getFwdBufferInfo(bufferOutput, PlaylistLevelType.MAIN); + protected getBufferOutput(): Bufferable | null { + return null; } public nextLevelSwitch(type: PlaylistLevelType) { const { levels, media, hls, config } = this; // ensure that media is defined and that metadata are available (to retrieve currentTime) if (media?.readyState && levels && hls && config) { - const bufferOutput = - this.mediaBuffer && - (this.altAudio === AlternateAudio.SWITCHED || - type === PlaylistLevelType.AUDIO) - ? this.mediaBuffer - : this.media; + const bufferOutput = this.getBufferOutput(); const bufferInfo = this.getFwdBufferInfo(bufferOutput, type); if (!bufferInfo) { return; diff --git a/src/controller/stream-controller.ts b/src/controller/stream-controller.ts index 76ed9b209cd..9d1c647fbbe 100644 --- a/src/controller/stream-controller.ts +++ b/src/controller/stream-controller.ts @@ -1,7 +1,4 @@ -import BaseStreamController, { - AlternateAudio, - State, -} from './base-stream-controller'; +import BaseStreamController, { State } from './base-stream-controller'; import { findFragmentByPTS } from './fragment-finders'; import { FragmentState } from './fragment-tracker'; import { MAX_START_GAP_JUMP } from './gap-controller'; @@ -52,6 +49,13 @@ import type { import type { Level } from '../types/level'; import type { Track, TrackSet } from '../types/track'; import type { TransmuxerResult } from '../types/transmuxer'; +import type { Bufferable, BufferInfo } from '../utils/buffer-helper'; + +export const enum AlternateAudio { + DISABLED = 0, + SWITCHING, + SWITCHED, +} const TICK_INTERVAL = 100; // how often to tick in ms @@ -63,7 +67,10 @@ export default class StreamController private level: number = -1; private _forceStartLoad: boolean = false; private _hasEnoughToStart: boolean = false; + private altAudio: AlternateAudio = AlternateAudio.DISABLED; private audioOnly: boolean = false; + private _couldBacktrack: boolean = false; + private _backtrackFragment: Fragment | null = null; private audioCodecSwitch: boolean = false; private videoBuffer: ExtendedSourceBuffer | null = null; @@ -405,6 +412,45 @@ export default class StreamController this.flushMainBuffer(0, Number.POSITIVE_INFINITY); } + /** + * Get the buffer output to use for buffer calculations. + * Override to use altAudio logic in stream-controller. + */ + protected getBufferOutput(): Bufferable | null { + if (this.mediaBuffer && this.altAudio === AlternateAudio.SWITCHED) { + return this.mediaBuffer; + } + return this.media; + } + + /** + * Get backtrack fragment. Override to return actual backtrack fragment. + */ + protected get backtrackFragment(): Fragment | null { + return this._backtrackFragment; + } + + /** + * Set backtrack fragment. Override to set actual backtrack fragment. + */ + protected set backtrackFragment(value: Fragment | null) { + this._backtrackFragment = value; + } + + /** + * Get could backtrack flag. Override to return actual value. + */ + protected get couldBacktrack(): boolean { + return this._couldBacktrack; + } + + /** + * Set could backtrack flag. Override to set actual value. + */ + protected set couldBacktrack(value: boolean) { + this._couldBacktrack = value; + } + /** * try to switch ASAP without breaking video playback: * in order to ensure smooth but quick level switching, @@ -1451,6 +1497,12 @@ export default class StreamController this.tickImmediate(); } + public getMainFwdBufferInfo(): BufferInfo | null { + // Observe video SourceBuffer (this.mediaBuffer) only when alt-audio is used, otherwise observe combined media buffer + const bufferOutput = this.getBufferOutput(); + return this.getFwdBufferInfo(bufferOutput, PlaylistLevelType.MAIN); + } + public get maxBufferLength(): number { const { levels, level } = this; const levelInfo = levels?.[level]; From 4389ed53a0e1ad8d7293d55e4f5866dea8cb9e26 Mon Sep 17 00:00:00 2001 From: Kyle Seager Date: Tue, 11 Nov 2025 18:34:39 -0700 Subject: [PATCH 09/11] Addressed PR comments and cleaned up changes --- api-extractor/report/hls.js.api.md | 17 +++++---- src/controller/audio-stream-controller.ts | 30 ++++++++-------- src/controller/base-stream-controller.ts | 42 ++++++++--------------- src/controller/stream-controller.ts | 38 ++++++++++++++++---- src/hls.ts | 9 ++--- 5 files changed, 72 insertions(+), 64 deletions(-) diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md index 9e1e3c589e1..5959dd2659d 100644 --- a/api-extractor/report/hls.js.api.md +++ b/api-extractor/report/hls.js.api.md @@ -185,7 +185,7 @@ export type AudioSelectionOption = { export class AudioStreamController extends BaseStreamController implements NetworkComponentAPI { constructor(hls: Hls, fragmentTracker: FragmentTracker, keyLoader: KeyLoader); // (undocumented) - protected checkFragmentChanged(type?: PlaylistLevelType): void; + protected checkFragmentChanged(type?: PlaylistLevelType): boolean; // (undocumented) doTick(): void; // (undocumented) @@ -199,9 +199,6 @@ export class AudioStreamController extends BaseStreamController implements Netwo // (undocumented) protected loadFragment(frag: Fragment, track: Level, targetBufferTime: number): void; get nextAudioTrack(): number; - // Warning: (ae-setter-with-docs) The doc comment for the property "nextAudioTrack" must appear on the getter, not the setter. - set nextAudioTrack(audioTrackId: number); - nextAudioTrackSwitch(): void; // (undocumented) protected onError(event: Events.ERROR, data: ErrorData): void; // (undocumented) @@ -397,9 +394,9 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP protected afterBufferFlushed(media: Bufferable, bufferType: SourceBufferName, playlistType: PlaylistLevelType): void; // (undocumented) protected alignPlaylists(details: LevelDetails, previousDetails: LevelDetails | undefined, switchDetails: LevelDetails | undefined): number; - protected get backtrackFragment(): Fragment | null; + protected get backtrackFragment(): Fragment | undefined; // Warning: (ae-setter-with-docs) The doc comment for the property "backtrackFragment" must appear on the getter, not the setter. - protected set backtrackFragment(_value: Fragment | null); + protected set backtrackFragment(_value: Fragment | undefined); // (undocumented) protected bitrateTest: boolean; // (undocumented) @@ -413,7 +410,7 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP okToFlushForwardBuffer: boolean; }; // (undocumented) - protected checkFragmentChanged(type?: PlaylistLevelType): void; + protected checkFragmentChanged(type?: PlaylistLevelType): boolean; // (undocumented) protected checkLiveUpdate(details: LevelDetails): void; // (undocumented) @@ -4579,9 +4576,11 @@ export class StreamController extends BaseStreamController implements NetworkCom constructor(hls: Hls, fragmentTracker: FragmentTracker, keyLoader: KeyLoader); // (undocumented) protected abortCurrentFrag(): void; - protected get backtrackFragment(): Fragment | null; + protected get backtrackFragment(): Fragment | undefined; // Warning: (ae-setter-with-docs) The doc comment for the property "backtrackFragment" must appear on the getter, not the setter. - protected set backtrackFragment(value: Fragment | null); + protected set backtrackFragment(value: Fragment | undefined); + // (undocumented) + protected checkFragmentChanged(type?: PlaylistLevelType): boolean; protected get couldBacktrack(): boolean; // Warning: (ae-setter-with-docs) The doc comment for the property "couldBacktrack" must appear on the getter, not the setter. protected set couldBacktrack(value: boolean); diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts index eb8095b3f93..082a8286358 100644 --- a/src/controller/audio-stream-controller.ts +++ b/src/controller/audio-stream-controller.ts @@ -495,9 +495,11 @@ class AudioStreamController if (altAudio) { this.switchingTrack = data; // main audio track are handled by stream-controller, just do something if switching to alt audio track - if (data.flushImmediate) { - this.flushAudioIfNeeded(data); + if (!data.flushImmediate) { + this.nextTrackId = data.id; + this.nextAudioTrackSwitch(); } + this.flushAudioIfNeeded(data); if (this.state !== State.STOPPED) { // switching to audio track, start timer if not already started this.setInterval(TICK_INTERVAL); @@ -783,16 +785,20 @@ class AudioStreamController type: PlaylistLevelType = PlaylistLevelType.AUDIO, ) { const previousFrag = this.fragPlaying; - - super.checkFragmentChanged(type); + const fragChanged = super.checkFragmentChanged(type); + if (!fragChanged) { + return false; + } const fragPlaying = this.fragPlaying; const fragPreviousLevel = previousFrag?.level; if (!fragPlaying || fragPlaying.level !== fragPreviousLevel) { + this.cleanupBackBuffer(PlaylistLevelType.AUDIO); if (this.switchingTrack) { this.completeAudioSwitch(this.switchingTrack); } } + return true; } protected onError(event: Events.ERROR, data: ErrorData) { @@ -902,7 +908,7 @@ class AudioStreamController const bufferFlushDelay = config.nextAudioTrackBufferFlushForwardOffset; const startOffset = Math.max( this.getLoadPosition() + bufferFlushDelay, - this.fragPrevious?.start || 0, + frag.start, ); super.flushMainBuffer( startOffset, @@ -1060,7 +1066,7 @@ class AudioStreamController } private flushAudioIfNeeded(switchingTrack: AudioTrackSwitchingData) { - if (this.media && this.bufferedTrack) { + if (this.media && this.bufferedTrack && switchingTrack.flushImmediate) { const { name, lang, assocLang, characteristics, audioCodec, channels } = this.bufferedTrack; if ( @@ -1086,7 +1092,7 @@ class AudioStreamController const { hls } = this; this.bufferedTrack = switchingTrack; this.switchingTrack = null; - hls.trigger(Events.AUDIO_TRACK_SWITCHED, switchingTrack); + hls.trigger(Events.AUDIO_TRACK_SWITCHED, { ...switchingTrack }); } /** @@ -1096,21 +1102,13 @@ class AudioStreamController return this.nextTrackId; } - /** - * Set next audio track index for seamless audio track switching. - * This schedules audio track switching without interrupting playback. - */ - set nextAudioTrack(audioTrackId: number) { - this.nextTrackId = audioTrackId; - } - /** * try to switch ASAP without breaking audio playback: * in order to ensure smooth but quick audio track switching, * we need to find the next flushable buffer range * we should take into account new segment fetch time */ - public nextAudioTrackSwitch(): void { + private nextAudioTrackSwitch(): void { super.nextLevelSwitch(PlaylistLevelType.AUDIO); } } diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts index 7d1e99b0221..f74c051a2be 100644 --- a/src/controller/base-stream-controller.ts +++ b/src/controller/base-stream-controller.ts @@ -204,15 +204,15 @@ export default class BaseStreamController * Get backtrack fragment. Returns null in base class. * Override in stream-controller to return actual backtrack fragment. */ - protected get backtrackFragment(): Fragment | null { - return null; + protected get backtrackFragment(): Fragment | undefined { + return undefined; } /** * Set backtrack fragment. No-op in base class. * Override in stream-controller to set actual backtrack fragment. */ - protected set backtrackFragment(_value: Fragment | null) {} + protected set backtrackFragment(_value: Fragment | undefined) {} /** * Get could backtrack flag. Returns false in base class. @@ -366,10 +366,13 @@ export default class BaseStreamController protected onManifestLoading() { this.initPTS = []; - this.levels = this.levelLastLoaded = this.fragCurrent = null; + this.fragPlaying = + this.levels = + this.levelLastLoaded = + this.fragCurrent = + null; this.lastCurrentTime = this.startPosition = 0; this.startFragRequested = false; - this.fragPlaying = this.backtrackFragment = null; } protected onError(event: Events.ERROR, data: ErrorData) {} @@ -2399,12 +2402,13 @@ export default class BaseStreamController type, ); if (fragPlayingCurrent && fragPlayingCurrent.start > 1) { + const isAudio = type === PlaylistLevelType.AUDIO; // flush buffer preceding current fragment (flush until current fragment start offset) // minus 1s to avoid video freezing, that could happen if we flush keyframe of current video ... this.flushMainBuffer( 0, - fragPlayingCurrent.start - 1, - type === PlaylistLevelType.AUDIO ? 'audio' : null, + fragPlayingCurrent.start - (isAudio ? 0 : 1), + isAudio ? 'audio' : null, ); } } @@ -2458,7 +2462,7 @@ export default class BaseStreamController protected checkFragmentChanged( type: PlaylistLevelType = PlaylistLevelType.MAIN, - ) { + ): boolean { const video = this.media; let fragPlayingCurrent: Fragment | null = null; if (video && video.readyState > 1 && video.seeking === false) { @@ -2481,7 +2485,7 @@ export default class BaseStreamController fragPlayingCurrent = this.getAppendedFrag(currentTime + 0.1, type); } if (fragPlayingCurrent) { - this.backtrackFragment = null; + this.backtrackFragment = undefined; const fragPlaying = this.fragPlaying; const fragCurrentLevel = fragPlayingCurrent.level; if ( @@ -2489,28 +2493,12 @@ export default class BaseStreamController fragPlayingCurrent.sn !== fragPlaying.sn || fragPlaying.level !== fragCurrentLevel ) { - if ( - type === PlaylistLevelType.AUDIO && - fragPlaying && - fragPlaying.level !== fragPlayingCurrent.level - ) { - this.flushMainBuffer(0, fragPlaying.end, PlaylistLevelType.AUDIO); - } this.fragPlaying = fragPlayingCurrent; - if (type === PlaylistLevelType.MAIN) { - this.hls.trigger(Events.FRAG_CHANGED, { frag: fragPlayingCurrent }); - if ( - !fragPlaying || - fragPlaying.level !== (fragCurrentLevel as number | undefined) - ) { - this.hls.trigger(Events.LEVEL_SWITCHED, { - level: fragCurrentLevel, - }); - } - } + return true; } } } + return false; } protected getBufferOutput(): Bufferable | null { diff --git a/src/controller/stream-controller.ts b/src/controller/stream-controller.ts index 9d1c647fbbe..8defcedd19c 100644 --- a/src/controller/stream-controller.ts +++ b/src/controller/stream-controller.ts @@ -70,7 +70,7 @@ export default class StreamController private altAudio: AlternateAudio = AlternateAudio.DISABLED; private audioOnly: boolean = false; private _couldBacktrack: boolean = false; - private _backtrackFragment: Fragment | null = null; + private _backtrackFragment: Fragment | undefined = undefined; private audioCodecSwitch: boolean = false; private videoBuffer: ExtendedSourceBuffer | null = null; @@ -316,7 +316,7 @@ export default class StreamController this.backtrackFragment && this.backtrackFragment.start > bufferInfo.end ) { - this.backtrackFragment = null; + this.backtrackFragment = undefined; } const targetBufferTime = this.backtrackFragment ? this.backtrackFragment.start @@ -338,7 +338,7 @@ export default class StreamController this.fragmentTracker.removeFragment(backtrackFrag); } } else if (this.backtrackFragment && bufferInfo.len) { - this.backtrackFragment = null; + this.backtrackFragment = undefined; } // Avoid loop loading by using nextLoadPosition set for backtracking and skipping consecutive GAP tags if (frag && this.isLoopLoading(frag, targetBufferTime)) { @@ -423,17 +423,42 @@ export default class StreamController return this.media; } + protected checkFragmentChanged( + type: PlaylistLevelType = PlaylistLevelType.AUDIO, + ): boolean { + const previousFrag = this.fragPlaying; + const fragChanged = super.checkFragmentChanged(type); + if (!fragChanged) { + return false; + } + + const fragPlaying = this.fragPlaying; + if (fragPlaying && previousFrag) { + const fragCurrentLevel = fragPlaying.level; + this.hls.trigger(Events.FRAG_CHANGED, { frag: fragPlaying }); + if ( + !fragPlaying || + fragPlaying.level !== (fragCurrentLevel as number | undefined) + ) { + this.hls.trigger(Events.LEVEL_SWITCHED, { + level: fragCurrentLevel, + }); + } + } + return true; + } + /** * Get backtrack fragment. Override to return actual backtrack fragment. */ - protected get backtrackFragment(): Fragment | null { + protected get backtrackFragment(): Fragment | undefined { return this._backtrackFragment; } /** * Set backtrack fragment. Override to set actual backtrack fragment. */ - protected set backtrackFragment(value: Fragment | null) { + protected set backtrackFragment(value: Fragment | undefined) { this._backtrackFragment = value; } @@ -462,8 +487,8 @@ export default class StreamController } protected abortCurrentFrag(): void { + this.backtrackFragment = undefined; super.abortCurrentFrag(); - this.backtrackFragment = null; } protected flushMainBuffer(startOffset: number, endOffset: number) { @@ -545,6 +570,7 @@ export default class StreamController this.log('Trigger BUFFER_RESET'); this.hls.trigger(Events.BUFFER_RESET, undefined); this.couldBacktrack = false; + this.backtrackFragment = undefined; this.altAudio = AlternateAudio.DISABLED; this.audioOnly = false; } diff --git a/src/hls.ts b/src/hls.ts index 1cc0fa06fde..8bbec17580c 100644 --- a/src/hls.ts +++ b/src/hls.ts @@ -1084,12 +1084,9 @@ export default class Hls implements HlsEventEmitter { * @param audioTrackId - Pass -1 for automatic level selection */ set nextAudioTrack(audioTrackId: number) { - this.logger.log(`set nextAudioTrack:${audioTrackId}`); - const { audioTrackController, audioStreamController } = this; - if (audioTrackController && audioStreamController) { - audioTrackController.nextAudioTrack = - audioStreamController.nextAudioTrack = audioTrackId; - audioStreamController.nextAudioTrackSwitch(); + const { audioTrackController } = this; + if (audioTrackController) { + audioTrackController.nextAudioTrack = audioTrackId; } } From afe606297103263f7f4f2f155bf3c7fdf878e7d0 Mon Sep 17 00:00:00 2001 From: Kyle Seager Date: Tue, 11 Nov 2025 19:16:59 -0700 Subject: [PATCH 10/11] Cleaned up usage of playlistType to use member variable rather than passing as a parameter --- api-extractor/report/hls.js.api.md | 24 +++---- src/controller/audio-stream-controller.ts | 30 ++------ src/controller/base-stream-controller.ts | 83 ++++++++++------------- src/controller/stream-controller.ts | 20 ++---- 4 files changed, 56 insertions(+), 101 deletions(-) diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md index 5959dd2659d..2460027289e 100644 --- a/api-extractor/report/hls.js.api.md +++ b/api-extractor/report/hls.js.api.md @@ -185,7 +185,7 @@ export type AudioSelectionOption = { export class AudioStreamController extends BaseStreamController implements NetworkComponentAPI { constructor(hls: Hls, fragmentTracker: FragmentTracker, keyLoader: KeyLoader); // (undocumented) - protected checkFragmentChanged(type?: PlaylistLevelType): boolean; + protected checkFragmentChanged(): boolean; // (undocumented) doTick(): void; // (undocumented) @@ -391,7 +391,7 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP constructor(hls: Hls, fragmentTracker: FragmentTracker, keyLoader: KeyLoader, logPrefix: string, playlistType: PlaylistLevelType); protected abortCurrentFrag(): void; // (undocumented) - protected afterBufferFlushed(media: Bufferable, bufferType: SourceBufferName, playlistType: PlaylistLevelType): void; + protected afterBufferFlushed(media: Bufferable, bufferType: SourceBufferName): void; // (undocumented) protected alignPlaylists(details: LevelDetails, previousDetails: LevelDetails | undefined, switchDetails: LevelDetails | undefined): number; protected get backtrackFragment(): Fragment | undefined; @@ -405,17 +405,17 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP protected buffering: boolean; // (undocumented) get bufferingEnabled(): boolean; - protected calculateOptimalSwitchPoint(nextLevel: Level, bufferInfo: BufferInfo, type: PlaylistLevelType): { + protected calculateOptimalSwitchPoint(nextLevel: Level, bufferInfo: BufferInfo): { fetchdelay: number; okToFlushForwardBuffer: boolean; }; // (undocumented) - protected checkFragmentChanged(type?: PlaylistLevelType): boolean; + protected checkFragmentChanged(): boolean; // (undocumented) protected checkLiveUpdate(details: LevelDetails): void; // (undocumented) protected checkRetryDate(): void; - protected cleanupBackBuffer(type: PlaylistLevelType): void; + protected cleanupBackBuffer(): void; // (undocumented) protected clearTrackerIfNeeded(frag: Fragment): void; // (undocumented) @@ -435,7 +435,7 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP protected flushBufferGap(frag: Fragment): void; // (undocumented) protected flushMainBuffer(startOffset: number, endOffset: number, type?: SourceBufferName | null): void; - protected followingBufferedFrag(frag: Fragment | null, type?: PlaylistLevelType): Fragment | null; + protected followingBufferedFrag(frag: Fragment | null): Fragment | null; // (undocumented) protected fragBufferedComplete(frag: Fragment, part: Part | null): void; // (undocumented) @@ -451,8 +451,8 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP // (undocumented) protected fragPrevious: MediaFragment | null; // (undocumented) - protected getAppendedFrag(position: number, playlistType?: PlaylistLevelType): Fragment | null; - protected getBufferedFrag(position: number, type?: PlaylistLevelType): Fragment | null; + protected getAppendedFrag(position: number): Fragment | null; + protected getBufferedFrag(position: number): Fragment | null; // (undocumented) protected getBufferOutput(): Bufferable | null; // (undocumented) @@ -515,8 +515,7 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP protected media: HTMLMediaElement | null; // (undocumented) protected mediaBuffer: Bufferable | null; - // (undocumented) - nextLevelSwitch(type: PlaylistLevelType): void; + nextLevelSwitch(): void; // (undocumented) protected nextLoadPosition: number; // (undocumented) @@ -571,7 +570,7 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP resumeBuffering(): void; // (undocumented) protected retryDate: number; - protected scheduleTrackSwitch(bufferInfo: BufferInfo, fetchdelay: number, okToFlushForwardBuffer: boolean, type: PlaylistLevelType): void; + protected scheduleTrackSwitch(bufferInfo: BufferInfo, fetchdelay: number, okToFlushForwardBuffer: boolean): void; // (undocumented) protected setStartPosition(details: LevelDetails, sliding: number): void; // (undocumented) @@ -4580,7 +4579,7 @@ export class StreamController extends BaseStreamController implements NetworkCom // Warning: (ae-setter-with-docs) The doc comment for the property "backtrackFragment" must appear on the getter, not the setter. protected set backtrackFragment(value: Fragment | undefined); // (undocumented) - protected checkFragmentChanged(type?: PlaylistLevelType): boolean; + protected checkFragmentChanged(): boolean; protected get couldBacktrack(): boolean; // Warning: (ae-setter-with-docs) The doc comment for the property "couldBacktrack" must appear on the getter, not the setter. protected set couldBacktrack(value: boolean); @@ -4613,7 +4612,6 @@ export class StreamController extends BaseStreamController implements NetworkCom get nextBufferedFrag(): Fragment | null; // (undocumented) get nextLevel(): number; - nextLevelSwitch(): void; // (undocumented) protected onError(event: Events.ERROR, data: ErrorData): void; // (undocumented) diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts index 082a8286358..30979c6bb00 100644 --- a/src/controller/audio-stream-controller.ts +++ b/src/controller/audio-stream-controller.ts @@ -309,7 +309,7 @@ class AudioStreamController } this.lastCurrentTime = media.currentTime; - this.checkFragmentChanged(PlaylistLevelType.AUDIO); + this.checkFragmentChanged(); } private doTickIdle() { @@ -348,11 +348,7 @@ class AudioStreamController const bufferable = this.mediaBuffer ? this.mediaBuffer : this.media; if (this.bufferFlushed && bufferable) { this.bufferFlushed = false; - this.afterBufferFlushed( - bufferable, - ElementaryStreamTypes.AUDIO, - PlaylistLevelType.AUDIO, - ); + this.afterBufferFlushed(bufferable, ElementaryStreamTypes.AUDIO); } const bufferInfo = this.getFwdBufferInfo( @@ -497,7 +493,7 @@ class AudioStreamController // main audio track are handled by stream-controller, just do something if switching to alt audio track if (!data.flushImmediate) { this.nextTrackId = data.id; - this.nextAudioTrackSwitch(); + this.nextLevelSwitch(); } this.flushAudioIfNeeded(data); if (this.state !== State.STOPPED) { @@ -781,11 +777,9 @@ class AudioStreamController return this.mediaBuffer ? this.mediaBuffer : this.media; } - protected checkFragmentChanged( - type: PlaylistLevelType = PlaylistLevelType.AUDIO, - ) { + protected checkFragmentChanged() { const previousFrag = this.fragPlaying; - const fragChanged = super.checkFragmentChanged(type); + const fragChanged = super.checkFragmentChanged(); if (!fragChanged) { return false; } @@ -793,7 +787,7 @@ class AudioStreamController const fragPlaying = this.fragPlaying; const fragPreviousLevel = previousFrag?.level; if (!fragPlaying || fragPlaying.level !== fragPreviousLevel) { - this.cleanupBackBuffer(PlaylistLevelType.AUDIO); + this.cleanupBackBuffer(); if (this.switchingTrack) { this.completeAudioSwitch(this.switchingTrack); } @@ -875,7 +869,7 @@ class AudioStreamController } const mediaBuffer = this.mediaBuffer || this.media; if (mediaBuffer) { - this.afterBufferFlushed(mediaBuffer, type, PlaylistLevelType.AUDIO); + this.afterBufferFlushed(mediaBuffer, type); this.tick(); } } @@ -1101,15 +1095,5 @@ class AudioStreamController get nextAudioTrack(): number { return this.nextTrackId; } - - /** - * try to switch ASAP without breaking audio playback: - * in order to ensure smooth but quick audio track switching, - * we need to find the next flushable buffer range - * we should take into account new segment fetch time - */ - private nextAudioTrackSwitch(): void { - super.nextLevelSwitch(PlaylistLevelType.AUDIO); - } } export default AudioStreamController; diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts index f74c051a2be..a6b32c53aa5 100644 --- a/src/controller/base-stream-controller.ts +++ b/src/controller/base-stream-controller.ts @@ -1353,12 +1353,9 @@ export default class BaseStreamController return false; } - protected getAppendedFrag( - position: number, - playlistType: PlaylistLevelType = PlaylistLevelType.MAIN, - ): Fragment | null { + protected getAppendedFrag(position: number): Fragment | null { const fragOrPart = (this.fragmentTracker as any) - ? this.fragmentTracker.getAppendedFrag(position, playlistType) + ? this.fragmentTracker.getAppendedFrag(position, this.playlistType) : null; if (fragOrPart && 'fragment' in fragOrPart) { return fragOrPart.fragment; @@ -2071,7 +2068,6 @@ export default class BaseStreamController protected afterBufferFlushed( media: Bufferable, bufferType: SourceBufferName, - playlistType: PlaylistLevelType, ) { if (!media) { return; @@ -2082,7 +2078,7 @@ export default class BaseStreamController this.fragmentTracker.detectEvictedFragments( bufferType, bufferedTimeRanges, - playlistType, + this.playlistType, ); if (this.state === State.ENDED) { this.resetLoadingState(); @@ -2301,14 +2297,13 @@ export default class BaseStreamController protected calculateOptimalSwitchPoint( nextLevel: Level, bufferInfo: BufferInfo, - type: PlaylistLevelType, ): { fetchdelay: number; okToFlushForwardBuffer: boolean } { let fetchdelay = 0; - const { hls, media, config, levels } = this; + const { hls, media, config, levels, playlistType } = this; const levelDetails = this.getLevelDetails(); if (media && !media.paused && levels) { const maxBitrate = - type === PlaylistLevelType.AUDIO + playlistType === PlaylistLevelType.AUDIO ? estimatedAudioBitrate(nextLevel.audioCodec, 128000) : nextLevel.maxBitrate; // add a safety delay of 1s @@ -2344,21 +2339,20 @@ export default class BaseStreamController bufferInfo: BufferInfo, fetchdelay: number, okToFlushForwardBuffer: boolean, - type: PlaylistLevelType, ): void { - const { media } = this; + const { media, playlistType } = this; if (!media || !bufferInfo) { return; } // find buffer range that will be reached once new fragment will be fetched const bufferedFrag = okToFlushForwardBuffer - ? this.getBufferedFrag(this.getLoadPosition() + fetchdelay, type) + ? this.getBufferedFrag(this.getLoadPosition() + fetchdelay) : null; if (bufferedFrag) { // we can flush buffer range following this one without stalling playback - const nextBufferedFrag = this.followingBufferedFrag(bufferedFrag, type); + const nextBufferedFrag = this.followingBufferedFrag(bufferedFrag); if (nextBufferedFrag) { // if we are here, we can also cancel any loading/demuxing in progress, as they are useless this.abortCurrentFrag(); @@ -2378,11 +2372,12 @@ export default class BaseStreamController fragDuration * (this.couldBacktrack ? 0.75 : 0.25), ), ); - const bufferType = type === PlaylistLevelType.MAIN ? null : 'audio'; + const bufferType = + playlistType === PlaylistLevelType.MAIN ? null : 'audio'; // Flush forward buffer from next buffered frag start to infinity this.flushMainBuffer(startPts, Number.POSITIVE_INFINITY, bufferType); // Flush back buffer (excluding current fragment) - this.cleanupBackBuffer(type); + this.cleanupBackBuffer(); } } } @@ -2390,19 +2385,16 @@ export default class BaseStreamController /** * Handle back-buffer cleanup during track switching */ - protected cleanupBackBuffer(type: PlaylistLevelType): void { - const { media } = this; + protected cleanupBackBuffer(): void { + const { media, playlistType } = this; if (!media) { return; } // remove back-buffer - const fragPlayingCurrent = this.getAppendedFrag( - this.getLoadPosition(), - type, - ); + const fragPlayingCurrent = this.getAppendedFrag(this.getLoadPosition()); if (fragPlayingCurrent && fragPlayingCurrent.start > 1) { - const isAudio = type === PlaylistLevelType.AUDIO; + const isAudio = playlistType === PlaylistLevelType.AUDIO; // flush buffer preceding current fragment (flush until current fragment start offset) // minus 1s to avoid video freezing, that could happen if we flush keyframe of current video ... this.flushMainBuffer( @@ -2416,23 +2408,17 @@ export default class BaseStreamController /** * Gets buffered fragment at the specified position */ - protected getBufferedFrag( - position: number, - type: PlaylistLevelType = PlaylistLevelType.MAIN, - ): Fragment | null { - return this.fragmentTracker.getBufferedFrag(position, type); + protected getBufferedFrag(position: number): Fragment | null { + return this.fragmentTracker.getBufferedFrag(position, this.playlistType); } /** * Gets the next buffered fragment following the given fragment */ - protected followingBufferedFrag( - frag: Fragment | null, - type: PlaylistLevelType = PlaylistLevelType.MAIN, - ): Fragment | null { + protected followingBufferedFrag(frag: Fragment | null): Fragment | null { if (frag) { // try to get range of next fragment (500ms after this range) - return this.getBufferedFrag(frag.end + 0.5, type); + return this.getBufferedFrag(frag.end + 0.5); } return null; } @@ -2460,9 +2446,7 @@ export default class BaseStreamController this.nextLoadPosition = this.getLoadPosition(); } - protected checkFragmentChanged( - type: PlaylistLevelType = PlaylistLevelType.MAIN, - ): boolean { + protected checkFragmentChanged(): boolean { const video = this.media; let fragPlayingCurrent: Fragment | null = null; if (video && video.readyState > 1 && video.seeking === false) { @@ -2475,14 +2459,14 @@ export default class BaseStreamController */ if (BufferHelper.isBuffered(video, currentTime)) { - fragPlayingCurrent = this.getAppendedFrag(currentTime, type); + fragPlayingCurrent = this.getAppendedFrag(currentTime); } else if (BufferHelper.isBuffered(video, currentTime + 0.1)) { /* ensure that FRAG_CHANGED event is triggered at startup, when first video frame is displayed and playback is paused. add a tolerance of 100ms, in case current position is not buffered, check if current pos+100ms is buffered and use that buffer range for FRAG_CHANGED event reporting */ - fragPlayingCurrent = this.getAppendedFrag(currentTime + 0.1, type); + fragPlayingCurrent = this.getAppendedFrag(currentTime + 0.1); } if (fragPlayingCurrent) { this.backtrackFragment = undefined; @@ -2505,31 +2489,32 @@ export default class BaseStreamController return null; } - public nextLevelSwitch(type: PlaylistLevelType) { - const { levels, media, hls, config } = this; + /** + * try to switch ASAP without breaking video playback: + * in order to ensure smooth but quick level switching, + * we need to find the next flushable buffer range + * we should take into account new segment fetch time + */ + public nextLevelSwitch() { + const { levels, media, hls, config, playlistType } = this; // ensure that media is defined and that metadata are available (to retrieve currentTime) if (media?.readyState && levels && hls && config) { const bufferOutput = this.getBufferOutput(); - const bufferInfo = this.getFwdBufferInfo(bufferOutput, type); + const bufferInfo = this.getFwdBufferInfo(bufferOutput, playlistType); if (!bufferInfo) { return; } const nextLevelId = - type === PlaylistLevelType.AUDIO + playlistType === PlaylistLevelType.AUDIO ? hls.nextAudioTrack : hls.nextLoadLevel; const nextLevel = levels[nextLevelId]; const { fetchdelay, okToFlushForwardBuffer } = - this.calculateOptimalSwitchPoint(nextLevel, bufferInfo, type); + this.calculateOptimalSwitchPoint(nextLevel, bufferInfo); - this.scheduleTrackSwitch( - bufferInfo, - fetchdelay, - okToFlushForwardBuffer, - type, - ); + this.scheduleTrackSwitch(bufferInfo, fetchdelay, okToFlushForwardBuffer); } this.tickImmediate(); } diff --git a/src/controller/stream-controller.ts b/src/controller/stream-controller.ts index 8defcedd19c..e231e0be320 100644 --- a/src/controller/stream-controller.ts +++ b/src/controller/stream-controller.ts @@ -354,7 +354,7 @@ export default class StreamController ? this.videoBuffer : this.mediaBuffer) || this.media; if (mediaBuffer) { - this.afterBufferFlushed(mediaBuffer, type, PlaylistLevelType.MAIN); + this.afterBufferFlushed(mediaBuffer, type); } } frag = this.getNextFragmentLoopLoading( @@ -423,11 +423,9 @@ export default class StreamController return this.media; } - protected checkFragmentChanged( - type: PlaylistLevelType = PlaylistLevelType.AUDIO, - ): boolean { + protected checkFragmentChanged(): boolean { const previousFrag = this.fragPlaying; - const fragChanged = super.checkFragmentChanged(type); + const fragChanged = super.checkFragmentChanged(); if (!fragChanged) { return false; } @@ -476,16 +474,6 @@ export default class StreamController this._couldBacktrack = value; } - /** - * try to switch ASAP without breaking video playback: - * in order to ensure smooth but quick level switching, - * we need to find the next flushable buffer range - * we should take into account new segment fetch time - */ - public nextLevelSwitch() { - super.nextLevelSwitch(PlaylistLevelType.MAIN); - } - protected abortCurrentFrag(): void { this.backtrackFragment = undefined; super.abortCurrentFrag(); @@ -1058,7 +1046,7 @@ export default class StreamController ? this.videoBuffer : this.mediaBuffer) || this.media; if (mediaBuffer) { - this.afterBufferFlushed(mediaBuffer, type, PlaylistLevelType.MAIN); + this.afterBufferFlushed(mediaBuffer, type); this.tick(); } } From a4f045535def0d23166c3431bb9d954d078c7936 Mon Sep 17 00:00:00 2001 From: Kyle Seager Date: Thu, 13 Nov 2025 00:17:45 -0700 Subject: [PATCH 11/11] Fixed bug where fragment from start was loaded when using nextAudioTrack api --- src/controller/audio-stream-controller.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts index 30979c6bb00..8ea97f8e4b6 100644 --- a/src/controller/audio-stream-controller.ts +++ b/src/controller/audio-stream-controller.ts @@ -179,7 +179,11 @@ class AudioStreamController } protected getLoadPosition(): number { - if (!this.startFragRequested && this.nextLoadPosition >= 0) { + if ( + !this.startFragRequested && + this.nextLoadPosition >= 0 && + this.switchingTrack?.flushImmediate !== false + ) { return this.nextLoadPosition; } return super.getLoadPosition();