diff --git a/_scripts/webpack.web.config.js b/_scripts/webpack.web.config.js index 664bc63e446fc..2d98325def716 100644 --- a/_scripts/webpack.web.config.js +++ b/_scripts/webpack.web.config.js @@ -31,7 +31,8 @@ const config = { filename: '[name].js', }, externals: { - 'youtubei.js': '{}' + 'youtubei.js': '{}', + googlevideo: '{}' }, module: { rules: [ @@ -131,7 +132,7 @@ const config = { 'process.env.SWIPER_VERSION': `'${swiperVersion}'` }), new webpack.ProvidePlugin({ - process: 'process/browser' + process: 'process/browser.js' }), new HtmlWebpackPlugin({ excludeChunks: ['processTaskWorker'], diff --git a/jsconfig.json b/jsconfig.json index dad8a794996bf..14e2302c9a784 100644 --- a/jsconfig.json +++ b/jsconfig.json @@ -5,6 +5,8 @@ "compilerOptions": { "strictNullChecks": true, "baseUrl": "./", + "module": "esnext", + "moduleResolution": "bundler", "paths": { "DB_HANDLERS_ELECTRON_RENDERER_OR_WEB": [ "src/datastores/handlers/electron", diff --git a/package.json b/package.json index bc6f664148930..233ad0ee80de1 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "autolinker": "^4.1.5", "bgutils-js": "^3.2.0", "electron-context-menu": "^4.1.0", + "googlevideo": "^4.0.2", "marked": "^16.1.1", "portal-vue": "^2.1.7", "process": "^0.11.10", diff --git a/src/renderer/components/ft-shaka-video-player/ft-shaka-video-player.js b/src/renderer/components/ft-shaka-video-player/ft-shaka-video-player.js index e552768978edf..f0d16a0fbe666 100644 --- a/src/renderer/components/ft-shaka-video-player/ft-shaka-video-player.js +++ b/src/renderer/components/ft-shaka-video-player/ft-shaka-video-player.js @@ -16,7 +16,6 @@ import { getSponsorBlockSegments, logShakaError, repairInvidiousManifest, - sortCaptions, translateSponsorBlockCategory } from '../../helpers/player/utils' import { @@ -24,6 +23,8 @@ import { showToast, writeFileWithPicker } from '../../helpers/utils' +import { MANIFEST_TYPE_SABR } from '../../helpers/player/SabrManifestParser' +import { setupSabrScheme } from '../../helpers/player/SabrSchemePlugin' /** @typedef {import('../../helpers/sponsorblock').SponsorBlockCategory} SponsorBlockCategory */ @@ -35,6 +36,7 @@ const USE_OVERFLOW_MENU_WIDTH_THRESHOLD = 634 const RequestType = shaka.net.NetworkingEngine.RequestType const AdvancedRequestType = shaka.net.NetworkingEngine.AdvancedRequestType const TrackLabelFormat = shaka.ui.Overlay.TrackLabelFormat +const { Severity: ErrorSeverity, Category: ErrorCategory, Code: ErrorCode } = shaka.util.Error /* Mapping of Shaka localization keys for control labels to FreeTube shortcuts. @@ -71,6 +73,10 @@ export default defineComponent({ type: String, required: true }, + sabrData: { + type: Object, + default: null + }, legacyFormats: { type: Array, default: () => ([]) @@ -188,31 +194,10 @@ export default defineComponent({ let startInFullscreen = props.startInFullscreen let startInPip = props.startInPip - /** - * @type {{ - * url: string, - * label: string, - * language: string, - * mimeType: string, - * isAutotranslated?: boolean - * }[]} - */ - let sortedCaptions - - // we don't need to sort if we only have one caption or don't have any - if (props.captions.length > 1) { - // theoretically we would resort when the language changes, but we can't remove captions that we already added to the player - sortedCaptions = sortCaptions(props.captions) - } else if (props.captions.length === 1) { - sortedCaptions = props.captions - } else { - sortedCaptions = [] - } - /** @type {number|null} */ let restoreCaptionIndex = null - if (store.getters.getEnableSubtitlesByDefault && sortedCaptions.length > 0) { + if (store.getters.getEnableSubtitlesByDefault && props.captions.length > 0) { restoreCaptionIndex = 0 } @@ -223,10 +208,6 @@ export default defineComponent({ height: 0, frameRate: 0 }, - playerDimensions: { - width: 0, - height: 0 - }, bitrate: '0', volume: '100', bandwidth: '0', @@ -243,6 +224,11 @@ export default defineComponent({ } }) + const playerDimensions = computed(() => ({ + width: playerWidth.value, + height: playerHeight.value + })) + // #region settings /** @type {import('vue').ComputedRef} */ @@ -797,7 +783,7 @@ export default defineComponent({ trackLabelFormat: hasMultipleAudioTracks.value ? TrackLabelFormat.LABEL : TrackLabelFormat.LANGUAGE, // Only set it to label if we added the captions ourselves, // some live streams come with subtitles in the DASH manifest, but without labels - textTrackLabelFormat: sortedCaptions.length > 0 ? TrackLabelFormat.LABEL : TrackLabelFormat.LANGUAGE, + textTrackLabelFormat: props.captions.length > 0 ? TrackLabelFormat.LABEL : TrackLabelFormat.LANGUAGE, displayInVrMode: useVrMode.value } @@ -1048,7 +1034,7 @@ export default defineComponent({ }) /** @type {ResizeObserver|null} */ - let resizeObserver = null + let containerResizeObserver = null /** @type {ResizeObserverCallback} */ function resized(entries) { @@ -1204,8 +1190,68 @@ export default defineComponent({ } } + const videoElementWidth = ref(0) + const videoElementHeight = ref(0) + + /** @type {ResizeObserver} */ + const videoResizeObserver = new ResizeObserver(() => { + if (video.value) { + const devicePixelRatio = window.devicePixelRatio > 1 ? window.devicePixelRatio : 1 + const video_ = video.value + + videoElementWidth.value = video_.clientWidth * devicePixelRatio + videoElementHeight.value = video_.clientHeight * devicePixelRatio + } + }) + + /** @type {PictureInPictureWindow | null} */ + let pipWindow = null + const pipWindowWidth = ref(null) + const pipWindowHeight = ref(null) + + /** + * @param {PictureInPictureEvent} event + */ + function handleEnterPictureInPicture(event) { + pipWindow = event.pictureInPictureWindow + handlePictureInPictureResize() + pipWindow.addEventListener('resize', handlePictureInPictureResize) + } + + function handleLeavePictureInPicture() { + pipWindow.removeEventListener('resize', handlePictureInPictureResize) + + pipWindow = null + pipWindowWidth.value = null + pipWindowHeight.value = null + } + + function handlePictureInPictureResize() { + const devicePixelRatio = window.devicePixelRatio > 1 ? window.devicePixelRatio : 1 + + pipWindowWidth.value = pipWindow.width * devicePixelRatio + pipWindowHeight.value = pipWindow.height * devicePixelRatio + } + + const playerWidth = computed(() => Math.round(pipWindowWidth.value ?? videoElementWidth.value)) + const playerHeight = computed(() => Math.round(pipWindowHeight.value ?? videoElementHeight.value)) + // #endregion video event handlers + // #region SABR + + /** @type {shaka.extern.Manifest | undefined} */ + let sabrManifest + + /** @type {() => void | undefined} */ + let cleanupSabrScheme + + if (process.env.SUPPORTS_LOCAL_API && props.sabrData) { + cleanupSabrScheme = /** @__NOINLINE__ */ setupSabrScheme(props.sabrData, () => player, () => sabrManifest, playerWidth, playerHeight) + } + + // #endregion SABR + // #region request/response filters /** @type {shaka.extern.RequestFilter} */ @@ -1215,7 +1261,7 @@ export default defineComponent({ // only when we aren't proxying through Invidious, // it doesn't like the range param and makes get requests to youtube anyway - if (url.hostname.endsWith('.googlevideo.com') && url.pathname === '/videoplayback') { + if (url.protocol !== 'sabr:' && url.hostname.endsWith('.googlevideo.com') && url.pathname === '/videoplayback') { request.method = 'POST' request.body = new Uint8Array([0x78, 0]) // protobuf: { 15: 0 } (no idea what it means but this is what YouTube uses) @@ -1229,13 +1275,15 @@ export default defineComponent({ } } - /** - * Handles Application Level Redirects - * Based on the example in the YouTube.js repository - * @type {shaka.extern.ResponseFilter} - */ + /** @type {shaka.extern.ResponseFilter} */ async function responseFilter(type, response, context) { if (type === RequestType.SEGMENT) { + const url = new URL(response.uri) + + if (url.protocol === 'sabr:') { + return + } + if (response.data && response.data.byteLength > 4 && new DataView(response.data).getUint32(0) === HTTP_IN_HEX) { // Interpret the response data as a URL string. @@ -1255,8 +1303,6 @@ export default defineComponent({ response.headers = redirectResponse.headers response.uri = redirectResponse.uri } else { - const url = new URL(response.uri) - // Fix positioning for auto-generated subtitles if (url.hostname.endsWith('.youtube.com') && url.pathname === '/api/timedtext' && url.searchParams.get('caps') === 'asr' && url.searchParams.get('kind') === 'asr' && url.searchParams.get('fmt') === 'vtt') { @@ -1435,12 +1481,6 @@ export default defineComponent({ updateLegacyQualityStats(activeLegacyFormat.value) } - const playerDimensions = video_.getBoundingClientRect() - stats.playerDimensions = { - width: Math.floor(playerDimensions.width), - height: Math.floor(playerDimensions.height) - } - if (!hasLoaded.value) { player.addEventListener('loaded', () => { if (showStats.value) { @@ -1496,14 +1536,13 @@ export default defineComponent({ stats.resolution.width = newTrack.width stats.resolution.height = newTrack.height } else { - // for videos with multiple audio tracks, youtube.js appends the track id to the itag, to make it unique - stats.codecs.audioItag = newTrack.originalAudioId.split('-')[0] + stats.codecs.audioItag = newTrack.originalAudioId.split('-', 1)[0] stats.codecs.audioCodec = newTrack.audioCodec if (props.format === 'dash') { stats.resolution.frameRate = newTrack.frameRate - stats.codecs.videoItag = newTrack.originalVideoId + stats.codecs.videoItag = newTrack.originalVideoId.split('-', 1)[0] stats.codecs.videoCodec = newTrack.videoCodec stats.resolution.width = newTrack.width @@ -1536,12 +1575,6 @@ export default defineComponent({ } function updateStats() { - const playerDimensions = video.value.getBoundingClientRect() - stats.playerDimensions = { - width: Math.floor(playerDimensions.width), - height: Math.floor(playerDimensions.height) - } - const playerStats = player.getStats() if (props.format !== 'audio') { @@ -2308,18 +2341,35 @@ export default defineComponent({ function handleError(error, context, details) { // These two errors are just wrappers around another error, so use the original error instead // As they can be nested (e.g. multiple googlevideo redirects because the Invidious server was far away from the user) we should pick the inner most one - while (error.code === shaka.util.Error.Code.REQUEST_FILTER_ERROR || error.code === shaka.util.Error.Code.RESPONSE_FILTER_ERROR) { + while (error.code === ErrorCode.REQUEST_FILTER_ERROR || error.code === ErrorCode.RESPONSE_FILTER_ERROR) { error = error.data[0] } + // Allow shaka-player to retry on potentially recoverable network errors + if (error.severity === ErrorSeverity.RECOVERABLE && error.category === ErrorCategory.NETWORK) { + /** @type {keyof ErrorCategory} */ + const categoryText = Object.keys(ErrorCategory).find((/** @type {keyof ErrorCategory} */ key) => ErrorCategory[key] === error.category) + + /** @type {keyof ErrorCode} */ + const codeText = Object.keys(ErrorCode).find((/** @type {keyof ErrorCode} */ key) => ErrorCode[key] === error.code) + + console.warn( + 'Recoverable network error retrying...\n' + + `Category: ${categoryText} (${error.category})\n` + + `Code: ${codeText} (${error.code})\n` + + 'Data', error.data + ) + return + } + logShakaError(error, context, props.videoId, details) // text related errors aren't serious (captions and seek bar thumbnails), so we should just log them // TODO: consider only emitting when the severity is crititcal? if ( !ignoreErrors && - error.category !== shaka.util.Error.Category.TEXT && - !(error.code === shaka.util.Error.Code.BAD_HTTP_STATUS && error.data[0].startsWith('https://www.youtube.com/api/timedtext')) + error.category !== ErrorCategory.TEXT && + !(error.code === ErrorCode.BAD_HTTP_STATUS && error.data[0].startsWith('https://www.youtube.com/api/timedtext')) ) { // don't react to multiple consecutive errors, otherwise we don't give the format fallback from the previous error a chance to work ignoreErrors = true @@ -2493,6 +2543,8 @@ export default defineComponent({ return } + videoResizeObserver.observe(videoElement) + registerScreenshotButton() registerAudioTrackSelection() registerAutoplayToggle() @@ -2507,8 +2559,8 @@ export default defineComponent({ } else { useOverFlowMenu.value = container.value.getBoundingClientRect().width <= USE_OVERFLOW_MENU_WIDTH_THRESHOLD - resizeObserver = new ResizeObserver(resized) - resizeObserver.observe(container.value) + containerResizeObserver = new ResizeObserver(resized) + containerResizeObserver.observe(container.value) } controls.addEventListener('uiupdated', addUICustomizations) @@ -2599,40 +2651,59 @@ export default defineComponent({ // ideally we would set this in the `streaming` event handler, but for HLS this is only set to true after the loaded event fires. isLive.value = player.isLive() - const promises = [] - - for (const caption of sortedCaptions) { - if (props.format === 'legacy') { - const url = new URL(caption.url) - - if (url.hostname.endsWith('.youtube.com') && url.pathname === '/api/timedtext' && - url.searchParams.get('caps') === 'asr' && url.searchParams.get('kind') === 'asr' && url.searchParams.get('fmt') === 'vtt') { - promises.push((async () => { - try { - const response = await fetch(caption.url) - let text = await response.text() - - // position:0% for LTR text and position:100% for RTL text - text = text.replaceAll(/ align:start position:(?:10)?0%$/gm, '') - - const url = `data:${caption.mimeType};charset=utf-8,${encodeURIComponent(text)}` - - await player.addTextTrackAsync( - url, + if (process.env.SUPPORTS_LOCAL_API && props.format !== 'legacy' && props.manifestMimeType === MANIFEST_TYPE_SABR) { + sabrManifest = player.getManifest() + } + + // For SABR we include the thumbnails and subtitles in the manifest + if (!process.env.SUPPORTS_LOCAL_API || props.format === 'legacy' || props.manifestMimeType !== MANIFEST_TYPE_SABR) { + const promises = [] + + for (const caption of props.captions) { + if (props.format === 'legacy') { + const url = new URL(caption.url) + + if (url.hostname.endsWith('.youtube.com') && url.pathname === '/api/timedtext' && + url.searchParams.get('caps') === 'asr' && url.searchParams.get('kind') === 'asr' && url.searchParams.get('fmt') === 'vtt') { + promises.push((async () => { + try { + const response = await fetch(caption.url) + let text = await response.text() + + // position:0% for LTR text and position:100% for RTL text + text = text.replaceAll(/ align:start position:(?:10)?0%$/gm, '') + + const url = `data:${caption.mimeType};charset=utf-8,${encodeURIComponent(text)}` + + await player.addTextTrackAsync( + url, + caption.language, + 'captions', + caption.mimeType, + undefined, // codec, only needed if the captions are inside a container (e.g. mp4) + caption.label + ) + } catch (error) { + if (error instanceof shaka.util.Error) { + handleError(error, 'addTextTrackAsync', caption) + } else { + console.error(error) + } + } + })()) + } else { + promises.push( + player.addTextTrackAsync( + caption.url, caption.language, 'captions', caption.mimeType, undefined, // codec, only needed if the captions are inside a container (e.g. mp4) caption.label ) - } catch (error) { - if (error instanceof shaka.util.Error) { - handleError(error, 'addTextTrackAsync', caption) - } else { - console.error(error) - } - } - })()) + .catch(error => handleError(error, 'addTextTrackAsync', caption)) + ) + } } else { promises.push( player.addTextTrackAsync( @@ -2646,32 +2717,20 @@ export default defineComponent({ .catch(error => handleError(error, 'addTextTrackAsync', caption)) ) } - } else { + } + + if (!isLive.value && props.storyboardSrc) { promises.push( - player.addTextTrackAsync( - caption.url, - caption.language, - 'captions', - caption.mimeType, - undefined, // codec, only needed if the captions are inside a container (e.g. mp4) - caption.label - ) - .catch(error => handleError(error, 'addTextTrackAsync', caption)) + // Only log the error, as the thumbnails are a nice to have + // If an error occurs with them, it's not critical + player.addThumbnailsTrack(props.storyboardSrc, 'text/vtt') + .catch(error => logShakaError(error, 'addThumbnailsTrack', props.videoId, props.storyboardSrc)) ) } - } - if (!isLive.value && props.storyboardSrc) { - promises.push( - // Only log the error, as the thumbnails are a nice to have - // If an error occurs with them, it's not critical - player.addThumbnailsTrack(props.storyboardSrc, 'text/vtt') - .catch(error => logShakaError(error, 'addThumbnailsTrack', props.videoId, props.storyboardSrc)) - ) + await Promise.all(promises) } - await Promise.all(promises) - if (restoreCaptionIndex !== null) { const index = restoreCaptionIndex restoreCaptionIndex = null @@ -2857,9 +2916,13 @@ export default defineComponent({ document.removeEventListener('keydown', keyboardShortcutHandler) - if (resizeObserver) { - resizeObserver.disconnect() - resizeObserver = null + if (containerResizeObserver) { + containerResizeObserver.disconnect() + containerResizeObserver = null + } + + if (videoResizeObserver) { + videoResizeObserver.disconnect() } cleanUpCustomPlayerControls() @@ -2932,6 +2995,10 @@ export default defineComponent({ player = null } + if (process.env.SUPPORTS_LOCAL_API && cleanupSabrScheme) { + cleanupSabrScheme() + } + // shaka-player doesn't clear these itself, which prevents shaka.ui.Overlay from being garbage collected // Should really be fixed in shaka-player but it's easier just to do it ourselves if (container.value) { @@ -2986,6 +3053,7 @@ export default defineComponent({ showStats, stats, + playerDimensions, autoplayVideos, sponsorBlockShowSkippedToast, @@ -3000,6 +3068,8 @@ export default defineComponent({ handleEnded, updateVolume, handleTimeupdate, + handleEnterPictureInPicture, + handleLeavePictureInPicture, valueChangeMessage, valueChangeIcon, diff --git a/src/renderer/components/ft-shaka-video-player/ft-shaka-video-player.vue b/src/renderer/components/ft-shaka-video-player/ft-shaka-video-player.vue index f1e8cc8d86e8f..f74b2287879e5 100644 --- a/src/renderer/components/ft-shaka-video-player/ft-shaka-video-player.vue +++ b/src/renderer/components/ft-shaka-video-player/ft-shaka-video-player.vue @@ -22,6 +22,8 @@ @canplay="handleCanPlay" @volumechange="updateVolume" @timeupdate="handleTimeupdate" + @enterpictureinpicture="handleEnterPictureInPicture" + @leavepictureinpicture="handleLeavePictureInPicture" />