Merge pull request FreeTubeApp#7081 from FreeTubeApp/v0.23.3

[Master] Bump package version from 0.23.2 to 0.23.3

Author: ChunkyProgrammer

package.json

@@ -2,7 +2,7 @@
   "name": "freetube",
   "productName": "FreeTube",
   "description": "A private YouTube client",
-  "version": "0.23.2",
+  "version": "0.23.3",
   "license": "AGPL-3.0-or-later",
   "main": "./dist/main.js",
   "private": true,
@@ -61,7 +61,7 @@
     "@fortawesome/vue-fontawesome": "^2.0.10",
     "@seald-io/nedb": "^4.0.4",
     "autolinker": "^4.1.0",
-    "bgutils-js": "^3.1.3",
+    "bgutils-js": "^3.2.0",
     "electron-context-menu": "^4.0.4",
     "marked": "^15.0.6",
     "path-browserify": "^1.0.1",
@@ -74,7 +74,7 @@
     "vue-observe-visibility": "^1.0.0",
     "vue-router": "^3.6.5",
     "vuex": "^3.6.2",
-    "youtubei.js": "^13.1.0"
+    "youtubei.js": "^13.3.0"
   },
   "devDependencies": {
     "@babel/core": "^7.26.7",

src/botGuardScript.js

@@ -1,37 +1,90 @@
-import { BG } from 'bgutils-js'
+import { BG, buildURL, GOOG_API_KEY } from 'bgutils-js'
 
 // This script has it's own webpack config, as it gets passed as a string to Electron's evaluateJavaScript function
 // in src/main/poTokenGenerator.js
-export default async function(visitorData) {
+
+/**
+ * Based on: https://github.com/LuanRT/BgUtils/blob/main/examples/node/innertube-challenge-fetcher-example.ts
+ * @param {string} videoId
+ * @param {string} visitorData
+ * @param {import('youtubei.js').Session['context']} context
+ */
+export default async function (videoId, visitorData, context) {
   const requestKey = 'O43z0dpjhgX20SCx4KAo'
 
-  const bgConfig = {
-    fetch: (input, init) => fetch(input, init),
-    requestKey,
-    globalObj: window,
-    identifier: visitorData
+  const challengeResponse = await fetch(
+    'https://www.youtube.com/youtubei/v1/att/get?prettyPrint=false&alt=json',
+    {
+      method: 'POST',
+      headers: {
+        Accept: '*/*',
+        'Content-Type': 'application/json',
+        'X-Goog-Visitor-Id': visitorData,
+        'X-Youtube-Client-Version': context.client.clientVersion,
+        'X-Youtube-Client-Name': '1'
+      },
+      body: JSON.stringify({
+        engagementType: 'ENGAGEMENT_TYPE_UNBOUND',
+        context
+      }),
+    }
+  )
+
+  if (!challengeResponse.ok) {
+    throw new Error(`Request to ${challengeResponse.url} failed with status ${challengeResponse.status}\n${await challengeResponse.text()}`)
   }
 
-  const challenge = await BG.Challenge.create(bgConfig)
+  const challengeData = await challengeResponse.json()
 
-  if (!challenge) {
-    throw new Error('Could not get challenge')
+  if (!challengeData.bgChallenge) {
+    throw new Error('Failed to get BotGuard challenge')
   }
 
-  const interpreterJavascript = challenge.interpreterJavascript.privateDoNotAccessOrElseSafeScriptWrappedValue
+  let interpreterUrl = challengeData.bgChallenge.interpreterUrl.privateDoNotAccessOrElseTrustedResourceUrlWrappedValue
+
+  if (interpreterUrl.startsWith('//')) {
+    interpreterUrl = `https:${interpreterUrl}`
+  }
+
+  const bgScriptResponse = await fetch(interpreterUrl)
+  const interpreterJavascript = await bgScriptResponse.text()
 
   if (interpreterJavascript) {
     // eslint-disable-next-line no-new-func
     new Function(interpreterJavascript)()
   } else {
-    console.warn('Unable to load VM.')
+    throw new Error('Could not load VM.')
   }
 
-  const poTokenResult = await BG.PoToken.generate({
-    program: challenge.program,
-    globalName: challenge.globalName,
-    bgConfig
+  const botGuard = await BG.BotGuardClient.create({
+    program: challengeData.bgChallenge.program,
+    globalName: challengeData.bgChallenge.globalName,
+    globalObj: window
   })
 
-  return poTokenResult.poToken
+  const webPoSignalOutput = []
+  const botGuardResponse = await botGuard.snapshot({ webPoSignalOutput })
+
+  const integrityTokenResponse = await fetch(buildURL('GenerateIT', true), {
+    method: 'POST',
+    headers: {
+      'content-type': 'application/json+protobuf',
+      'x-goog-api-key': GOOG_API_KEY,
+      'x-user-agent': 'grpc-web-javascript/0.1',
+    },
+    body: JSON.stringify([requestKey, botGuardResponse])
+  })
+
+  const response = await integrityTokenResponse.json()
+
+  if (typeof response[0] !== 'string') {
+    throw new Error('Could not get integrity token')
+  }
+
+  const integrityTokenBasedMinter = await BG.WebPoMinter.create({ integrityToken: response[0] }, webPoSignalOutput)
+
+  const contentPoToken = await integrityTokenBasedMinter.mintAsWebsafeString(videoId)
+  const sessionPoToken = await integrityTokenBasedMinter.mintAsWebsafeString(visitorData)
+
+  return { contentPoToken, sessionPoToken }
 }

src/constants.js

@@ -47,7 +47,7 @@ const IpcChannels = {
 
   SET_INVIDIOUS_AUTHORIZATION: 'set-invidious-authorization',
 
-  GENERATE_PO_TOKEN: 'generate-po-token',
+  GENERATE_PO_TOKENS: 'generate-po-tokens',
 
   WRITE_SCREENSHOT: 'write-screenshot',
 }

src/main/index.js

@@ -287,6 +287,8 @@ function runApp() {
     })
   }
 
+  let proxyUrl
+
   app.on('ready', async (_, __) => {
     if (process.env.NODE_ENV === 'production') {
       protocol.handle('app', async (request) => {
@@ -400,8 +402,10 @@ function runApp() {
     }
 
     if (useProxy) {
+      proxyUrl = `${proxyProtocol}://${proxyHostname}:${proxyPort}`
+
       session.defaultSession.setProxy({
-        proxyRules: `${proxyProtocol}://${proxyHostname}:${proxyPort}`
+        proxyRules: proxyUrl
       })
     }
 
@@ -883,19 +887,21 @@ function runApp() {
     })
   })
 
-  ipcMain.handle(IpcChannels.GENERATE_PO_TOKEN, (_, visitorData) => {
-    return generatePoToken(visitorData)
+  ipcMain.handle(IpcChannels.GENERATE_PO_TOKENS, (_, videoId, visitorData, context) => {
+    return generatePoToken(videoId, visitorData, context, proxyUrl)
   })
 
   ipcMain.on(IpcChannels.ENABLE_PROXY, (_, url) => {
     session.defaultSession.setProxy({
       proxyRules: url
     })
+    proxyUrl = url
     session.defaultSession.closeAllConnections()
   })
 
   ipcMain.on(IpcChannels.DISABLE_PROXY, () => {
     session.defaultSession.setProxy({})
+    proxyUrl = undefined
     session.defaultSession.closeAllConnections()
   })
 

src/main/poTokenGenerator.js

@@ -9,10 +9,13 @@ import { join } from 'path'
  * This is intentionally split out into it's own thing, with it's own temporary in-memory session,
  * as the BotGuard stuff accesses the global `document` and `window` objects and also requires making some requests.
  * So we definitely don't want it running in the same places as the rest of the FreeTube code with the user data.
+ * @param {string} videoId
  * @param {string} visitorData
- * @returns {Promise<string>}
+ * @param {string} context
+ * @param {string|undefined} proxyUrl
+ * @returns {Promise<{ contentPoToken: string, sessionPoToken: string }>}
  */
-export async function generatePoToken(visitorData) {
+export async function generatePoToken(videoId, visitorData, context, proxyUrl) {
   const sessionUuid = crypto.randomUUID()
 
   const theSession = session.fromPartition(`potoken-${sessionUuid}`, { cache: false })
@@ -21,12 +24,34 @@ export async function generatePoToken(visitorData) {
   // eslint-disable-next-line n/no-callback-literal
   theSession.setPermissionRequestHandler((webContents, permission, callback) => callback(false))
 
-  theSession.setUserAgent(
-    theSession.getUserAgent()
-      .split(' ')
-      .filter(part => !part.includes('Electron'))
-      .join(' ')
-  )
+  theSession.setUserAgent(session.defaultSession.getUserAgent())
+
+  if (proxyUrl) {
+    await theSession.setProxy({
+      proxyRules: proxyUrl
+    })
+  }
+
+  theSession.webRequest.onBeforeSendHeaders({
+    urls: ['https://www.google.com/js/*', 'https://www.youtube.com/youtubei/*']
+  }, ({ requestHeaders, url }, callback) => {
+    if (url.startsWith('https://www.youtube.com/youtubei/')) {
+      // make InnerTube requests work with the fetch function
+      // InnerTube rejects requests if the referer isn't YouTube or empty
+      requestHeaders.Referer = 'https://www.youtube.com/'
+      requestHeaders.Origin = 'https://www.youtube.com'
+
+      requestHeaders['Sec-Fetch-Site'] = 'same-origin'
+      requestHeaders['Sec-Fetch-Mode'] = 'same-origin'
+      requestHeaders['X-Youtube-Bootstrap-Logged-In'] = 'false'
+    } else {
+      requestHeaders['Sec-Fetch-Dest'] = 'script'
+      requestHeaders['Sec-Fetch-Site'] = 'cross-site'
+      requestHeaders['Accept-Language'] = '*'
+    }
+
+    callback({ requestHeaders })
+  })
 
   const webContentsView = new WebContentsView({
     webPreferences: {
@@ -35,7 +60,8 @@ export async function generatePoToken(visitorData) {
       sandbox: true,
       v8CacheOptions: 'none',
       session: theSession,
-      offscreen: true
+      offscreen: true,
+      webSecurity: false
     }
   })
 
@@ -51,43 +77,8 @@ export async function generatePoToken(visitorData) {
 
   webContentsView.webContents.debugger.attach()
 
-  await webContentsView.webContents.loadURL('data:text/html,', {
-    baseURLForDataURL: 'https://www.youtube.com'
-  })
-
-  await webContentsView.webContents.debugger.sendCommand('Emulation.setUserAgentOverride', {
-    userAgent: theSession.getUserAgent(),
-    acceptLanguage: 'en-US',
-    platform: 'Win32',
-    userAgentMetadata: {
-      brands: [
-        {
-          brand: 'Not/A)Brand',
-          version: '99'
-        },
-        {
-          brand: 'Chromium',
-          version: process.versions.chrome.split('.')[0]
-        }
-      ],
-      fullVersionList: [
-        {
-          brand: 'Not/A)Brand',
-          version: '99.0.0.0'
-        },
-        {
-          brand: 'Chromium',
-          version: process.versions.chrome
-        }
-      ],
-      platform: 'Windows',
-      platformVersion: '10.0.0',
-      architecture: 'x86',
-      model: '',
-      mobile: false,
-      bitness: '64',
-      wow64: false
-    }
+  await webContentsView.webContents.loadURL('data:text/html,<!DOCTYPE html><html lang="en"><head><title></title></head><body></body></html>', {
+    baseURLForDataURL: 'https://www.youtube.com/'
   })
 
   await webContentsView.webContents.debugger.sendCommand('Emulation.setDeviceMetricsOverride', {
@@ -105,7 +96,7 @@ export async function generatePoToken(visitorData) {
     }
   })
 
-  const script = await getScript(visitorData)
+  const script = await getScript(videoId, visitorData, context)
 
   const response = await webContentsView.webContents.executeJavaScript(script)
 
@@ -118,9 +109,11 @@ export async function generatePoToken(visitorData) {
 let cachedScript
 
 /**
+ * @param {string} videoId
  * @param {string} visitorData
+ * @param {string} context
  */
-async function getScript(visitorData) {
+async function getScript(videoId, visitorData, context) {
   if (!cachedScript) {
     const pathToScript = process.env.NODE_ENV === 'development'
       ? join(__dirname, '../../dist/botGuardScript.js')
@@ -133,8 +126,8 @@ async function getScript(visitorData) {
 
     const functionName = match[1]
 
-    cachedScript = content.replace(match[0], `;${functionName}("FT_VISITOR_DATA")`)
+    cachedScript = content.replace(match[0], `;${functionName}(FT_PARAMS)`)
   }
 
-  return cachedScript.replace('FT_VISITOR_DATA', visitorData)
+  return cachedScript.replace('FT_PARAMS', `"${videoId}","${visitorData}",${context}`)
 }

src/renderer/components/ft-shaka-video-player/ft-shaka-video-player.js

@@ -65,7 +65,7 @@ export default defineComponent({
     },
     manifestSrc: {
       type: String,
-      required: true
+      default: null
     },
     manifestMimeType: {
       type: String,

src/renderer/helpers/api/invidious.js

@@ -859,7 +859,9 @@ export function convertInvidiousToLocalFormat(format) {
   const [initStart, initEnd] = format.init.split('-')
   const [indexStart, indexEnd] = format.index.split('-')
 
-  const duration = parseInt(parseFloat(new URL(format.url).searchParams.get('dur')) * 1000)
+  const url = new URL(format.url)
+
+  const duration = parseInt(parseFloat(url.searchParams.get('dur')) * 1000)
 
   // only converts the properties that are needed to generate a DASH manifest with YouTube.js
   // audioQuality and qualityLabel don't go inside the DASH manifest, but are used by YouTube.js
@@ -896,6 +898,27 @@ export function convertInvidiousToLocalFormat(format) {
         })
   })
 
+  if (localFormat.has_audio && url.searchParams.has('xtags')) {
+    const xtags = url.searchParams.get('xtags').split(':')
+
+    localFormat.language = xtags.find((tag) => tag.startsWith('lang='))?.split('=')[1] || null
+    localFormat.is_drc = xtags.includes('drc=1')
+
+    const audioContent = xtags.find((tag) => tag.startsWith('acont='))?.split('=')[1]
+    localFormat.is_dubbed = audioContent === 'dubbed'
+    localFormat.is_descriptive = audioContent === 'descriptive'
+    localFormat.is_secondary = audioContent === 'secondary'
+    localFormat.is_auto_dubbed = audioContent === 'dubbed-auto'
+    localFormat.is_original = audioContent === 'original' ||
+      (
+        !localFormat.is_dubbed &&
+        !localFormat.is_descriptive &&
+        !localFormat.is_secondary &&
+        !localFormat.is_auto_dubbed &&
+        !localFormat.is_drc
+      )
+  }
+
   return localFormat
 }