Skip to content

Commit dd223af

Browse files
StypoxStypox
authored
Merge pull request #11955 from Stypox/po-token
[YouTube] Add support for poTokens
2 parents ba86ce1 + dbee8d8 commit dd223af

File tree

10 files changed

+855
-19
lines changed

10 files changed

+855
-19
lines changed

‎app/build.gradle‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,7 @@ dependencies {
208208
implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751'
209209
// WORKAROUND: if you get errors with the NewPipeExtractor dependency, replace `v0.24.3` with
210210
// the corresponding commit hash, since JitPack is sometimes buggy
211-
implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.24.4'
211+
implementation 'com.github.TeamNewPipe:NewPipeExtractor:9f83b385a'
212212
implementation 'com.github.TeamNewPipe:NoNonsense-FilePicker:5.0.0'
213213

214214
/** Checkstyle **/
@@ -241,6 +241,7 @@ dependencies {
241241
implementation "androidx.work:work-runtime-ktx:${androidxWorkVersion}"
242242
implementation "androidx.work:work-rxjava3:${androidxWorkVersion}"
243243
implementation 'com.google.android.material:material:1.11.0'
244+
implementation "androidx.webkit:webkit:1.9.0"
244245

245246
/** Third-party libraries **/
246247
// Instance state boilerplate elimination

‎app/src/main/assets/po_token.html‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
<!DOCTYPE html>
2+
<html lang="en"><head><title></title><script>
3+
/**
4+
* Factory method to create and load a BotGuardClient instance.
5+
* @param options - Configuration options for the BotGuardClient.
6+
* @returns A promise that resolves to a loaded BotGuardClient instance.
7+
*/
8+
function loadBotGuard(challengeData) {
9+
this.vm = this[challengeData.globalName];
10+
this.program = challengeData.program;
11+
this.vmFunctions = {};
12+
this.syncSnapshotFunction = null;
13+
14+
if (!this.vm)
15+
throw new Error('[BotGuardClient]: VM not found in the global object');
16+
17+
if (!this.vm.a)
18+
throw new Error('[BotGuardClient]: Could not load program');
19+
20+
const vmFunctionsCallback = function (
21+
asyncSnapshotFunction,
22+
shutdownFunction,
23+
passEventFunction,
24+
checkCameraFunction
25+
) {
26+
this.vmFunctions = {
27+
asyncSnapshotFunction: asyncSnapshotFunction,
28+
shutdownFunction: shutdownFunction,
29+
passEventFunction: passEventFunction,
30+
checkCameraFunction: checkCameraFunction
31+
};
32+
};
33+
34+
this.syncSnapshotFunction = this.vm.a(this.program, vmFunctionsCallback, true, this.userInteractionElement, function () {/** no-op */ }, [ [], [] ])[0]
35+
36+
// an asynchronous function runs in the background and it will eventually call
37+
// `vmFunctionsCallback`, however we need to manually tell JavaScript to pass
38+
// control to the things running in the background by interrupting this async
39+
// function in any way, e.g. with a delay of 1ms. The loop is most probably not
40+
// needed but is there just because.
41+
return new Promise(function (resolve, reject) {
42+
i = 0
43+
refreshIntervalId = setInterval(function () {
44+
if (!!this.vmFunctions.asyncSnapshotFunction) {
45+
resolve(this)
46+
clearInterval(refreshIntervalId);
47+
}
48+
if (i >= 10000) {
49+
reject("asyncSnapshotFunction is null even after 10 seconds")
50+
clearInterval(refreshIntervalId);
51+
}
52+
i += 1;
53+
}, 1);
54+
})
55+
}
56+
57+
/**
58+
* Takes a snapshot asynchronously.
59+
* @returns The snapshot result.
60+
* @example
61+
* ```ts
62+
* const result = await botguard.snapshot({
63+
* contentBinding: {
64+
* c: "a=6&a2=10&b=SZWDwKVIuixOp7Y4euGTgwckbJA&c=1729143849&d=1&t=7200&c1a=1&c6a=1&c6b=1&hh=HrMb5mRWTyxGJphDr0nW2Oxonh0_wl2BDqWuLHyeKLo",
65+
* e: "ENGAGEMENT_TYPE_VIDEO_LIKE",
66+
* encryptedVideoId: "P-vC09ZJcnM"
67+
* }
68+
* });
69+
*
70+
* console.log(result);
71+
* ```
72+
*/
73+
function snapshot(args) {
74+
return new Promise(function (resolve, reject) {
75+
if (!this.vmFunctions.asyncSnapshotFunction)
76+
return reject(new Error('[BotGuardClient]: Async snapshot function not found'));
77+
78+
this.vmFunctions.asyncSnapshotFunction(function (response) { resolve(response) }, [
79+
args.contentBinding,
80+
args.signedTimestamp,
81+
args.webPoSignalOutput,
82+
args.skipPrivacyBuffer
83+
]);
84+
});
85+
}
86+
87+
function runBotGuard(challengeData) {
88+
const interpreterJavascript = challengeData.interpreterJavascript.privateDoNotAccessOrElseSafeScriptWrappedValue;
89+
90+
if (interpreterJavascript) {
91+
new Function(interpreterJavascript)();
92+
} else throw new Error('Could not load VM');
93+
94+
const webPoSignalOutput = [];
95+
return loadBotGuard({
96+
globalName: challengeData.globalName,
97+
globalObj: this,
98+
program: challengeData.program
99+
}).then(function (botguard) {
100+
return botguard.snapshot({ webPoSignalOutput: webPoSignalOutput })
101+
}).then(function (botguardResponse) {
102+
return { webPoSignalOutput: webPoSignalOutput, botguardResponse: botguardResponse }
103+
})
104+
}
105+
106+
function obtainPoToken(webPoSignalOutput, integrityToken, identifier) {
107+
const getMinter = webPoSignalOutput[0];
108+
109+
if (!getMinter)
110+
throw new Error('PMD:Undefined');
111+
112+
const mintCallback = getMinter(integrityToken);
113+
114+
if (!(mintCallback instanceof Function))
115+
throw new Error('APF:Failed');
116+
117+
const result = mintCallback(identifier);
118+
119+
if (!result)
120+
throw new Error('YNJ:Undefined');
121+
122+
if (!(result instanceof Uint8Array))
123+
throw new Error('ODM:Invalid');
124+
125+
return result;
126+
}
127+
</script></head><body></body></html>

‎app/src/main/java/org/schabi/newpipe/App.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import org.schabi.newpipe.error.ReCaptchaActivity;
1818
import org.schabi.newpipe.extractor.NewPipe;
1919
import org.schabi.newpipe.extractor.downloader.Downloader;
20+
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
2021
import org.schabi.newpipe.ktx.ExceptionUtils;
2122
import org.schabi.newpipe.settings.NewPipeSettings;
2223
import org.schabi.newpipe.util.BridgeStateSaverInitializer;
@@ -26,6 +27,7 @@
2627
import org.schabi.newpipe.util.image.ImageStrategy;
2728
import org.schabi.newpipe.util.image.PicassoHelper;
2829
import org.schabi.newpipe.util.image.PreferredImageQuality;
30+
import org.schabi.newpipe.util.potoken.PoTokenProviderImpl;
2931

3032
import java.io.IOException;
3133
import java.io.InterruptedIOException;
@@ -118,6 +120,8 @@ public void onCreate() {
118120
&& prefs.getBoolean(getString(R.string.show_image_indicators_key), false));
119121

120122
configureRxJavaErrorHandler();
123+
124+
YoutubeStreamExtractor.setPoTokenProvider(PoTokenProviderImpl.INSTANCE);
121125
}
122126

123127
@Override

‎app/src/main/java/org/schabi/newpipe/player/datasource/YoutubeHttpDataSource.java‎

Lines changed: 21 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,12 @@
1414
import static com.google.android.exoplayer2.util.Util.castNonNull;
1515
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent;
1616
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent;
17+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTvHtml5UserAgent;
1718
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl;
1819
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl;
20+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5StreamingUrl;
1921
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl;
20-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5SimplyEmbeddedPlayerStreamingUrl;
22+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebEmbeddedPlayerStreamingUrl;
2123
import static java.lang.Math.min;
2224

2325
import android.net.Uri;
@@ -270,6 +272,7 @@ public YoutubeHttpDataSource createDataSource() {
270272

271273
private static final String RN_PARAMETER = "&rn=";
272274
private static final String YOUTUBE_BASE_URL = "https://www.youtube.com";
275+
private static final byte[] POST_BODY = new byte[] {0x78, 0};
273276

274277
private final boolean allowCrossProtocolRedirects;
275278
private final boolean rangeParameterEnabled;
@@ -658,8 +661,11 @@ private HttpURLConnection makeConnection(
658661
}
659662
}
660663

664+
final boolean isTvHtml5StreamingUrl = isTvHtml5StreamingUrl(requestUrl);
665+
661666
if (isWebStreamingUrl(requestUrl)
662-
|| isTvHtml5SimplyEmbeddedPlayerStreamingUrl(requestUrl)) {
667+
|| isTvHtml5StreamingUrl
668+
|| isWebEmbeddedPlayerStreamingUrl(requestUrl)) {
663669
httpURLConnection.setRequestProperty(HttpHeaders.ORIGIN, YOUTUBE_BASE_URL);
664670
httpURLConnection.setRequestProperty(HttpHeaders.REFERER, YOUTUBE_BASE_URL);
665671
httpURLConnection.setRequestProperty(HttpHeaders.SEC_FETCH_DEST, "empty");
@@ -679,6 +685,9 @@ private HttpURLConnection makeConnection(
679685
} else if (isIosStreamingUrl) {
680686
httpURLConnection.setRequestProperty(HttpHeaders.USER_AGENT,
681687
getIosUserAgent(null));
688+
} else if (isTvHtml5StreamingUrl) {
689+
httpURLConnection.setRequestProperty(HttpHeaders.USER_AGENT,
690+
getTvHtml5UserAgent());
682691
} else {
683692
// non-mobile user agent
684693
httpURLConnection.setRequestProperty(HttpHeaders.USER_AGENT, DownloaderImpl.USER_AGENT);
@@ -687,22 +696,16 @@ private HttpURLConnection makeConnection(
687696
httpURLConnection.setRequestProperty(HttpHeaders.ACCEPT_ENCODING,
688697
allowGzip ? "gzip" : "identity");
689698
httpURLConnection.setInstanceFollowRedirects(followRedirects);
690-
httpURLConnection.setDoOutput(httpBody != null);
691-
692-
// Mobile clients uses POST requests to fetch contents
693-
httpURLConnection.setRequestMethod(isAndroidStreamingUrl || isIosStreamingUrl
694-
? "POST"
695-
: DataSpec.getStringForHttpMethod(httpMethod));
696-
697-
if (httpBody != null) {
698-
httpURLConnection.setFixedLengthStreamingMode(httpBody.length);
699-
httpURLConnection.connect();
700-
final OutputStream os = httpURLConnection.getOutputStream();
701-
os.write(httpBody);
702-
os.close();
703-
} else {
704-
httpURLConnection.connect();
705-
}
699+
// Most clients use POST requests to fetch contents
700+
httpURLConnection.setRequestMethod("POST");
701+
httpURLConnection.setDoOutput(true);
702+
httpURLConnection.setFixedLengthStreamingMode(POST_BODY.length);
703+
httpURLConnection.connect();
704+
705+
final OutputStream os = httpURLConnection.getOutputStream();
706+
os.write(POST_BODY);
707+
os.close();
708+
706709
return httpURLConnection;
707710
}
708711

‎app/src/main/java/org/schabi/newpipe/util/DeviceUtils.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import android.view.KeyEvent;
1818
import android.view.WindowInsets;
1919
import android.view.WindowManager;
20+
import android.webkit.CookieManager;
2021

2122
import androidx.annotation.Dimension;
2223
import androidx.annotation.NonNull;
@@ -335,4 +336,17 @@ public static boolean shouldSupportMediaTunneling() {
335336
&& !TX_50JXW834
336337
&& !HMB9213NW;
337338
}
339+
340+
/**
341+
* @return whether the device has support for WebView, see
342+
* <a href="https://stackoverflow.com/a/69626735">https://stackoverflow.com/a/69626735</a>
343+
*/
344+
public static boolean supportsWebView() {
345+
try {
346+
CookieManager.getInstance();
347+
return true;
348+
} catch (final Throwable ignored) {
349+
return false;
350+
}
351+
}
338352
}

‎app/src/main/java/org/schabi/newpipe/util/potoken/JavaScriptUtil.kt‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
package org.schabi.newpipe.util.potoken
2+
3+
import com.grack.nanojson.JsonObject
4+
import com.grack.nanojson.JsonParser
5+
import com.grack.nanojson.JsonWriter
6+
import okio.ByteString.Companion.decodeBase64
7+
import okio.ByteString.Companion.toByteString
8+
9+
/**
10+
* Parses the raw challenge data obtained from the Create endpoint and returns an object that can be
11+
* embedded in a JavaScript snippet.
12+
*/
13+
fun parseChallengeData(rawChallengeData: String): String {
14+
val scrambled = JsonParser.array().from(rawChallengeData)
15+
16+
val challengeData = if (scrambled.size > 1 && scrambled.isString(1)) {
17+
val descrambled = descramble(scrambled.getString(1))
18+
JsonParser.array().from(descrambled)
19+
} else {
20+
scrambled.getArray(1)
21+
}
22+
23+
val messageId = challengeData.getString(0)
24+
val interpreterHash = challengeData.getString(3)
25+
val program = challengeData.getString(4)
26+
val globalName = challengeData.getString(5)
27+
val clientExperimentsStateBlob = challengeData.getString(7)
28+
29+
val privateDoNotAccessOrElseSafeScriptWrappedValue = challengeData.getArray(1, null)?.find { it is String }
30+
val privateDoNotAccessOrElseTrustedResourceUrlWrappedValue = challengeData.getArray(2, null)?.find { it is String }
31+
32+
return JsonWriter.string(
33+
JsonObject.builder()
34+
.value("messageId", messageId)
35+
.`object`("interpreterJavascript")
36+
.value("privateDoNotAccessOrElseSafeScriptWrappedValue", privateDoNotAccessOrElseSafeScriptWrappedValue)
37+
.value("privateDoNotAccessOrElseTrustedResourceUrlWrappedValue", privateDoNotAccessOrElseTrustedResourceUrlWrappedValue)
38+
.end()
39+
.value("interpreterHash", interpreterHash)
40+
.value("program", program)
41+
.value("globalName", globalName)
42+
.value("clientExperimentsStateBlob", clientExperimentsStateBlob)
43+
.done()
44+
)
45+
}
46+
47+
/**
48+
* Parses the raw integrity token data obtained from the GenerateIT endpoint to a JavaScript
49+
* `Uint8Array` that can be embedded directly in JavaScript code, and an [Int] representing the
50+
* duration of this token in seconds.
51+
*/
52+
fun parseIntegrityTokenData(rawIntegrityTokenData: String): Pair<String, Long> {
53+
val integrityTokenData = JsonParser.array().from(rawIntegrityTokenData)
54+
return base64ToU8(integrityTokenData.getString(0)) to integrityTokenData.getLong(1)
55+
}
56+
57+
/**
58+
* Converts a string (usually the identifier used as input to `obtainPoToken`) to a JavaScript
59+
* `Uint8Array` that can be embedded directly in JavaScript code.
60+
*/
61+
fun stringToU8(identifier: String): String {
62+
return newUint8Array(identifier.toByteArray())
63+
}
64+
65+
/**
66+
* Takes a poToken encoded as a sequence of bytes represented as integers separated by commas
67+
* (e.g. "97,98,99" would be "abc"), which is the output of `Uint8Array::toString()` in JavaScript,
68+
* and converts it to the specific base64 representation for poTokens.
69+
*/
70+
fun u8ToBase64(poToken: String): String {
71+
return poToken.split(",")
72+
.map { it.toUByte().toByte() }
73+
.toByteArray()
74+
.toByteString()
75+
.base64()
76+
.replace("+", "-")
77+
.replace("/", "_")
78+
}
79+
80+
/**
81+
* Takes the scrambled challenge, decodes it from base64, adds 97 to each byte.
82+
*/
83+
private fun descramble(scrambledChallenge: String): String {
84+
return base64ToByteString(scrambledChallenge)
85+
.map { (it + 97).toByte() }
86+
.toByteArray()
87+
.decodeToString()
88+
}
89+
90+
/**
91+
* Decodes a base64 string encoded in the specific base64 representation used by YouTube, and
92+
* returns a JavaScript `Uint8Array` that can be embedded directly in JavaScript code.
93+
*/
94+
private fun base64ToU8(base64: String): String {
95+
return newUint8Array(base64ToByteString(base64))
96+
}
97+
98+
private fun newUint8Array(contents: ByteArray): String {
99+
return "new Uint8Array([" + contents.joinToString(separator = ",") { it.toUByte().toString() } + "])"
100+
}
101+
102+
/**
103+
* Decodes a base64 string encoded in the specific base64 representation used by YouTube.
104+
*/
105+
private fun base64ToByteString(base64: String): ByteArray {
106+
val base64Mod = base64
107+
.replace('-', '+')
108+
.replace('_', '/')
109+
.replace('.', '=')
110+
111+
return (base64Mod.decodeBase64() ?: throw PoTokenException("Cannot base64 decode"))
112+
.toByteArray()
113+
}

0 commit comments

Comments
 (0)