Skip to content

Commit c0f20c6

Browse files
litetexlitetex
committed
Fixed all YTMixPlaylists
Added option to choose if you want to consent or not - currently this is done by a static variable in ``YoutubeParsingHelper`` - may not be the best long-term solution but for now the tests work again (in EU countries) 🥳
1 parent 6b7a987 commit c0f20c6

File tree

5 files changed

+98
-61
lines changed

5 files changed

+98
-61
lines changed

‎extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ConsentRequiredException.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package org.schabi.newpipe.extractor.exceptions;
2+
3+
public class ConsentRequiredException extends ParsingException {
4+
5+
public ConsentRequiredException(final String message) {
6+
super(message);
7+
}
8+
9+
public ConsentRequiredException(final String message, final Throwable cause) {
10+
super(message, cause);
11+
}
12+
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 35 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -20,21 +20,12 @@
2020

2121
package org.schabi.newpipe.extractor.services.youtube;
2222

23-
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
24-
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
25-
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
26-
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
27-
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
28-
import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray;
29-
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
30-
3123
import com.grack.nanojson.JsonArray;
3224
import com.grack.nanojson.JsonBuilder;
3325
import com.grack.nanojson.JsonObject;
3426
import com.grack.nanojson.JsonParser;
3527
import com.grack.nanojson.JsonParserException;
3628
import com.grack.nanojson.JsonWriter;
37-
3829
import org.schabi.newpipe.extractor.MetaInfo;
3930
import org.schabi.newpipe.extractor.downloader.Response;
4031
import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException;
@@ -51,6 +42,8 @@
5142
import org.schabi.newpipe.extractor.utils.RandomStringFromAlphabetGenerator;
5243
import org.schabi.newpipe.extractor.utils.Utils;
5344

45+
import javax.annotation.Nonnull;
46+
import javax.annotation.Nullable;
5447
import java.io.IOException;
5548
import java.io.UnsupportedEncodingException;
5649
import java.net.MalformedURLException;
@@ -73,8 +66,13 @@
7366
import java.util.Random;
7467
import java.util.regex.Pattern;
7568

76-
import javax.annotation.Nonnull;
77-
import javax.annotation.Nullable;
69+
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
70+
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
71+
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
72+
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
73+
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
74+
import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray;
75+
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
7876

7977
public final class YoutubeParsingHelper {
8078

@@ -233,16 +231,19 @@ private YoutubeParsingHelper() {
233231
* The three digits at the end can be random, but are required.
234232
* </p>
235233
*/
236-
private static final String CONSENT_COOKIE_VALUE = "PENDING+";
237-
234+
private static final String CONSENT_COOKIE_PENDING_VALUE = "PENDING+";
238235
/**
239-
* YouTube {@code CONSENT} cookie.
236+
* {@code YES+} means that the user did submit their choices and accepted all cookies.
240237
*
241238
* <p>
242-
* Should prevent redirect to {@code consent.youtube.com}.
239+
* Therefore, YouTube & Google can track the user, because they did give consent.
240+
* </p>
241+
*
242+
* <p>
243+
* The three digits at the end can be random, but are required.
243244
* </p>
244245
*/
245-
private static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE;
246+
private static final String CONSENT_COOKIE_YES_VALUE = "YES+";
246247

247248
private static final String FEED_BASE_CHANNEL_ID =
248249
"https://www.youtube.com/feeds/videos.xml?channel_id=";
@@ -253,6 +254,13 @@ private YoutubeParsingHelper() {
253254
private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID");
254255
private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS");
255256

257+
/**
258+
* {@code false} (default) will use {@link #CONSENT_COOKIE_PENDING_VALUE}.
259+
* <br/>
260+
* {@code true} will use {@link #CONSENT_COOKIE_YES_VALUE}.
261+
*/
262+
private static boolean consentAccepted = false;
263+
256264
private static boolean isGoogleURL(final String url) {
257265
final String cachedUrl = extractCachedUrlIfNeeded(url);
258266
try {
@@ -1318,7 +1326,6 @@ public static void addClientInfoHeaders(@Nonnull final Map<String, List<String>>
13181326

13191327
/**
13201328
* Add the <code>CONSENT</code> cookie to prevent redirect to <code>consent.youtube.com</code>
1321-
* @see #CONSENT_COOKIE
13221329
* @param headers the headers which should be completed
13231330
*/
13241331
@SuppressWarnings("ArraysAsListWithZeroOrOneArgument")
@@ -1332,8 +1339,9 @@ public static void addCookieHeader(@Nonnull final Map<String, List<String>> head
13321339

13331340
@Nonnull
13341341
public static String generateConsentCookie() {
1335-
final int statusCode = 100 + numberGenerator.nextInt(900);
1336-
return CONSENT_COOKIE + statusCode;
1342+
return "CONSENT="
1343+
+ (isConsentAccepted() ? CONSENT_COOKIE_YES_VALUE : CONSENT_COOKIE_PENDING_VALUE)
1344+
+ (100 + numberGenerator.nextInt(900));
13371345
}
13381346

13391347
public static String extractCookieValue(final String cookieName,
@@ -1553,16 +1561,6 @@ public static boolean isVerified(final JsonArray badges) {
15531561
return false;
15541562
}
15551563

1556-
@Nonnull
1557-
public static String unescapeDocument(@Nonnull final String doc) {
1558-
return doc
1559-
.replaceAll("\\\\x22", "\"")
1560-
.replaceAll("\\\\x7b", "{")
1561-
.replaceAll("\\\\x7d", "}")
1562-
.replaceAll("\\\\x5b", "[")
1563-
.replaceAll("\\\\x5d", "]");
1564-
}
1565-
15661564
/**
15671565
* Generate a content playback nonce (also called {@code cpn}), sent by YouTube clients in
15681566
* playback requests (and also for some clients, in the player request body).
@@ -1633,4 +1631,12 @@ public static boolean isAndroidStreamingUrl(@Nonnull final String url) {
16331631
public static boolean isIosStreamingUrl(@Nonnull final String url) {
16341632
return Parser.isMatch(C_IOS_PATTERN, url);
16351633
}
1634+
1635+
public static void setConsentAccepted(final boolean accepted) {
1636+
consentAccepted = accepted;
1637+
}
1638+
1639+
public static boolean isConsentAccepted() {
1640+
return consentAccepted;
1641+
}
16361642
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java‎

Lines changed: 31 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,15 @@
11
package org.schabi.newpipe.extractor.services.youtube.extractors;
22

3-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
4-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
5-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
6-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
7-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
8-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
9-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
10-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
11-
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
12-
import static org.schabi.newpipe.extractor.utils.Utils.getQueryValue;
13-
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
14-
import static org.schabi.newpipe.extractor.utils.Utils.stringToURL;
15-
163
import com.grack.nanojson.JsonArray;
174
import com.grack.nanojson.JsonBuilder;
185
import com.grack.nanojson.JsonObject;
196
import com.grack.nanojson.JsonWriter;
20-
217
import org.schabi.newpipe.extractor.ListExtractor;
228
import org.schabi.newpipe.extractor.Page;
239
import org.schabi.newpipe.extractor.StreamingService;
2410
import org.schabi.newpipe.extractor.downloader.Downloader;
2511
import org.schabi.newpipe.extractor.downloader.Response;
12+
import org.schabi.newpipe.extractor.exceptions.ConsentRequiredException;
2613
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
2714
import org.schabi.newpipe.extractor.exceptions.ParsingException;
2815
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@@ -35,6 +22,8 @@
3522
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
3623
import org.schabi.newpipe.extractor.utils.JsonUtils;
3724

25+
import javax.annotation.Nonnull;
26+
import javax.annotation.Nullable;
3827
import java.io.IOException;
3928
import java.net.URL;
4029
import java.nio.charset.StandardCharsets;
@@ -43,8 +32,18 @@
4332
import java.util.Map;
4433
import java.util.Objects;
4534

46-
import javax.annotation.Nonnull;
47-
import javax.annotation.Nullable;
35+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
36+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
37+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addYouTubeHeaders;
38+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
39+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
40+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
41+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
42+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
43+
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
44+
import static org.schabi.newpipe.extractor.utils.Utils.getQueryValue;
45+
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
46+
import static org.schabi.newpipe.extractor.utils.Utils.stringToURL;
4847

4948
/**
5049
* A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist).
@@ -89,16 +88,26 @@ public void onFetchPage(@Nonnull final Downloader downloader)
8988
final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(StandardCharsets.UTF_8);
9089

9190
final Map<String, List<String>> headers = new HashMap<>();
92-
addClientInfoHeaders(headers);
91+
// Cookie is required due to consent
92+
addYouTubeHeaders(headers);
9393

9494
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey()
9595
+ DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
9696

9797
initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
98-
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
99-
.getObject("playlist").getObject("playlist");
98+
playlistData = initialData
99+
.getObject("contents")
100+
.getObject("twoColumnWatchNextResults")
101+
.getObject("playlist")
102+
.getObject("playlist");
100103
if (isNullOrEmpty(playlistData)) {
101-
throw new ExtractionException("Could not get playlistData");
104+
final ExtractionException ex = new ExtractionException("Could not get playlistData");
105+
if (!YoutubeParsingHelper.isConsentAccepted()) {
106+
throw new ConsentRequiredException(
107+
"Consent is required in some countries to view Mix playlists",
108+
ex);
109+
}
110+
throw ex;
102111
}
103112
cookieValue = extractCookieValue(COOKIE_NAME, response);
104113
}
@@ -212,7 +221,8 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
212221

213222
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
214223
final Map<String, List<String>> headers = new HashMap<>();
215-
addClientInfoHeaders(headers);
224+
// Cookie is required due to consent
225+
addYouTubeHeaders(headers);
216226

217227
final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(),
218228
getExtractorLocalization());

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,6 @@
11
package org.schabi.newpipe.extractor.services.youtube;
22

3-
import static org.junit.jupiter.api.Assertions.assertEquals;
4-
import static org.junit.jupiter.api.Assertions.assertFalse;
5-
import static org.junit.jupiter.api.Assertions.assertThrows;
6-
import static org.junit.jupiter.api.Assertions.assertTrue;
7-
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
8-
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
9-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
10-
113
import com.grack.nanojson.JsonWriter;
12-
134
import org.junit.jupiter.api.BeforeAll;
145
import org.junit.jupiter.api.Test;
156
import org.schabi.newpipe.downloader.DownloaderFactory;
@@ -31,6 +22,17 @@
3122
import java.util.Map;
3223
import java.util.Set;
3324

25+
import static org.junit.jupiter.api.Assertions.assertEquals;
26+
import static org.junit.jupiter.api.Assertions.assertFalse;
27+
import static org.junit.jupiter.api.Assertions.assertThrows;
28+
import static org.junit.jupiter.api.Assertions.assertTrue;
29+
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
30+
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
31+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
32+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
33+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
34+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
35+
3436
public class YoutubeMixPlaylistExtractorTest {
3537

3638
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/mix/";
@@ -45,6 +47,7 @@ public static class Mix {
4547
@BeforeAll
4648
public static void setUp() throws Exception {
4749
YoutubeTestsUtils.ensureStateless();
50+
YoutubeParsingHelper.setConsentAccepted(true);
4851
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "mix"));
4952
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
5053
extractor = (YoutubeMixPlaylistExtractor) YouTube
@@ -140,6 +143,7 @@ public static class MixWithIndex {
140143
@BeforeAll
141144
public static void setUp() throws Exception {
142145
YoutubeTestsUtils.ensureStateless();
146+
YoutubeParsingHelper.setConsentAccepted(true);
143147
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "mixWithIndex"));
144148
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
145149
extractor = (YoutubeMixPlaylistExtractor) YouTube
@@ -221,11 +225,12 @@ void getPlaylistType() throws ParsingException {
221225
}
222226

223227
public static class MyMix {
224-
private static final String VIDEO_ID = "_AzeUSL9lZc";
228+
private static final String VIDEO_ID = "YVkUvmDQ3HY";
225229

226230
@BeforeAll
227231
public static void setUp() throws Exception {
228232
YoutubeTestsUtils.ensureStateless();
233+
YoutubeParsingHelper.setConsentAccepted(true);
229234
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "myMix"));
230235
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
231236
extractor = (YoutubeMixPlaylistExtractor) YouTube
@@ -249,7 +254,7 @@ void getName() throws Exception {
249254
void getThumbnailUrl() throws Exception {
250255
final String thumbnailUrl = extractor.getThumbnailUrl();
251256
assertIsSecureUrl(thumbnailUrl);
252-
assertTrue(thumbnailUrl.startsWith("https://i.ytimg.com/vi/_AzeUSL9lZc"));
257+
assertTrue(thumbnailUrl.startsWith("https://i.ytimg.com/vi/" + VIDEO_ID));
253258
}
254259

255260
@Test
@@ -316,6 +321,7 @@ public static class Invalid {
316321
@BeforeAll
317322
public static void setUp() throws IOException {
318323
YoutubeTestsUtils.ensureStateless();
324+
YoutubeParsingHelper.setConsentAccepted(true);
319325
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "invalid"));
320326
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
321327
}
@@ -350,6 +356,7 @@ public static class ChannelMix {
350356
@BeforeAll
351357
public static void setUp() throws Exception {
352358
YoutubeTestsUtils.ensureStateless();
359+
YoutubeParsingHelper.setConsentAccepted(true);
353360
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "channelMix"));
354361
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
355362
extractor = (YoutubeMixPlaylistExtractor) YouTube
@@ -414,6 +421,7 @@ public static class GenreMix {
414421
@BeforeAll
415422
public static void setUp() throws Exception {
416423
YoutubeTestsUtils.ensureStateless();
424+
YoutubeParsingHelper.setConsentAccepted(true);
417425
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "genreMix"));
418426
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
419427
extractor = (YoutubeMixPlaylistExtractor) YouTube

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ private YoutubeTestsUtils() {
2121
* </p>
2222
*/
2323
public static void ensureStateless() {
24+
YoutubeParsingHelper.setConsentAccepted(false);
2425
YoutubeParsingHelper.resetClientVersionAndKey();
2526
YoutubeParsingHelper.setNumberGenerator(new Random(1));
2627
YoutubeStreamExtractor.resetDeobfuscationCode();

0 commit comments

Comments
 (0)