Skip to content

Commit ecfc370

Browse files
litetexlitetex
committed
Fixed all YTMixPlaylists
Added option to choose if you want to consent or not - currently this is done by a static variable in ``YoutubeParsingHelper`` - may not be the best long-term solution but for now the tests work again (in EU countries) 🥳
1 parent 2b6fe29 commit ecfc370

File tree

5 files changed

+89
-51
lines changed

5 files changed

+89
-51
lines changed

‎extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ConsentRequiredException.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package org.schabi.newpipe.extractor.exceptions;
2+
3+
public class ConsentRequiredException extends ParsingException {
4+
5+
public ConsentRequiredException(final String message) {
6+
super(message);
7+
}
8+
9+
public ConsentRequiredException(final String message, final Throwable cause) {
10+
super(message, cause);
11+
}
12+
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 26 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
2828
import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray;
2929
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
30-
3130
import static java.util.Collections.singletonList;
3231

3332
import com.grack.nanojson.JsonArray;
@@ -245,16 +244,19 @@ private YoutubeParsingHelper() {
245244
* The three digits at the end can be random, but are required.
246245
* </p>
247246
*/
248-
private static final String CONSENT_COOKIE_VALUE = "PENDING+";
249-
247+
private static final String CONSENT_COOKIE_PENDING_VALUE = "PENDING+";
250248
/**
251-
* YouTube {@code CONSENT} cookie.
249+
* {@code YES+} means that the user did submit their choices and accepted all cookies.
252250
*
253251
* <p>
254-
* Should prevent redirect to {@code consent.youtube.com}.
252+
* Therefore, YouTube & Google can track the user, because they did give consent.
253+
* </p>
254+
*
255+
* <p>
256+
* The three digits at the end can be random, but are required.
255257
* </p>
256258
*/
257-
private static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE;
259+
private static final String CONSENT_COOKIE_YES_VALUE = "YES+";
258260

259261
private static final String FEED_BASE_CHANNEL_ID =
260262
"https://www.youtube.com/feeds/videos.xml?channel_id=";
@@ -265,6 +267,13 @@ private YoutubeParsingHelper() {
265267
private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID");
266268
private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS");
267269

270+
/**
271+
* {@code false} (default) will use {@link #CONSENT_COOKIE_PENDING_VALUE}.
272+
* <br/>
273+
* {@code true} will use {@link #CONSENT_COOKIE_YES_VALUE}.
274+
*/
275+
private static boolean consentAccepted = false;
276+
268277
private static boolean isGoogleURL(final String url) {
269278
final String cachedUrl = extractCachedUrlIfNeeded(url);
270279
try {
@@ -1378,7 +1387,6 @@ public static Map<String, List<String>> getCookieHeader() {
13781387

13791388
/**
13801389
* Add the <code>CONSENT</code> cookie to prevent redirect to <code>consent.youtube.com</code>
1381-
* @see #CONSENT_COOKIE
13821390
* @param headers the headers which should be completed
13831391
*/
13841392
public static void addCookieHeader(@Nonnull final Map<String, List<String>> headers) {
@@ -1391,8 +1399,9 @@ public static void addCookieHeader(@Nonnull final Map<String, List<String>> head
13911399

13921400
@Nonnull
13931401
public static String generateConsentCookie() {
1394-
final int statusCode = 100 + numberGenerator.nextInt(900);
1395-
return CONSENT_COOKIE + statusCode;
1402+
return "CONSENT="
1403+
+ (isConsentAccepted() ? CONSENT_COOKIE_YES_VALUE : CONSENT_COOKIE_PENDING_VALUE)
1404+
+ (100 + numberGenerator.nextInt(900));
13961405
}
13971406

13981407
public static String extractCookieValue(final String cookieName,
@@ -1612,16 +1621,6 @@ public static boolean isVerified(final JsonArray badges) {
16121621
return false;
16131622
}
16141623

1615-
@Nonnull
1616-
public static String unescapeDocument(@Nonnull final String doc) {
1617-
return doc
1618-
.replaceAll("\\\\x22", "\"")
1619-
.replaceAll("\\\\x7b", "{")
1620-
.replaceAll("\\\\x7d", "}")
1621-
.replaceAll("\\\\x5b", "[")
1622-
.replaceAll("\\\\x5d", "]");
1623-
}
1624-
16251624
/**
16261625
* Generate a content playback nonce (also called {@code cpn}), sent by YouTube clients in
16271626
* playback requests (and also for some clients, in the player request body).
@@ -1692,4 +1691,12 @@ public static boolean isAndroidStreamingUrl(@Nonnull final String url) {
16921691
public static boolean isIosStreamingUrl(@Nonnull final String url) {
16931692
return Parser.isMatch(C_IOS_PATTERN, url);
16941693
}
1694+
1695+
public static void setConsentAccepted(final boolean accepted) {
1696+
consentAccepted = accepted;
1697+
}
1698+
1699+
public static boolean isConsentAccepted() {
1700+
return consentAccepted;
1701+
}
16951702
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java‎

Lines changed: 31 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,15 @@
11
package org.schabi.newpipe.extractor.services.youtube.extractors;
22

3-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
4-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
5-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
6-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
7-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
8-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
9-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
10-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
11-
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
12-
import static org.schabi.newpipe.extractor.utils.Utils.getQueryValue;
13-
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
14-
import static org.schabi.newpipe.extractor.utils.Utils.stringToURL;
15-
163
import com.grack.nanojson.JsonArray;
174
import com.grack.nanojson.JsonBuilder;
185
import com.grack.nanojson.JsonObject;
196
import com.grack.nanojson.JsonWriter;
20-
217
import org.schabi.newpipe.extractor.ListExtractor;
228
import org.schabi.newpipe.extractor.Page;
239
import org.schabi.newpipe.extractor.StreamingService;
2410
import org.schabi.newpipe.extractor.downloader.Downloader;
2511
import org.schabi.newpipe.extractor.downloader.Response;
12+
import org.schabi.newpipe.extractor.exceptions.ConsentRequiredException;
2613
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
2714
import org.schabi.newpipe.extractor.exceptions.ParsingException;
2815
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@@ -35,6 +22,8 @@
3522
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
3623
import org.schabi.newpipe.extractor.utils.JsonUtils;
3724

25+
import javax.annotation.Nonnull;
26+
import javax.annotation.Nullable;
3827
import java.io.IOException;
3928
import java.net.URL;
4029
import java.nio.charset.StandardCharsets;
@@ -43,8 +32,18 @@
4332
import java.util.Map;
4433
import java.util.Objects;
4534

46-
import javax.annotation.Nonnull;
47-
import javax.annotation.Nullable;
35+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
36+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
37+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addYouTubeHeaders;
38+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
39+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
40+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
41+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
42+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
43+
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
44+
import static org.schabi.newpipe.extractor.utils.Utils.getQueryValue;
45+
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
46+
import static org.schabi.newpipe.extractor.utils.Utils.stringToURL;
4847

4948
/**
5049
* A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist).
@@ -89,16 +88,26 @@ public void onFetchPage(@Nonnull final Downloader downloader)
8988
final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(StandardCharsets.UTF_8);
9089

9190
final Map<String, List<String>> headers = new HashMap<>();
92-
addClientInfoHeaders(headers);
91+
// Cookie is required due to consent
92+
addYouTubeHeaders(headers);
9393

9494
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey()
9595
+ DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
9696

9797
initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
98-
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
99-
.getObject("playlist").getObject("playlist");
98+
playlistData = initialData
99+
.getObject("contents")
100+
.getObject("twoColumnWatchNextResults")
101+
.getObject("playlist")
102+
.getObject("playlist");
100103
if (isNullOrEmpty(playlistData)) {
101-
throw new ExtractionException("Could not get playlistData");
104+
final ExtractionException ex = new ExtractionException("Could not get playlistData");
105+
if (!YoutubeParsingHelper.isConsentAccepted()) {
106+
throw new ConsentRequiredException(
107+
"Consent is required in some countries to view Mix playlists",
108+
ex);
109+
}
110+
throw ex;
102111
}
103112
cookieValue = extractCookieValue(COOKIE_NAME, response);
104113
}
@@ -212,7 +221,8 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
212221

213222
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
214223
final Map<String, List<String>> headers = new HashMap<>();
215-
addClientInfoHeaders(headers);
224+
// Cookie is required due to consent
225+
addYouTubeHeaders(headers);
216226

217227
final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(),
218228
getExtractorLocalization());

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,6 @@
11
package org.schabi.newpipe.extractor.services.youtube;
22

3-
import static org.junit.jupiter.api.Assertions.assertEquals;
4-
import static org.junit.jupiter.api.Assertions.assertFalse;
5-
import static org.junit.jupiter.api.Assertions.assertThrows;
6-
import static org.junit.jupiter.api.Assertions.assertTrue;
7-
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
8-
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
9-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
10-
113
import com.grack.nanojson.JsonWriter;
12-
134
import org.junit.jupiter.api.BeforeAll;
145
import org.junit.jupiter.api.Test;
156
import org.schabi.newpipe.downloader.DownloaderFactory;
@@ -31,6 +22,17 @@
3122
import java.util.Map;
3223
import java.util.Set;
3324

25+
import static org.junit.jupiter.api.Assertions.assertEquals;
26+
import static org.junit.jupiter.api.Assertions.assertFalse;
27+
import static org.junit.jupiter.api.Assertions.assertThrows;
28+
import static org.junit.jupiter.api.Assertions.assertTrue;
29+
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
30+
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
31+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
32+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
33+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
34+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
35+
3436
public class YoutubeMixPlaylistExtractorTest {
3537

3638
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/mix/";
@@ -45,6 +47,7 @@ public static class Mix {
4547
@BeforeAll
4648
public static void setUp() throws Exception {
4749
YoutubeTestsUtils.ensureStateless();
50+
YoutubeParsingHelper.setConsentAccepted(true);
4851
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "mix"));
4952
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
5053
extractor = (YoutubeMixPlaylistExtractor) YouTube
@@ -140,6 +143,7 @@ public static class MixWithIndex {
140143
@BeforeAll
141144
public static void setUp() throws Exception {
142145
YoutubeTestsUtils.ensureStateless();
146+
YoutubeParsingHelper.setConsentAccepted(true);
143147
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "mixWithIndex"));
144148
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
145149
extractor = (YoutubeMixPlaylistExtractor) YouTube
@@ -221,11 +225,12 @@ void getPlaylistType() throws ParsingException {
221225
}
222226

223227
public static class MyMix {
224-
private static final String VIDEO_ID = "_AzeUSL9lZc";
228+
private static final String VIDEO_ID = "YVkUvmDQ3HY";
225229

226230
@BeforeAll
227231
public static void setUp() throws Exception {
228232
YoutubeTestsUtils.ensureStateless();
233+
YoutubeParsingHelper.setConsentAccepted(true);
229234
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "myMix"));
230235
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
231236
extractor = (YoutubeMixPlaylistExtractor) YouTube
@@ -249,7 +254,7 @@ void getName() throws Exception {
249254
void getThumbnailUrl() throws Exception {
250255
final String thumbnailUrl = extractor.getThumbnailUrl();
251256
assertIsSecureUrl(thumbnailUrl);
252-
assertTrue(thumbnailUrl.startsWith("https://i.ytimg.com/vi/_AzeUSL9lZc"));
257+
assertTrue(thumbnailUrl.startsWith("https://i.ytimg.com/vi/" + VIDEO_ID));
253258
}
254259

255260
@Test
@@ -316,6 +321,7 @@ public static class Invalid {
316321
@BeforeAll
317322
public static void setUp() throws IOException {
318323
YoutubeTestsUtils.ensureStateless();
324+
YoutubeParsingHelper.setConsentAccepted(true);
319325
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "invalid"));
320326
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
321327
}
@@ -350,6 +356,7 @@ public static class ChannelMix {
350356
@BeforeAll
351357
public static void setUp() throws Exception {
352358
YoutubeTestsUtils.ensureStateless();
359+
YoutubeParsingHelper.setConsentAccepted(true);
353360
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "channelMix"));
354361
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
355362
extractor = (YoutubeMixPlaylistExtractor) YouTube
@@ -414,6 +421,7 @@ public static class GenreMix {
414421
@BeforeAll
415422
public static void setUp() throws Exception {
416423
YoutubeTestsUtils.ensureStateless();
424+
YoutubeParsingHelper.setConsentAccepted(true);
417425
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "genreMix"));
418426
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
419427
extractor = (YoutubeMixPlaylistExtractor) YouTube

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ private YoutubeTestsUtils() {
2121
* </p>
2222
*/
2323
public static void ensureStateless() {
24+
YoutubeParsingHelper.setConsentAccepted(false);
2425
YoutubeParsingHelper.resetClientVersionAndKey();
2526
YoutubeParsingHelper.setNumberGenerator(new Random(1));
2627
YoutubeStreamExtractor.resetDeobfuscationCode();

0 commit comments

Comments
 (0)