Skip to content

Commit d26efa1

Browse files
litetexStypox
litetex
authored and
Stypox
committed
Fixed all YTMixPlaylists
Added option to choose if you want to consent or not - currently this is done by a static variable in ``YoutubeParsingHelper`` - may not be the best long-term solution but for now the tests work again (in EU countries) 🥳
1 parent 689f03f commit d26efa1

File tree

5 files changed

+89
-51
lines changed

5 files changed

+89
-51
lines changed

‎extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ConsentRequiredException.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package org.schabi.newpipe.extractor.exceptions;
2+
3+
public class ConsentRequiredException extends ParsingException {
4+
5+
public ConsentRequiredException(final String message) {
6+
super(message);
7+
}
8+
9+
public ConsentRequiredException(final String message, final Throwable cause) {
10+
super(message, cause);
11+
}
12+
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 26 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
2828
import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray;
2929
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
30-
3130
import static java.util.Collections.singletonList;
3231

3332
import com.grack.nanojson.JsonArray;
@@ -234,16 +233,19 @@ private YoutubeParsingHelper() {
234233
* The three digits at the end can be random, but are required.
235234
* </p>
236235
*/
237-
private static final String CONSENT_COOKIE_VALUE = "PENDING+";
238-
236+
private static final String CONSENT_COOKIE_PENDING_VALUE = "PENDING+";
239237
/**
240-
* YouTube {@code CONSENT} cookie.
238+
* {@code YES+} means that the user did submit their choices and accepted all cookies.
241239
*
242240
* <p>
243-
* Should prevent redirect to {@code consent.youtube.com}.
241+
* Therefore, YouTube & Google can track the user, because they did give consent.
242+
* </p>
243+
*
244+
* <p>
245+
* The three digits at the end can be random, but are required.
244246
* </p>
245247
*/
246-
private static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE;
248+
private static final String CONSENT_COOKIE_YES_VALUE = "YES+";
247249

248250
private static final String FEED_BASE_CHANNEL_ID =
249251
"https://www.youtube.com/feeds/videos.xml?channel_id=";
@@ -254,6 +256,13 @@ private YoutubeParsingHelper() {
254256
private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID");
255257
private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS");
256258

259+
/**
260+
* {@code false} (default) will use {@link #CONSENT_COOKIE_PENDING_VALUE}.
261+
* <br/>
262+
* {@code true} will use {@link #CONSENT_COOKIE_YES_VALUE}.
263+
*/
264+
private static boolean consentAccepted = false;
265+
257266
private static boolean isGoogleURL(final String url) {
258267
final String cachedUrl = extractCachedUrlIfNeeded(url);
259268
try {
@@ -1321,7 +1330,6 @@ public static Map<String, List<String>> getCookieHeader() {
13211330

13221331
/**
13231332
* Add the <code>CONSENT</code> cookie to prevent redirect to <code>consent.youtube.com</code>
1324-
* @see #CONSENT_COOKIE
13251333
* @param headers the headers which should be completed
13261334
*/
13271335
public static void addCookieHeader(@Nonnull final Map<String, List<String>> headers) {
@@ -1334,8 +1342,9 @@ public static void addCookieHeader(@Nonnull final Map<String, List<String>> head
13341342

13351343
@Nonnull
13361344
public static String generateConsentCookie() {
1337-
final int statusCode = 100 + numberGenerator.nextInt(900);
1338-
return CONSENT_COOKIE + statusCode;
1345+
return "CONSENT="
1346+
+ (isConsentAccepted() ? CONSENT_COOKIE_YES_VALUE : CONSENT_COOKIE_PENDING_VALUE)
1347+
+ (100 + numberGenerator.nextInt(900));
13391348
}
13401349

13411350
public static String extractCookieValue(final String cookieName,
@@ -1555,16 +1564,6 @@ public static boolean isVerified(final JsonArray badges) {
15551564
return false;
15561565
}
15571566

1558-
@Nonnull
1559-
public static String unescapeDocument(@Nonnull final String doc) {
1560-
return doc
1561-
.replaceAll("\\\\x22", "\"")
1562-
.replaceAll("\\\\x7b", "{")
1563-
.replaceAll("\\\\x7d", "}")
1564-
.replaceAll("\\\\x5b", "[")
1565-
.replaceAll("\\\\x5d", "]");
1566-
}
1567-
15681567
/**
15691568
* Generate a content playback nonce (also called {@code cpn}), sent by YouTube clients in
15701569
* playback requests (and also for some clients, in the player request body).
@@ -1635,4 +1634,12 @@ public static boolean isAndroidStreamingUrl(@Nonnull final String url) {
16351634
public static boolean isIosStreamingUrl(@Nonnull final String url) {
16361635
return Parser.isMatch(C_IOS_PATTERN, url);
16371636
}
1637+
1638+
public static void setConsentAccepted(final boolean accepted) {
1639+
consentAccepted = accepted;
1640+
}
1641+
1642+
public static boolean isConsentAccepted() {
1643+
return consentAccepted;
1644+
}
16381645
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java‎

Lines changed: 31 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,15 @@
11
package org.schabi.newpipe.extractor.services.youtube.extractors;
22

3-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
4-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
5-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
6-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
7-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
8-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
9-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
10-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
11-
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
12-
import static org.schabi.newpipe.extractor.utils.Utils.getQueryValue;
13-
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
14-
import static org.schabi.newpipe.extractor.utils.Utils.stringToURL;
15-
163
import com.grack.nanojson.JsonArray;
174
import com.grack.nanojson.JsonBuilder;
185
import com.grack.nanojson.JsonObject;
196
import com.grack.nanojson.JsonWriter;
20-
217
import org.schabi.newpipe.extractor.ListExtractor;
228
import org.schabi.newpipe.extractor.Page;
239
import org.schabi.newpipe.extractor.StreamingService;
2410
import org.schabi.newpipe.extractor.downloader.Downloader;
2511
import org.schabi.newpipe.extractor.downloader.Response;
12+
import org.schabi.newpipe.extractor.exceptions.ConsentRequiredException;
2613
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
2714
import org.schabi.newpipe.extractor.exceptions.ParsingException;
2815
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@@ -35,6 +22,8 @@
3522
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
3623
import org.schabi.newpipe.extractor.utils.JsonUtils;
3724

25+
import javax.annotation.Nonnull;
26+
import javax.annotation.Nullable;
3827
import java.io.IOException;
3928
import java.net.URL;
4029
import java.nio.charset.StandardCharsets;
@@ -43,8 +32,18 @@
4332
import java.util.Map;
4433
import java.util.Objects;
4534

46-
import javax.annotation.Nonnull;
47-
import javax.annotation.Nullable;
35+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
36+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
37+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addYouTubeHeaders;
38+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
39+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
40+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
41+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
42+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
43+
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
44+
import static org.schabi.newpipe.extractor.utils.Utils.getQueryValue;
45+
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
46+
import static org.schabi.newpipe.extractor.utils.Utils.stringToURL;
4847

4948
/**
5049
* A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist).
@@ -89,16 +88,26 @@ public void onFetchPage(@Nonnull final Downloader downloader)
8988
final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(StandardCharsets.UTF_8);
9089

9190
final Map<String, List<String>> headers = new HashMap<>();
92-
addClientInfoHeaders(headers);
91+
// Cookie is required due to consent
92+
addYouTubeHeaders(headers);
9393

9494
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey()
9595
+ DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
9696

9797
initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
98-
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
99-
.getObject("playlist").getObject("playlist");
98+
playlistData = initialData
99+
.getObject("contents")
100+
.getObject("twoColumnWatchNextResults")
101+
.getObject("playlist")
102+
.getObject("playlist");
100103
if (isNullOrEmpty(playlistData)) {
101-
throw new ExtractionException("Could not get playlistData");
104+
final ExtractionException ex = new ExtractionException("Could not get playlistData");
105+
if (!YoutubeParsingHelper.isConsentAccepted()) {
106+
throw new ConsentRequiredException(
107+
"Consent is required in some countries to view Mix playlists",
108+
ex);
109+
}
110+
throw ex;
102111
}
103112
cookieValue = extractCookieValue(COOKIE_NAME, response);
104113
}
@@ -212,7 +221,8 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
212221

213222
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
214223
final Map<String, List<String>> headers = new HashMap<>();
215-
addClientInfoHeaders(headers);
224+
// Cookie is required due to consent
225+
addYouTubeHeaders(headers);
216226

217227
final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(),
218228
getExtractorLocalization());

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,6 @@
11
package org.schabi.newpipe.extractor.services.youtube;
22

3-
import static org.junit.jupiter.api.Assertions.assertEquals;
4-
import static org.junit.jupiter.api.Assertions.assertFalse;
5-
import static org.junit.jupiter.api.Assertions.assertThrows;
6-
import static org.junit.jupiter.api.Assertions.assertTrue;
7-
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
8-
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
9-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
10-
113
import com.grack.nanojson.JsonWriter;
12-
134
import org.junit.jupiter.api.BeforeAll;
145
import org.junit.jupiter.api.Test;
156
import org.schabi.newpipe.downloader.DownloaderFactory;
@@ -31,6 +22,17 @@
3122
import java.util.Map;
3223
import java.util.Set;
3324

25+
import static org.junit.jupiter.api.Assertions.assertEquals;
26+
import static org.junit.jupiter.api.Assertions.assertFalse;
27+
import static org.junit.jupiter.api.Assertions.assertThrows;
28+
import static org.junit.jupiter.api.Assertions.assertTrue;
29+
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
30+
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
31+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
32+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
33+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
34+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
35+
3436
public class YoutubeMixPlaylistExtractorTest {
3537

3638
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/mix/";
@@ -45,6 +47,7 @@ public static class Mix {
4547
@BeforeAll
4648
public static void setUp() throws Exception {
4749
YoutubeTestsUtils.ensureStateless();
50+
YoutubeParsingHelper.setConsentAccepted(true);
4851
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "mix"));
4952
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
5053
extractor = (YoutubeMixPlaylistExtractor) YouTube
@@ -140,6 +143,7 @@ public static class MixWithIndex {
140143
@BeforeAll
141144
public static void setUp() throws Exception {
142145
YoutubeTestsUtils.ensureStateless();
146+
YoutubeParsingHelper.setConsentAccepted(true);
143147
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "mixWithIndex"));
144148
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
145149
extractor = (YoutubeMixPlaylistExtractor) YouTube
@@ -221,11 +225,12 @@ void getPlaylistType() throws ParsingException {
221225
}
222226

223227
public static class MyMix {
224-
private static final String VIDEO_ID = "_AzeUSL9lZc";
228+
private static final String VIDEO_ID = "YVkUvmDQ3HY";
225229

226230
@BeforeAll
227231
public static void setUp() throws Exception {
228232
YoutubeTestsUtils.ensureStateless();
233+
YoutubeParsingHelper.setConsentAccepted(true);
229234
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "myMix"));
230235
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
231236
extractor = (YoutubeMixPlaylistExtractor) YouTube
@@ -249,7 +254,7 @@ void getName() throws Exception {
249254
void getThumbnailUrl() throws Exception {
250255
final String thumbnailUrl = extractor.getThumbnailUrl();
251256
assertIsSecureUrl(thumbnailUrl);
252-
assertTrue(thumbnailUrl.startsWith("https://i.ytimg.com/vi/_AzeUSL9lZc"));
257+
assertTrue(thumbnailUrl.startsWith("https://i.ytimg.com/vi/" + VIDEO_ID));
253258
}
254259

255260
@Test
@@ -316,6 +321,7 @@ public static class Invalid {
316321
@BeforeAll
317322
public static void setUp() throws IOException {
318323
YoutubeTestsUtils.ensureStateless();
324+
YoutubeParsingHelper.setConsentAccepted(true);
319325
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "invalid"));
320326
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
321327
}
@@ -350,6 +356,7 @@ public static class ChannelMix {
350356
@BeforeAll
351357
public static void setUp() throws Exception {
352358
YoutubeTestsUtils.ensureStateless();
359+
YoutubeParsingHelper.setConsentAccepted(true);
353360
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "channelMix"));
354361
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
355362
extractor = (YoutubeMixPlaylistExtractor) YouTube
@@ -414,6 +421,7 @@ public static class GenreMix {
414421
@BeforeAll
415422
public static void setUp() throws Exception {
416423
YoutubeTestsUtils.ensureStateless();
424+
YoutubeParsingHelper.setConsentAccepted(true);
417425
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "genreMix"));
418426
dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
419427
extractor = (YoutubeMixPlaylistExtractor) YouTube

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ private YoutubeTestsUtils() {
2121
* </p>
2222
*/
2323
public static void ensureStateless() {
24+
YoutubeParsingHelper.setConsentAccepted(false);
2425
YoutubeParsingHelper.resetClientVersionAndKey();
2526
YoutubeParsingHelper.setNumberGenerator(new Random(1));
2627
YoutubeStreamExtractor.resetDeobfuscationCode();

0 commit comments

Comments
 (0)