Skip to content

Commit d6577e5

Browse files
litetexlitetex
authored
Merge pull request TeamNewPipe#882 from litetex/fix-all-tests
Fix all tests
2 parents 0a5ad90 + 0beb55a commit d6577e5

File tree

64 files changed

+8214
-1175
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

64 files changed

+8214
-1175
lines changed

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 37 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
2828
import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray;
2929
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
30-
3130
import static java.util.Collections.singletonList;
3231

3332
import com.grack.nanojson.JsonArray;
@@ -234,36 +233,31 @@ private YoutubeParsingHelper() {
234233

235234
private static Random numberGenerator = new SecureRandom();
236235

236+
private static final String FEED_BASE_CHANNEL_ID =
237+
"https://www.youtube.com/feeds/videos.xml?channel_id=";
238+
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
239+
private static final Pattern C_WEB_PATTERN = Pattern.compile("&c=WEB");
240+
private static final Pattern C_TVHTML5_SIMPLY_EMBEDDED_PLAYER_PATTERN =
241+
Pattern.compile("&c=TVHTML5_SIMPLY_EMBEDDED_PLAYER");
242+
private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID");
243+
private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS");
244+
237245
/**
238-
* {@code PENDING+} means that the user did not yet submit their choices.
246+
* Determines how the consent cookie (that is required for YouTube) will be generated.
239247
*
240248
* <p>
241-
* Therefore, YouTube & Google should not track the user, because they did not give consent.
249+
* {@code false} (default) will use {@code PENDING+}.
250+
* {@code true} will use {@code YES+}.
242251
* </p>
243252
*
244253
* <p>
245-
* The three digits at the end can be random, but are required.
254+
* Setting this value to <code>true</code> is currently needed if you want to watch
255+
* Mix Playlists in some countries (EU).
246256
* </p>
247-
*/
248-
private static final String CONSENT_COOKIE_VALUE = "PENDING+";
249-
250-
/**
251-
* YouTube {@code CONSENT} cookie.
252257
*
253-
* <p>
254-
* Should prevent redirect to {@code consent.youtube.com}.
255-
* </p>
258+
* @see #generateConsentCookie()
256259
*/
257-
private static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE;
258-
259-
private static final String FEED_BASE_CHANNEL_ID =
260-
"https://www.youtube.com/feeds/videos.xml?channel_id=";
261-
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
262-
private static final Pattern C_WEB_PATTERN = Pattern.compile("&c=WEB");
263-
private static final Pattern C_TVHTML5_SIMPLY_EMBEDDED_PLAYER_PATTERN =
264-
Pattern.compile("&c=TVHTML5_SIMPLY_EMBEDDED_PLAYER");
265-
private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID");
266-
private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS");
260+
private static boolean consentAccepted = false;
267261

268262
private static boolean isGoogleURL(final String url) {
269263
final String cachedUrl = extractCachedUrlIfNeeded(url);
@@ -1378,7 +1372,6 @@ public static Map<String, List<String>> getCookieHeader() {
13781372

13791373
/**
13801374
* Add the <code>CONSENT</code> cookie to prevent redirect to <code>consent.youtube.com</code>
1381-
* @see #CONSENT_COOKIE
13821375
* @param headers the headers which should be completed
13831376
*/
13841377
public static void addCookieHeader(@Nonnull final Map<String, List<String>> headers) {
@@ -1391,8 +1384,13 @@ public static void addCookieHeader(@Nonnull final Map<String, List<String>> head
13911384

13921385
@Nonnull
13931386
public static String generateConsentCookie() {
1394-
final int statusCode = 100 + numberGenerator.nextInt(900);
1395-
return CONSENT_COOKIE + statusCode;
1387+
return "CONSENT=" + (isConsentAccepted()
1388+
// YES+ means that the user did submit their choices and allows tracking.
1389+
? "YES+"
1390+
// PENDING+ means that the user did not yet submit their choices.
1391+
// YT & Google should not track the user, because they did not give consent.
1392+
// The three digits at the end can be random, but are required.
1393+
: "PENDING+" + (100 + numberGenerator.nextInt(900)));
13961394
}
13971395

13981396
public static String extractCookieValue(final String cookieName,
@@ -1612,16 +1610,6 @@ public static boolean isVerified(final JsonArray badges) {
16121610
return false;
16131611
}
16141612

1615-
@Nonnull
1616-
public static String unescapeDocument(@Nonnull final String doc) {
1617-
return doc
1618-
.replaceAll("\\\\x22", "\"")
1619-
.replaceAll("\\\\x7b", "{")
1620-
.replaceAll("\\\\x7d", "}")
1621-
.replaceAll("\\\\x5b", "[")
1622-
.replaceAll("\\\\x5d", "]");
1623-
}
1624-
16251613
/**
16261614
* Generate a content playback nonce (also called {@code cpn}), sent by YouTube clients in
16271615
* playback requests (and also for some clients, in the player request body).
@@ -1692,4 +1680,18 @@ public static boolean isAndroidStreamingUrl(@Nonnull final String url) {
16921680
public static boolean isIosStreamingUrl(@Nonnull final String url) {
16931681
return Parser.isMatch(C_IOS_PATTERN, url);
16941682
}
1683+
1684+
/**
1685+
* @see #consentAccepted
1686+
*/
1687+
public static void setConsentAccepted(final boolean accepted) {
1688+
consentAccepted = accepted;
1689+
}
1690+
1691+
/**
1692+
* @see #consentAccepted
1693+
*/
1694+
public static boolean isConsentAccepted() {
1695+
return consentAccepted;
1696+
}
16951697
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,13 @@
55
import org.schabi.newpipe.extractor.utils.Parser;
66
import org.schabi.newpipe.extractor.utils.StringUtils;
77

8-
import javax.annotation.Nonnull;
98
import java.util.HashMap;
109
import java.util.Map;
1110
import java.util.regex.Matcher;
1211
import java.util.regex.Pattern;
1312

13+
import javax.annotation.Nonnull;
14+
1415
/**
1516
* YouTube's streaming URLs of HTML5 clients are protected with a cipher, which modifies their
1617
* {@code n} query parameter.
@@ -128,16 +129,25 @@ private static String parseDecodeFunction(final String playerJsCode, final Strin
128129
private static String parseWithParenthesisMatching(final String playerJsCode,
129130
final String functionName) {
130131
final String functionBase = functionName + "=function";
131-
return functionBase + StringUtils.matchToClosingParenthesis(playerJsCode, functionBase)
132-
+ ";";
132+
return validateFunction(functionBase
133+
+ StringUtils.matchToClosingParenthesis(playerJsCode, functionBase)
134+
+ ";");
133135
}
134136

135137
@Nonnull
136138
private static String parseWithRegex(final String playerJsCode, final String functionName)
137139
throws Parser.RegexException {
138-
final Pattern functionPattern = Pattern.compile(functionName + "=function(.*?}};)\n",
140+
final Pattern functionPattern = Pattern.compile(functionName + "=function(.*?};)\n",
139141
Pattern.DOTALL);
140-
return "function " + functionName + Parser.matchGroup1(functionPattern, playerJsCode);
142+
return validateFunction("function "
143+
+ functionName
144+
+ Parser.matchGroup1(functionPattern, playerJsCode));
145+
}
146+
147+
@Nonnull
148+
private static String validateFunction(@Nonnull final String function) {
149+
JavaScript.compileOrThrow(function);
150+
return function;
141151
}
142152

143153
private static boolean containsNParam(final String url) {

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
44
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
5-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
5+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addYouTubeHeaders;
66
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
77
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
88
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
@@ -23,6 +23,7 @@
2323
import org.schabi.newpipe.extractor.StreamingService;
2424
import org.schabi.newpipe.extractor.downloader.Downloader;
2525
import org.schabi.newpipe.extractor.downloader.Response;
26+
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
2627
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
2728
import org.schabi.newpipe.extractor.exceptions.ParsingException;
2829
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@@ -89,16 +90,26 @@ public void onFetchPage(@Nonnull final Downloader downloader)
8990
final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(StandardCharsets.UTF_8);
9091

9192
final Map<String, List<String>> headers = new HashMap<>();
92-
addClientInfoHeaders(headers);
93+
// Cookie is required due to consent
94+
addYouTubeHeaders(headers);
9395

9496
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey()
9597
+ DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
9698

9799
initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
98-
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
99-
.getObject("playlist").getObject("playlist");
100+
playlistData = initialData
101+
.getObject("contents")
102+
.getObject("twoColumnWatchNextResults")
103+
.getObject("playlist")
104+
.getObject("playlist");
100105
if (isNullOrEmpty(playlistData)) {
101-
throw new ExtractionException("Could not get playlistData");
106+
final ExtractionException ex = new ExtractionException("Could not get playlistData");
107+
if (!YoutubeParsingHelper.isConsentAccepted()) {
108+
throw new ContentNotAvailableException(
109+
"Consent is required in some countries to view Mix playlists",
110+
ex);
111+
}
112+
throw ex;
102113
}
103114
cookieValue = extractCookieValue(COOKIE_NAME, response);
104115
}
@@ -212,7 +223,8 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
212223

213224
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
214225
final Map<String, List<String>> headers = new HashMap<>();
215-
addClientInfoHeaders(headers);
226+
// Cookie is required due to consent
227+
addYouTubeHeaders(headers);
216228

217229
final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(),
218230
getExtractorLocalization());

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java‎

Lines changed: 37 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
import org.schabi.newpipe.extractor.InfoItem;
2424
import org.schabi.newpipe.extractor.MetaInfo;
25+
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
2526
import org.schabi.newpipe.extractor.Page;
2627
import org.schabi.newpipe.extractor.StreamingService;
2728
import org.schabi.newpipe.extractor.downloader.Downloader;
@@ -31,7 +32,6 @@
3132
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
3233
import org.schabi.newpipe.extractor.localization.DateWrapper;
3334
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
34-
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
3535
import org.schabi.newpipe.extractor.search.SearchExtractor;
3636
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
3737
import org.schabi.newpipe.extractor.utils.JsonUtils;
@@ -43,6 +43,7 @@
4343
import java.util.HashMap;
4444
import java.util.List;
4545
import java.util.Map;
46+
import java.util.stream.Collectors;
4647

4748
import javax.annotation.Nonnull;
4849
import javax.annotation.Nullable;
@@ -133,55 +134,52 @@ public void onFetchPage(@Nonnull final Downloader downloader)
133134
}
134135
}
135136

136-
@Nonnull
137-
@Override
138-
public String getUrl() throws ParsingException {
139-
return super.getUrl();
137+
private List<JsonObject> getItemSectionRendererContents() {
138+
return initialData
139+
.getObject("contents")
140+
.getObject("tabbedSearchResultsRenderer")
141+
.getArray("tabs")
142+
.getObject(0)
143+
.getObject("tabRenderer")
144+
.getObject("content")
145+
.getObject("sectionListRenderer")
146+
.getArray("contents")
147+
.stream()
148+
.filter(JsonObject.class::isInstance)
149+
.map(JsonObject.class::cast)
150+
.map(c -> c.getObject("itemSectionRenderer"))
151+
.filter(isr -> !isr.isEmpty())
152+
.map(isr -> isr
153+
.getArray("contents")
154+
.getObject(0))
155+
.collect(Collectors.toList());
140156
}
141157

142158
@Nonnull
143159
@Override
144160
public String getSearchSuggestion() throws ParsingException {
145-
final JsonObject itemSectionRenderer = JsonUtils.getArray(JsonUtils.getArray(initialData,
146-
"contents.tabbedSearchResultsRenderer.tabs").getObject(0),
147-
"tabRenderer.content.sectionListRenderer.contents")
148-
.getObject(0)
149-
.getObject("itemSectionRenderer");
150-
if (itemSectionRenderer.isEmpty()) {
151-
return "";
161+
for (final JsonObject obj : getItemSectionRendererContents()) {
162+
final JsonObject didYouMeanRenderer = obj
163+
.getObject("didYouMeanRenderer");
164+
final JsonObject showingResultsForRenderer = obj
165+
.getObject("showingResultsForRenderer");
166+
167+
if (!didYouMeanRenderer.isEmpty()) {
168+
return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery"));
169+
} else if (!showingResultsForRenderer.isEmpty()) {
170+
return JsonUtils.getString(showingResultsForRenderer,
171+
"correctedQueryEndpoint.searchEndpoint.query");
172+
}
152173
}
153174

154-
final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents")
155-
.getObject(0).getObject("didYouMeanRenderer");
156-
final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents")
157-
.getObject(0)
158-
.getObject("showingResultsForRenderer");
159-
160-
if (!didYouMeanRenderer.isEmpty()) {
161-
return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery"));
162-
} else if (!showingResultsForRenderer.isEmpty()) {
163-
return JsonUtils.getString(showingResultsForRenderer,
164-
"correctedQueryEndpoint.searchEndpoint.query");
165-
} else {
166-
return "";
167-
}
175+
return "";
168176
}
169177

170178
@Override
171179
public boolean isCorrectedSearch() throws ParsingException {
172-
final JsonObject itemSectionRenderer = JsonUtils.getArray(JsonUtils.getArray(initialData,
173-
"contents.tabbedSearchResultsRenderer.tabs").getObject(0),
174-
"tabRenderer.content.sectionListRenderer.contents")
175-
.getObject(0)
176-
.getObject("itemSectionRenderer");
177-
if (itemSectionRenderer.isEmpty()) {
178-
return false;
179-
}
180-
181-
final JsonObject firstContent = itemSectionRenderer.getArray("contents").getObject(0);
182-
183-
return firstContent.has("didYouMeanRenderer")
184-
|| firstContent.has("showingResultsForRenderer");
180+
return getItemSectionRendererContents()
181+
.stream()
182+
.anyMatch(obj -> obj.has("showingResultsForRenderer"));
185183
}
186184

187185
@Nonnull

‎extractor/src/main/java/org/schabi/newpipe/extractor/utils/JavaScript.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,18 @@ public final class JavaScript {
99
private JavaScript() {
1010
}
1111

12+
public static void compileOrThrow(final String function) {
13+
try {
14+
final Context context = Context.enter();
15+
context.setOptimizationLevel(-1);
16+
17+
// If it doesn't compile it throws an exception here
18+
context.compileString(function, null, 1, null);
19+
} finally {
20+
Context.exit();
21+
}
22+
}
23+
1224
public static String run(final String function,
1325
final String functionName,
1426
final String... parameters) {

0 commit comments

Comments
 (0)