Skip to content

Commit e3317ac

Browse files
Isira-SeneviratneIsira-Seneviratne
committed
Merge branch 'dev' into Refactor-Optional
# Conflicts: # extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java # extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
2 parents 9c8cab6 + 59b620c commit e3317ac

File tree

17 files changed

+3231
-175
lines changed

17 files changed

+3231
-175
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,10 +59,11 @@ jobs:
5959
./gradlew check \
6060
-Dorg.gradle.welcome=never \
6161
--stacktrace \
62-
-Ddownloader=$downloader_type
62+
-Ddownloader=$downloader_type \
63+
-Dcom.google.protobuf.error_on_unsafe_pre22_gencode
6364
6465
- name: Upload test reports when failure occurs
65-
uses: actions/upload-artifact@v4
66+
uses: actions/upload-artifact@v5
6667
if: failure()
6768
with:
6869
name: NewPipeExtractor-test-reports

‎build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ allprojects {
2929
ext {
3030
nanojsonVersion = "e9d656ddb49a412a5a0a5d5ef20ca7ef09549996"
3131
jsr305Version = "3.0.2"
32-
junitVersion = "5.14.0"
32+
junitVersion = "5.14.1"
3333
checkstyleVersion = "10.26.1"
3434
}
3535
}

‎extractor/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ checkstyleTest {
2828

2929
ext {
3030
rhinoVersion = '1.8.0'
31-
protobufVersion = '4.32.1'
31+
protobufVersion = '4.33.0'
3232
}
3333

3434
dependencies {

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 60 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -20,31 +20,12 @@
2020

2121
package org.schabi.newpipe.extractor.services.youtube;
2222

23-
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
24-
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION;
25-
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM;
26-
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION;
27-
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL;
28-
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_USER_AGENT_VERSION;
29-
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_USER_AGENT;
30-
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID;
31-
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME;
32-
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_HARDCODED_CLIENT_VERSION;
33-
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_CLIENT_ID;
34-
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_CLIENT_NAME;
35-
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_HARDCODED_CLIENT_VERSION;
36-
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
37-
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
38-
import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray;
39-
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
40-
4123
import com.grack.nanojson.JsonArray;
4224
import com.grack.nanojson.JsonBuilder;
4325
import com.grack.nanojson.JsonObject;
4426
import com.grack.nanojson.JsonParser;
4527
import com.grack.nanojson.JsonParserException;
4628
import com.grack.nanojson.JsonWriter;
47-
4829
import org.jsoup.nodes.Entities;
4930
import org.schabi.newpipe.extractor.Image;
5031
import org.schabi.newpipe.extractor.Image.ResolutionLevel;
@@ -63,6 +44,8 @@
6344
import org.schabi.newpipe.extractor.utils.RandomStringFromAlphabetGenerator;
6445
import org.schabi.newpipe.extractor.utils.Utils;
6546

47+
import javax.annotation.Nonnull;
48+
import javax.annotation.Nullable;
6649
import java.io.IOException;
6750
import java.net.MalformedURLException;
6851
import java.net.URL;
@@ -79,8 +62,23 @@
7962
import java.util.stream.Collectors;
8063
import java.util.stream.Stream;
8164

82-
import javax.annotation.Nonnull;
83-
import javax.annotation.Nullable;
65+
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
66+
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION;
67+
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM;
68+
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION;
69+
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL;
70+
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_USER_AGENT_VERSION;
71+
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_USER_AGENT;
72+
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID;
73+
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME;
74+
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_HARDCODED_CLIENT_VERSION;
75+
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_CLIENT_ID;
76+
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_CLIENT_NAME;
77+
import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_HARDCODED_CLIENT_VERSION;
78+
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
79+
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
80+
import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray;
81+
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
8482

8583
public final class YoutubeParsingHelper {
8684

@@ -198,7 +196,7 @@ private YoutubeParsingHelper() {
198196

199197
private static boolean consentAccepted = false;
200198

201-
private static final Predicate<String> STRING_PREDICATE = text -> !text.isEmpty();
199+
public static final Predicate<String> STRING_PREDICATE = text -> !text.isBlank();
202200

203201
public static boolean isGoogleURL(final String url) {
204202
final String cachedUrl = extractCachedUrlIfNeeded(url);
@@ -757,10 +755,18 @@ public static Optional<String> getUrlFromNavigationEndpoint(
757755
.map(id -> "https://www.youtube.com/playlist?list=" + id);
758756
})
759757
.or(() -> {
760-
final var metadata = navigationEndpoint.getObject("commandMetadata")
761-
.getObject("webCommandMetadata");
762-
return Optional.ofNullable(metadata.getString("url"))
763-
.map(url -> "https://www.youtube.com" + url);
758+
final var listItems = navigationEndpoint.getObject("showDialogCommand")
759+
.getObject("panelLoadingStrategy").getObject("inlineContent")
760+
.getObject("dialogViewModel").getObject("customContent")
761+
.getObject("listViewModel")
762+
.getArray("listItems");
763+
764+
// the first item seems to always be the channel that actually uploaded the
765+
// video, i.e. it appears in their video feed
766+
final var command = listItems.getObject(0).getObject("listItemViewModel")
767+
.getObject("rendererContext").getObject("commandContext")
768+
.getObject("onTap").getObject("innertubeCommand");
769+
return getUrlFromNavigationEndpoint(command);
764770
})
765771
.filter(STRING_PREDICATE);
766772
}
@@ -1248,36 +1254,25 @@ public static String extractCachedUrlIfNeeded(final String url) {
12481254
return url;
12491255
}
12501256

1251-
public static boolean isVerified(final JsonArray badges) {
1252-
if (Utils.isNullOrEmpty(badges)) {
1253-
return false;
1254-
}
1255-
1256-
for (final Object badge : badges) {
1257-
final String style = ((JsonObject) badge).getObject("metadataBadgeRenderer")
1258-
.getString("style");
1259-
if (style != null && (style.equals("BADGE_STYLE_TYPE_VERIFIED")
1260-
|| style.equals("BADGE_STYLE_TYPE_VERIFIED_ARTIST"))) {
1261-
return true;
1262-
}
1263-
}
1264-
1265-
return false;
1257+
public static boolean isVerified(@Nonnull final JsonArray badges) {
1258+
return badges.streamAsJsonObjects()
1259+
.anyMatch(badge -> {
1260+
final String style = badge.getObject("metadataBadgeRenderer")
1261+
.getString("style");
1262+
return "BADGE_STYLE_TYPE_VERIFIED".equals(style)
1263+
|| "BADGE_STYLE_TYPE_VERIFIED_ARTIST".equals(style);
1264+
});
12661265
}
12671266

12681267
public static boolean hasArtistOrVerifiedIconBadgeAttachment(
12691268
@Nonnull final JsonArray attachmentRuns) {
1270-
return attachmentRuns.stream()
1271-
.filter(JsonObject.class::isInstance)
1272-
.map(JsonObject.class::cast)
1269+
return attachmentRuns.streamAsJsonObjects()
12731270
.anyMatch(attachmentRun -> attachmentRun.getObject("element")
12741271
.getObject("type")
12751272
.getObject("imageType")
12761273
.getObject("image")
12771274
.getArray("sources")
1278-
.stream()
1279-
.filter(JsonObject.class::isInstance)
1280-
.map(JsonObject.class::cast)
1275+
.streamAsJsonObjects()
12811276
.anyMatch(source -> {
12821277
final String imageName = source.getObject("clientResource")
12831278
.getString("imageName");
@@ -1541,4 +1536,22 @@ public static JsonBuilder<JsonObject> prepareJsonBuilder(
15411536

15421537
return builder;
15431538
}
1539+
1540+
/**
1541+
* Gets the first collaborator, which is the channel that owns the video,
1542+
* i.e. the video is displayed on their channel page.
1543+
*
1544+
* @param renderer JSON object for the video renderer
1545+
* @return An {@link Optional} containing the first collaborator, if one is present
1546+
*/
1547+
@Nonnull
1548+
public static Optional<JsonObject> getFirstCollaborator(final JsonObject renderer) {
1549+
final JsonArray listItems = renderer.getObject("navigationEndpoint")
1550+
.getObject("showDialogCommand").getObject("panelLoadingStrategy")
1551+
.getObject("inlineContent").getObject("dialogViewModel")
1552+
.getObject("customContent").getObject("listViewModel")
1553+
.getArray("listItems");
1554+
return Optional.ofNullable(listItems.getObject(0)
1555+
.getObject("listItemViewModel", null));
1556+
}
15441557
}

0 commit comments

Comments
 (0)