From 57bd1204e8eb48800d0477d5d9c6044aab4266df Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Fri, 10 Oct 2025 08:02:06 +0530 Subject: [PATCH 1/7] [YouTube] Refactor some extraction code using Optional --- .../youtube/YoutubeChannelHelper.java | 7 +- .../youtube/YoutubeMetaInfoHelper.java | 48 +++-- .../youtube/YoutubeParsingHelper.java | 168 +++++++----------- .../YoutubeBaseShowInfoItemExtractor.java | 6 +- .../extractors/YoutubeChannelExtractor.java | 5 +- .../YoutubeChannelInfoItemExtractor.java | 22 +-- .../extractors/YoutubeCommentsExtractor.java | 2 +- .../YoutubeCommentsInfoItemExtractor.java | 17 +- ...YoutubeMixOrPlaylistInfoItemExtractor.java | 17 +- .../YoutubeMixPlaylistExtractor.java | 7 +- ...MusicAlbumOrPlaylistInfoItemExtractor.java | 11 +- .../YoutubeMusicArtistInfoItemExtractor.java | 9 +- .../YoutubeMusicSearchExtractor.java | 3 +- ...tubeMusicSongOrVideoInfoItemExtractor.java | 9 +- .../extractors/YoutubePlaylistExtractor.java | 31 ++-- .../YoutubePlaylistInfoItemExtractor.java | 23 +-- .../YoutubeReelInfoItemExtractor.java | 19 +- .../extractors/YoutubeSearchExtractor.java | 13 +- .../YoutubeShowRendererInfoItemExtractor.java | 23 +-- .../extractors/YoutubeStreamExtractor.java | 150 +++++++--------- .../YoutubeStreamInfoItemExtractor.java | 115 +++++------- .../kiosk/YoutubeTrendingExtractor.java | 18 +- 22 files changed, 276 insertions(+), 447 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java index c3faf673f8..a66ffaeccc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java @@ -519,22 +519,23 @@ public static String getChannelName(@Nullable final ChannelHeader channelHeader, } return Optional.ofNullable(channelHeader) - .map(header -> { + .flatMap(header -> { final JsonObject channelJson = header.json; switch (header.headerType) { case PAGE: - return channelJson.getObject(CONTENT) + final String pageTitle = channelJson.getObject(CONTENT) .getObject(PAGE_HEADER_VIEW_MODEL) .getObject(TITLE) .getObject("dynamicTextViewModel") .getObject("text") .getString(CONTENT, channelJson.getString("pageTitle")); + return Optional.ofNullable(pageTitle); case CAROUSEL: case INTERACTIVE_TABBED: return getTextFromObject(channelJson.getObject(TITLE)); case C4_TABBED: default: - return channelJson.getString(TITLE); + return Optional.ofNullable(channelJson.getString(TITLE)); } }) // The channel name from a microformatDataRenderer may be different from the one diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMetaInfoHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMetaInfoHelper.java index 214a13c405..672c7b4cef 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMetaInfoHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMetaInfoHelper.java @@ -5,7 +5,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isGoogleURL; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; import com.grack.nanojson.JsonArray; @@ -67,14 +66,11 @@ public static List getMetaInfo(@Nonnull final JsonArray contents) private static MetaInfo getInfoPanelContent(@Nonnull final JsonObject infoPanelContentRenderer) throws ParsingException { final MetaInfo metaInfo = new MetaInfo(); - final StringBuilder sb = new StringBuilder(); - for (final Object paragraph : infoPanelContentRenderer.getArray("paragraphs")) { - if (sb.length() != 0) { - sb.append("
"); - } - sb.append(getTextFromObject((JsonObject) paragraph)); - } - metaInfo.setContent(new Description(sb.toString(), Description.HTML)); + final String description = infoPanelContentRenderer.getArray("paragraphs") + .streamAsJsonObjects() + .map(paragraph -> getTextFromObject(paragraph).orElse("")) + .collect(Collectors.joining("
")); + metaInfo.setContent(new Description(description, Description.HTML)); if (infoPanelContentRenderer.has("sourceEndpoint")) { final String metaInfoLinkUrl = getUrlFromNavigationEndpoint( infoPanelContentRenderer.getObject("sourceEndpoint")); @@ -86,10 +82,9 @@ private static MetaInfo getInfoPanelContent(@Nonnull final JsonObject infoPanelC } final String metaInfoLinkText = getTextFromObject( - infoPanelContentRenderer.getObject("inlineSource")); - if (isNullOrEmpty(metaInfoLinkText)) { - throw new ParsingException("Could not get metadata info link text."); - } + infoPanelContentRenderer.getObject("inlineSource")) + .orElseThrow(() -> + new ParsingException("Could not get metadata info link text.")); metaInfo.addUrlText(metaInfoLinkText); } @@ -101,13 +96,12 @@ private static MetaInfo getClarificationRenderer( @Nonnull final JsonObject clarificationRenderer) throws ParsingException { final MetaInfo metaInfo = new MetaInfo(); - final String title = getTextFromObject(clarificationRenderer - .getObject("contentTitle")); - final String text = getTextFromObject(clarificationRenderer - .getObject("text")); - if (title == null || text == null) { - throw new ParsingException("Could not extract clarification renderer content"); - } + final String title = getTextFromObject(clarificationRenderer.getObject("contentTitle")) + .orElseThrow(() -> + new ParsingException("Could not extract clarification renderer content")); + final String text = getTextFromObject(clarificationRenderer.getObject("text")) + .orElseThrow(() -> + new ParsingException("Could not extract clarification renderer content")); metaInfo.setTitle(title); metaInfo.setContent(new Description(text, Description.PLAIN_TEXT)); @@ -122,11 +116,9 @@ private static MetaInfo getClarificationRenderer( throw new ParsingException("Could not get metadata info URL", e); } - final String metaInfoLinkText = getTextFromObject( - actionButton.getObject("text")); - if (isNullOrEmpty(metaInfoLinkText)) { - throw new ParsingException("Could not get metadata info link text."); - } + final String metaInfoLinkText = getTextFromObject(actionButton.getObject("text")) + .orElseThrow(() -> + new ParsingException("Could not get metadata info link text.")); metaInfo.addUrlText(metaInfoLinkText); } @@ -138,9 +130,9 @@ private static MetaInfo getClarificationRenderer( if (url != null && !isGoogleURL(url)) { try { metaInfo.addUrl(new URL(url)); - final String description = getTextFromObject(clarificationRenderer - .getObject("secondarySource")); - metaInfo.addUrlText(description == null ? url : description); + final String urlText = getTextFromObject(clarificationRenderer + .getObject("secondarySource")).orElse(url); + metaInfo.addUrlText(urlText); } catch (final MalformedURLException e) { throw new ParsingException("Could not get metadata info secondary URL", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index df295f0d5c..ae949dd361 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -776,125 +776,80 @@ public static String getUrlFromNavigationEndpoint( * * @param textObject JSON object to get the text from * @param html whether to return HTML, by parsing the {@code navigationEndpoint} - * @return text in the JSON object or {@code null} + * @return text in the JSON object as an {@link Optional} */ - @Nullable - public static String getTextFromObject(final JsonObject textObject, final boolean html) { - if (isNullOrEmpty(textObject)) { - return null; - } - - if (textObject.has("simpleText")) { - return textObject.getString("simpleText"); - } - - final JsonArray runs = textObject.getArray("runs"); - if (runs.isEmpty()) { - return null; - } - - final StringBuilder textBuilder = new StringBuilder(); - for (final Object o : runs) { - final JsonObject run = (JsonObject) o; - String text = run.getString("text"); - - if (html) { - if (run.has("navigationEndpoint")) { - final String url = getUrlFromNavigationEndpoint( - run.getObject("navigationEndpoint")); - if (!isNullOrEmpty(url)) { - text = "" + Entities.escape(text) - + ""; + @Nonnull + public static Optional getTextFromObject(@Nonnull final JsonObject textObject, + final boolean html) { + return Optional.ofNullable(textObject.getString("simpleText")) + .or(() -> { + final var runs = textObject.getArray("runs"); + final String text = runs.streamAsJsonObjects() + .map(run -> { + String textString = run.getString("text"); + + if (html) { + final String url = getUrlFromNavigationEndpoint( + run.getObject("navigationEndpoint")); + if (!isNullOrEmpty(url)) { + textString = "" + + Entities.escape(textString) + ""; + } + + if (run.getBoolean("strikethrough")) { + textString = "" + textString + ""; + } + if (run.getBoolean("italics")) { + textString = "" + textString + ""; + } + if (run.getBoolean("bold")) { + textString = "" + textString + ""; + } + } + + return textString; + }) + .collect(Collectors.joining()); + + final String string; + if (html) { + string = text.replaceAll("\\n", "
") + .replaceAll(" {2}", "  "); + } else { + string = text; } - } - - final boolean bold = run.has("bold") - && run.getBoolean("bold"); - final boolean italic = run.has("italics") - && run.getBoolean("italics"); - final boolean strikethrough = run.has("strikethrough") - && run.getBoolean("strikethrough"); - - if (bold) { - textBuilder.append(""); - } - if (italic) { - textBuilder.append(""); - } - if (strikethrough) { - textBuilder.append(""); - } - - textBuilder.append(text); - - if (strikethrough) { - textBuilder.append(""); - } - if (italic) { - textBuilder.append(""); - } - if (bold) { - textBuilder.append(""); - } - } else { - textBuilder.append(text); - } - } - - String text = textBuilder.toString(); - - if (html) { - text = text.replaceAll("\\n", "
"); - text = text.replaceAll(" {2}", "  "); - } - - return text; + return Optional.of(string); + }) + .filter(text -> !text.isEmpty()); } @Nonnull public static String getTextFromObjectOrThrow(final JsonObject textObject, final String error) throws ParsingException { - final String result = getTextFromObject(textObject); - if (result == null) { - throw new ParsingException("Could not extract text: " + error); - } - return result; + return getTextFromObject(textObject) + .orElseThrow(() -> new ParsingException("Could not extract text: " + error)); } - @Nullable - public static String getTextFromObject(final JsonObject textObject) { + @Nonnull + public static Optional getTextFromObject(@Nonnull final JsonObject textObject) { return getTextFromObject(textObject, false); } - @Nullable - public static String getUrlFromObject(final JsonObject textObject) { - if (isNullOrEmpty(textObject)) { - return null; - } - - final JsonArray runs = textObject.getArray("runs"); - if (runs.isEmpty()) { - return null; - } - - for (final Object textPart : runs) { - final String url = getUrlFromNavigationEndpoint(((JsonObject) textPart) - .getObject("navigationEndpoint")); - if (!isNullOrEmpty(url)) { - return url; - } - } - - return null; + @Nonnull + public static Optional getUrlFromObject(@Nonnull final JsonObject textObject) { + return textObject.getArray("runs").streamAsJsonObjects() + .map(textPart -> getUrlFromNavigationEndpoint(textPart + .getObject("navigationEndpoint"))) + .filter(url -> !isNullOrEmpty(url)) + .findFirst(); } - @Nullable - public static String getTextAtKey(@Nonnull final JsonObject jsonObject, final String theKey) { - if (jsonObject.isString(theKey)) { - return jsonObject.getString(theKey); - } else { - return getTextFromObject(jsonObject.getObject(theKey)); - } + @Nonnull + public static Optional getTextAtKey(@Nonnull final JsonObject jsonObject, + final String theKey) { + return Optional.ofNullable(jsonObject.getString(theKey)) + .filter(text -> !text.isEmpty()) + .or(() -> getTextFromObject(jsonObject.getObject(theKey))); } public static String fixThumbnailUrl(@Nonnull final String thumbnailUrl) { @@ -1222,7 +1177,8 @@ public static void defaultAlertsCheck(@Nonnull final JsonObject initialData) final JsonArray alerts = initialData.getArray("alerts"); if (!isNullOrEmpty(alerts)) { final JsonObject alertRenderer = alerts.getObject(0).getObject("alertRenderer"); - final String alertText = getTextFromObject(alertRenderer.getObject("text")); + final String alertText = getTextFromObject(alertRenderer.getObject("text")) + .orElse(null); final String alertType = alertRenderer.getString("type", ""); if (alertType.equalsIgnoreCase("ERROR")) { if (alertText != null diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBaseShowInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBaseShowInfoItemExtractor.java index 67254302fc..a3bdb554e8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBaseShowInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBaseShowInfoItemExtractor.java @@ -49,10 +49,8 @@ public long getStreamCount() throws ParsingException { final String streamCountText = getTextFromObject( showRenderer.getObject("thumbnailOverlays") .getObject("thumbnailOverlayBottomPanelRenderer") - .getObject("text")); - if (streamCountText == null) { - throw new ParsingException("Could not get stream count"); - } + .getObject("text")) + .orElseThrow(() -> new ParsingException("Could not get stream count")); try { // The data returned could be a human/shortened number, but no show with more than 1000 diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 147345526a..92fe80428d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -257,7 +257,7 @@ public long getSubscriberCount() throws ParsingException { if (textObject != null) { try { - return Utils.mixedNumberWordToLong(getTextFromObject(textObject)); + return Utils.mixedNumberWordToLong(getTextFromObject(textObject).orElse("")); } catch (final NumberFormatException e) { throw new ParsingException("Could not get subscriber count", e); } @@ -321,7 +321,8 @@ public String getDescription() throws ParsingException { The description extracted is incomplete and the original one can be only accessed from the About tab */ - return getTextFromObject(channelHeader.json.getObject("description")); + return getTextFromObject(channelHeader.json.getObject("description")) + .orElse(null); } return jsonResponse.getObject(METADATA) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 38861f4e51..cc22b92275 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -47,14 +47,9 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor public YoutubeChannelInfoItemExtractor(final JsonObject channelInfoItem) { this.channelInfoItem = channelInfoItem; - - boolean wHandle = false; - final String subscriberCountText = getTextFromObject( - channelInfoItem.getObject("subscriberCountText")); - if (subscriberCountText != null) { - wHandle = subscriberCountText.startsWith("@"); - } - this.withHandle = wHandle; + this.withHandle = getTextFromObject(channelInfoItem.getObject("subscriberCountText")) + .map(text -> text.startsWith("@")) + .orElse(false); } @Nonnull @@ -70,7 +65,7 @@ public List getThumbnails() throws ParsingException { @Override public String getName() throws ParsingException { try { - return getTextFromObject(channelInfoItem.getObject("title")); + return getTextFromObject(channelInfoItem.getObject("title")).orElse(null); } catch (final Exception e) { throw new ParsingException("Could not get name", e); } @@ -97,14 +92,14 @@ public long getSubscriberCount() throws ParsingException { if (withHandle) { if (channelInfoItem.has("videoCountText")) { return Utils.mixedNumberWordToLong(getTextFromObject( - channelInfoItem.getObject("videoCountText"))); + channelInfoItem.getObject("videoCountText")).orElse("")); } else { return -1; } } return Utils.mixedNumberWordToLong(getTextFromObject( - channelInfoItem.getObject("subscriberCountText"))); + channelInfoItem.getObject("subscriberCountText")).orElse("")); } catch (final Exception e) { throw new ParsingException("Could not get subscriber count", e); } @@ -120,7 +115,7 @@ public long getStreamCount() throws ParsingException { } return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject( - channelInfoItem.getObject("videoCountText")))); + channelInfoItem.getObject("videoCountText")).orElse(""))); } catch (final Exception e) { throw new ParsingException("Could not get stream count", e); } @@ -139,7 +134,8 @@ public String getDescription() throws ParsingException { return null; } - return getTextFromObject(channelInfoItem.getObject("descriptionSnippet")); + return getTextFromObject(channelInfoItem.getObject("descriptionSnippet")) + .orElse(null); } catch (final Exception e) { throw new ParsingException("Could not get description", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 8667768a4b..c7fbffce3e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -393,7 +393,7 @@ public int getCommentsCount() throws ExtractionException { try { return Integer.parseInt( - Utils.removeNonDigitCharacters(getTextFromObject(countText)) + Utils.removeNonDigitCharacters(getTextFromObject(countText).orElse("")) ); } catch (final Exception e) { throw new ExtractionException("Unable to get comments count", e); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java index ddc7b7bcc0..45371f9035 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java @@ -66,7 +66,8 @@ public List getThumbnails() throws ParsingException { @Override public String getName() throws ParsingException { try { - return getTextFromObject(JsonUtils.getObject(commentRenderer, "authorText")); + return getTextFromObject(JsonUtils.getObject(commentRenderer, "authorText")) + .orElse(""); } catch (final Exception e) { return ""; } @@ -75,8 +76,8 @@ public String getName() throws ParsingException { @Override public String getTextualUploadDate() throws ParsingException { try { - return getTextFromObject(JsonUtils.getObject(commentRenderer, - "publishedTimeText")); + return getTextFromObject(JsonUtils.getObject(commentRenderer, "publishedTimeText")) + .orElse(null); } catch (final Exception e) { throw new ParsingException("Could not get publishedTimeText", e); } @@ -170,10 +171,7 @@ public String getTextualLikeCount() throws ParsingException { } final JsonObject voteCountObj = JsonUtils.getObject(commentRenderer, "voteCount"); - if (voteCountObj.isEmpty()) { - return ""; - } - return getTextFromObject(voteCountObj); + return getTextFromObject(voteCountObj).orElse(""); } catch (final Exception e) { throw new ParsingException("Could not get the vote count", e); } @@ -189,7 +187,7 @@ public Description getCommentText() throws ParsingException { // https://github.com/TeamNewPipe/NewPipeExtractor/issues/380#issuecomment-668808584 return Description.EMPTY_DESCRIPTION; } - final String commentText = getTextFromObject(contentText, true); + final String commentText = getTextFromObject(contentText, true).orElse(""); // YouTube adds U+FEFF in some comments. // eg. https://www.youtube.com/watch?v=Nj4F63E59io final String commentTextBomRemoved = Utils.removeUTF8BOM(commentText); @@ -235,7 +233,8 @@ public boolean isUploaderVerified() throws ParsingException { @Override public String getUploaderName() throws ParsingException { try { - return getTextFromObject(JsonUtils.getObject(commentRenderer, "authorText")); + return getTextFromObject(JsonUtils.getObject(commentRenderer, "authorText")) + .orElse(""); } catch (final Exception e) { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java index 16c7a3e3ea..43a0c0b99b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java @@ -26,11 +26,8 @@ public YoutubeMixOrPlaylistInfoItemExtractor(final JsonObject mixInfoItem) { @Override public String getName() throws ParsingException { - final String name = getTextFromObject(mixInfoItem.getObject("title")); - if (isNullOrEmpty(name)) { - throw new ParsingException("Could not get name"); - } - return name; + return getTextFromObject(mixInfoItem.getObject("title")) + .orElseThrow(() -> new ParsingException("Could not get name")); } @Override @@ -51,7 +48,8 @@ public List getThumbnails() throws ParsingException { @Override public String getUploaderName() throws ParsingException { // this will be a list of uploaders for mixes - return YoutubeParsingHelper.getTextFromObject(mixInfoItem.getObject("longBylineText")); + return YoutubeParsingHelper.getTextFromObject(mixInfoItem.getObject("longBylineText")) + .orElse(null); } @Override @@ -69,10 +67,9 @@ public boolean isUploaderVerified() throws ParsingException { @Override public long getStreamCount() throws ParsingException { final String countString = YoutubeParsingHelper.getTextFromObject( - mixInfoItem.getObject("videoCountShortText")); - if (countString == null) { - throw new ParsingException("Could not extract item count for playlist/mix info item"); - } + mixInfoItem.getObject("videoCountShortText")) + .orElseThrow(() -> new ParsingException("Could not extract item count for" + + " playlist/mix info item")); try { return Integer.parseInt(countString); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 1df90f967f..7f6c40c94a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -126,11 +126,8 @@ public void onFetchPage(@Nonnull final Downloader downloader) @Nonnull @Override public String getName() throws ParsingException { - final String name = YoutubeParsingHelper.getTextAtKey(playlistData, "title"); - if (isNullOrEmpty(name)) { - throw new ParsingException("Could not get playlist name"); - } - return name; + return YoutubeParsingHelper.getTextAtKey(playlistData, "title") + .orElseThrow(() -> new ParsingException("Could not get playlist name")); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java index 1b3fccfb27..5506bda989 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java @@ -49,16 +49,11 @@ public List getThumbnails() throws ParsingException { @Override public String getName() throws ParsingException { - final String name = getTextFromObject(albumOrPlaylistInfoItem.getArray("flexColumns") + return getTextFromObject(albumOrPlaylistInfoItem.getArray("flexColumns") .getObject(0) .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text")); - - if (!isNullOrEmpty(name)) { - return name; - } - - throw new ParsingException("Could not get name"); + .getObject("text")) + .orElseThrow(() -> new ParsingException("Could not get name")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java index 4fe2dc666f..e5afcc3df5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java @@ -40,14 +40,11 @@ public List getThumbnails() throws ParsingException { @Override public String getName() throws ParsingException { - final String name = getTextFromObject(artistInfoItem.getArray("flexColumns") + return getTextFromObject(artistInfoItem.getArray("flexColumns") .getObject(0) .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text")); - if (!isNullOrEmpty(name)) { - return name; - } - throw new ParsingException("Could not get name"); + .getObject("text")) + .orElseThrow(() -> new ParsingException("Could not get name")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index a116eb312e..e6a4079a7e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -151,7 +151,8 @@ public String getSearchSuggestion() throws ParsingException { .getObject("didYouMeanRenderer"); if (!didYouMeanRenderer.isEmpty()) { - return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); + return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")) + .orElse(""); } // NOTE: As of 2025-07 "showing results for ..." doesn't seem to be returned by diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java index 11b220288c..78fdd98fb7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java @@ -46,14 +46,11 @@ public String getUrl() throws ParsingException { @Override public String getName() throws ParsingException { - final String name = getTextFromObject(songOrVideoInfoItem.getArray("flexColumns") + return getTextFromObject(songOrVideoInfoItem.getArray("flexColumns") .getObject(0) .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text")); - if (!isNullOrEmpty(name)) { - return name; - } - throw new ParsingException("Could not get name"); + .getObject("text")) + .orElseThrow(() -> new ParsingException("Could not get name")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 3e2fd89d5e..dbeb15bff4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -178,14 +178,10 @@ private JsonObject getPlaylistHeader() { @Nonnull @Override public String getName() throws ParsingException { - final String name = getTextFromObject(getPlaylistInfo().getObject("title")); - if (!isNullOrEmpty(name)) { - return name; - } - - return browseMetadataResponse.getObject(MICROFORMAT) - .getObject("microformatDataRenderer") - .getString("title"); + return getTextFromObject(getPlaylistInfo().getObject("title")) + .orElseGet(() -> browseMetadataResponse.getObject(MICROFORMAT) + .getObject("microformatDataRenderer") + .getString("title")); } @Nonnull @@ -240,7 +236,7 @@ public String getUploaderName() throws ParsingException { try { return getTextFromObject(isNewPlaylistInterface ? getPlaylistHeader().getObject("ownerText") - : getUploaderInfo().getObject("title")); + : getUploaderInfo().getObject("title")).orElse(null); } catch (final Exception e) { throw new ParsingException("Could not get playlist uploader name", e); } @@ -272,7 +268,8 @@ public boolean isUploaderVerified() throws ParsingException { public long getStreamCount() throws ParsingException { if (isNewPlaylistInterface) { final String numVideosText = - getTextFromObject(getPlaylistHeader().getObject("numVideosText")); + getTextFromObject(getPlaylistHeader().getObject("numVideosText")) + .orElse(null); if (numVideosText != null) { try { return Long.parseLong(Utils.removeNonDigitCharacters(numVideosText)); @@ -283,7 +280,8 @@ public long getStreamCount() throws ParsingException { final String firstByLineRendererText = getTextFromObject( getPlaylistHeader().getArray("byline") .getObject(0) - .getObject("text")); + .getObject("text")) + .orElse(null); if (firstByLineRendererText != null) { try { @@ -298,7 +296,7 @@ public long getStreamCount() throws ParsingException { (isNewPlaylistInterface ? getPlaylistHeader() : getPlaylistInfo()) .getArray("briefStats"); if (!briefStats.isEmpty()) { - final String briefsStatsText = getTextFromObject(briefStats.getObject(0)); + final var briefsStatsText = getTextFromObject(briefStats.getObject(0)).orElse(null); if (briefsStatsText != null) { return Long.parseLong(Utils.removeNonDigitCharacters(briefsStatsText)); } @@ -307,7 +305,7 @@ public long getStreamCount() throws ParsingException { final JsonArray stats = (isNewPlaylistInterface ? getPlaylistHeader() : getPlaylistInfo()) .getArray("stats"); if (!stats.isEmpty()) { - final String statsText = getTextFromObject(stats.getObject(0)); + final var statsText = getTextFromObject(stats.getObject(0)).orElse(null); if (statsText != null) { return Long.parseLong(Utils.removeNonDigitCharacters(statsText)); } @@ -319,11 +317,8 @@ public long getStreamCount() throws ParsingException { @Nonnull @Override public Description getDescription() throws ParsingException { - final String description = getTextFromObject( - getPlaylistInfo().getObject("description"), - true - ); - + final var descriptionObj = getPlaylistInfo().getObject("description"); + final String description = getTextFromObject(descriptionObj, true).orElse(null); return new Description(description, Description.HTML); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java index a0584a20fb..3392bdc129 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java @@ -12,6 +12,7 @@ import javax.annotation.Nonnull; import java.util.List; +import java.util.Optional; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; @@ -45,7 +46,7 @@ public List getThumbnails() throws ParsingException { @Override public String getName() throws ParsingException { try { - return getTextFromObject(playlistInfoItem.getObject("title")); + return getTextFromObject(playlistInfoItem.getObject("title")).orElse(null); } catch (final Exception e) { throw new ParsingException("Could not get name", e); } @@ -64,7 +65,7 @@ public String getUrl() throws ParsingException { @Override public String getUploaderName() throws ParsingException { try { - return getTextFromObject(playlistInfoItem.getObject("longBylineText")); + return getTextFromObject(playlistInfoItem.getObject("longBylineText")).orElse(null); } catch (final Exception e) { throw new ParsingException("Could not get uploader name", e); } @@ -73,7 +74,7 @@ public String getUploaderName() throws ParsingException { @Override public String getUploaderUrl() throws ParsingException { try { - return getUrlFromObject(playlistInfoItem.getObject("longBylineText")); + return getUrlFromObject(playlistInfoItem.getObject("longBylineText")).orElse(null); } catch (final Exception e) { throw new ParsingException("Could not get uploader url", e); } @@ -90,18 +91,10 @@ public boolean isUploaderVerified() throws ParsingException { @Override public long getStreamCount() throws ParsingException { - String videoCountText = playlistInfoItem.getString("videoCount"); - if (videoCountText == null) { - videoCountText = getTextFromObject(playlistInfoItem.getObject("videoCountText")); - } - - if (videoCountText == null) { - videoCountText = getTextFromObject(playlistInfoItem.getObject("videoCountShortText")); - } - - if (videoCountText == null) { - throw new ParsingException("Could not get stream count"); - } + final var videoCountText = Optional.ofNullable(playlistInfoItem.getString("videoCount")) + .or(() -> getTextFromObject(playlistInfoItem.getObject("videoCountText"))) + .or(() -> getTextFromObject(playlistInfoItem.getObject("videoCountShortText"))) + .orElseThrow(() -> new ParsingException("Could not get stream count")); try { return Long.parseLong(Utils.removeNonDigitCharacters(videoCountText)); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java index 71c7440b42..dd699d7515 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java @@ -2,7 +2,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonObject; @@ -45,7 +44,7 @@ public YoutubeReelInfoItemExtractor(@Nonnull final JsonObject reelInfo) { @Override public String getName() throws ParsingException { - return getTextFromObject(reelInfo.getObject("headline")); + return getTextFromObject(reelInfo.getObject("headline")).orElse(null); } @Override @@ -71,17 +70,13 @@ public StreamType getStreamType() throws ParsingException { @Override public long getViewCount() throws ParsingException { - final String viewCountText = getTextFromObject(reelInfo.getObject("viewCountText")); - if (!isNullOrEmpty(viewCountText)) { - // This approach is language dependent - if (viewCountText.toLowerCase().contains("no views")) { - return 0; - } - - return Utils.mixedNumberWordToLong(viewCountText); + final String viewCountText = getTextFromObject(reelInfo.getObject("viewCountText")) + .orElseThrow(() -> new ParsingException("Could not get short view count")); + // This approach is language dependent + if (viewCountText.toLowerCase().contains("no views")) { + return 0; } - - throw new ParsingException("Could not get short view count"); + return Utils.mixedNumberWordToLong(viewCountText); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index e888787cf9..ec12b8820c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -35,7 +35,6 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.Objects; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -130,11 +129,11 @@ public String getSearchSuggestion() throws ParsingException { "correctedQueryEndpoint.searchEndpoint.query"); } - return Objects.requireNonNullElse( - getTextFromObject(itemSectionRenderer.getArray("contents") - .getObject(0) - .getObject("showingResultsForRenderer") - .getObject("correctedQuery")), ""); + final var query = itemSectionRenderer.getArray("contents") + .getObject(0) + .getObject("showingResultsForRenderer") + .getObject("correctedQuery"); + return getTextFromObject(query).orElse(""); } @Override @@ -231,7 +230,7 @@ private void collectStreamsFrom(final MultiInfoItemsCollector collector, if (item.has("backgroundPromoRenderer")) { throw new NothingFoundException( getTextFromObject(item.getObject("backgroundPromoRenderer") - .getObject("bodyText"))); + .getObject("bodyText")).orElse("")); } else if (item.has("videoRenderer") && extractVideoResults) { collector.commit(new YoutubeStreamInfoItemExtractor( item.getObject("videoRenderer"), timeAgoParser)); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeShowRendererInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeShowRendererInfoItemExtractor.java index c7119907e0..0c2f02c5d8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeShowRendererInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeShowRendererInfoItemExtractor.java @@ -7,7 +7,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromObject; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; /** * A {@link YoutubeBaseShowInfoItemExtractor} implementation for {@code showRenderer}s. @@ -27,26 +26,16 @@ class YoutubeShowRendererInfoItemExtractor extends YoutubeBaseShowInfoItemExtrac @Override public String getUploaderName() throws ParsingException { - String name = getTextFromObject(longBylineText); - if (isNullOrEmpty(name)) { - name = getTextFromObject(shortBylineText); - if (isNullOrEmpty(name)) { - throw new ParsingException("Could not get uploader name"); - } - } - return name; + return getTextFromObject(longBylineText) + .or(() -> getTextFromObject(shortBylineText)) + .orElseThrow(() -> new ParsingException("Could not get uploader name")); } @Override public String getUploaderUrl() throws ParsingException { - String uploaderUrl = getUrlFromObject(longBylineText); - if (uploaderUrl == null) { - uploaderUrl = getUrlFromObject(shortBylineText); - if (uploaderUrl == null) { - throw new ParsingException("Could not get uploader URL"); - } - } - return uploaderUrl; + return getUrlFromObject(longBylineText) + .or(() -> getUrlFromObject(shortBylineText)) + .orElseThrow(() -> new ParsingException("Could not get uploader URL")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index e6fa782ce7..1e1ce7b65f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -153,20 +153,10 @@ public YoutubeStreamExtractor(final StreamingService service, final LinkHandler @Override public String getName() throws ParsingException { assertPageFetched(); - String title; - // Try to get the video's original title, which is untranslated - title = playerResponse.getObject("videoDetails").getString("title"); - - if (isNullOrEmpty(title)) { - title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title")); - - if (isNullOrEmpty(title)) { - throw new ParsingException("Could not get name"); - } - } - - return title; + return Optional.ofNullable(playerResponse.getObject("videoDetails").getString("title")) + .or(() -> getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title"))) + .orElseThrow(() -> new ParsingException("Could not get name")); } @Nullable @@ -193,18 +183,19 @@ public String getTextualUploadDate() { return null; } - final var textObject = getVideoPrimaryInfoRenderer().getObject("dateText"); - final String rendererDateText = getTextFromObject(textObject); - if (rendererDateText == null) { - return null; - } else if (rendererDateText.startsWith(PREMIERED_ON)) { // Premiered on 21 Feb 2020 - return rendererDateText.substring(PREMIERED_ON.length()); - } else if (rendererDateText.startsWith(PREMIERED)) { - // Premiered 20 hours ago / Premiered Feb 21, 2020 - return rendererDateText.substring(PREMIERED.length()); - } else { - return rendererDateText; - } + return getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")) + .map(rendererDateText -> { + if (rendererDateText.startsWith(PREMIERED_ON)) { + // Premiered on 21 Feb 2020 + return rendererDateText.substring(PREMIERED_ON.length()); + } else if (rendererDateText.startsWith(PREMIERED)) { + // Premiered 20 hours ago / Premiered Feb 21, 2020 + return rendererDateText.substring(PREMIERED.length()); + } else { + return rendererDateText; + } + }) + .orElse(null); } @Override @@ -256,31 +247,26 @@ public List getThumbnails() throws ParsingException { @Nonnull @Override - public Description getDescription() throws ParsingException { + public Description getDescription() { assertPageFetched(); // Description with more info on links - final String videoSecondaryInfoRendererDescription = getTextFromObject( - getVideoSecondaryInfoRenderer().getObject("description"), - true); - if (!isNullOrEmpty(videoSecondaryInfoRendererDescription)) { - return new Description(videoSecondaryInfoRendererDescription, Description.HTML); - } - - final String attributedDescription = attributedDescriptionToHtml( - getVideoSecondaryInfoRenderer().getObject("attributedDescription")); - if (!isNullOrEmpty(attributedDescription)) { - return new Description(attributedDescription, Description.HTML); - } - - String description = playerResponse.getObject("videoDetails") - .getString("shortDescription"); - if (description == null) { - final JsonObject descriptionObject = playerMicroFormatRenderer.getObject("description"); - description = getTextFromObject(descriptionObject); - } - - // Raw non-html description - return new Description(description, Description.PLAIN_TEXT); + final var renderer = getVideoSecondaryInfoRenderer(); + return getTextFromObject(renderer.getObject("description"), true) + .or(() -> { + final var description = renderer.getObject("attributedDescription"); + return Optional.ofNullable(attributedDescriptionToHtml(description)); + }) + .map(description -> new Description(description, Description.HTML)) + .orElseGet(() -> { + final var shortDescription = playerResponse.getObject("videoDetails") + .getString("shortDescription"); + final String description = Optional.ofNullable(shortDescription) + .or(() -> getTextFromObject(playerMicroFormatRenderer + .getObject("description"))) + .orElse(null); + // Raw non-html description + return new Description(description, Description.PLAIN_TEXT); + }); } @Override @@ -370,16 +356,11 @@ public long getTimeStamp() throws ParsingException { @Override public long getViewCount() throws ParsingException { - String views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount") - .getObject("videoViewCountRenderer").getObject("viewCount")); - - if (isNullOrEmpty(views)) { - views = playerResponse.getObject("videoDetails").getString("viewCount"); - - if (isNullOrEmpty(views)) { - throw new ParsingException("Could not get view count"); - } - } + final var views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount") + .getObject("videoViewCountRenderer").getObject("viewCount")) + .or(() -> Optional.ofNullable(playerResponse.getObject("videoDetails") + .getString("viewCount"))) + .orElseThrow(() -> new ParsingException("Could not get view count")); if (views.toLowerCase().contains("no views")) { return 0; @@ -575,7 +556,7 @@ public long getUploaderSubscriberCount() throws ParsingException { } try { return Utils.mixedNumberWordToLong(getTextFromObject(videoOwnerRenderer - .getObject("subscriberCountText"))); + .getObject("subscriberCountText")).orElse("")); } catch (final NumberFormatException e) { throw new ParsingException("Could not get uploader subscriber count", e); } @@ -784,13 +765,9 @@ public MultiInfoItemsCollector getRelatedItems() throws ExtractionException { */ @Override public String getErrorMessage() { - try { - return getTextFromObject(playerResponse.getObject(PLAYABILITY_STATUS) - .getObject("errorScreen").getObject("playerErrorMessageRenderer") - .getObject("reason")); - } catch (final NullPointerException e) { - return null; // No error message - } + return getTextFromObject(playerResponse.getObject(PLAYABILITY_STATUS) + .getObject("errorScreen").getObject("playerErrorMessageRenderer") + .getObject("reason")).orElse(null); } /*////////////////////////////////////////////////////////////////////////// @@ -881,17 +858,20 @@ private static void checkPlayabilityStatus(@Nonnull final JsonObject playability } if (reason.contains("unavailable")) { - final String detailedErrorMessage = getTextFromObject(playabilityStatus + final var subreason = playabilityStatus .getObject("errorScreen") .getObject("playerErrorMessageRenderer") - .getObject("subreason")); - if (detailedErrorMessage != null && detailedErrorMessage.contains("country")) { - throw new GeographicRestrictionException( - "This video is not available in client's country."); - } else { - throw new ContentNotAvailableException( - Objects.requireNonNullElse(detailedErrorMessage, reason)); - } + .getObject("subreason"); + throw getTextFromObject(subreason) + .map(message -> { + if (message.contains("country")) { + return new GeographicRestrictionException("This video is not " + + "available in client's country."); + } else { + return new ContentNotAvailableException(message); + } + }) + .orElse(new ContentNotAvailableException(reason)); } if (reason.contains("age-restricted")) { @@ -1536,19 +1516,17 @@ public String getCategory() { @Nonnull @Override public String getLicence() throws ParsingException { - final JsonObject metadataRowRenderer = getVideoSecondaryInfoRenderer() + final var metadataRowRenderer = getVideoSecondaryInfoRenderer() .getObject("metadataRowContainer") .getObject("metadataRowContainerRenderer") .getArray("rows") .getObject(0) .getObject("metadataRowRenderer"); - - final JsonArray contents = metadataRowRenderer.getArray("contents"); - final String license = getTextFromObject(contents.getObject(0)); - return license != null - && "Licence".equals(getTextFromObject(metadataRowRenderer.getObject("title"))) - ? license - : "YouTube licence"; + final var contents = metadataRowRenderer.getArray("contents"); + final var title = getTextFromObject(metadataRowRenderer.getObject("title")).orElse(null); + return getTextFromObject(contents.getObject(0)) + .filter(license -> "Licence".equals(title)) + .orElse("YouTube licence"); } @Override @@ -1613,10 +1591,8 @@ public List getStreamSegments() throws ParsingException { break; } - final String title = getTextFromObject(segmentJson.getObject("title")); - if (isNullOrEmpty(title)) { - throw new ParsingException("Could not get stream segment title."); - } + final String title = getTextFromObject(segmentJson.getObject("title")) + .orElseThrow(() -> new ParsingException("Could not get stream segment title.")); final StreamSegment segment = new StreamSegment(title, startTimeSeconds); segment.setUrl(getUrl() + "?t=" + startTimeSeconds); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 7b4deaaa58..65b15df972 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -48,6 +48,7 @@ import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.List; +import java.util.Optional; import java.util.regex.Pattern; public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @@ -130,11 +131,8 @@ public String getUrl() throws ParsingException { @Override public String getName() throws ParsingException { - final String name = getTextFromObject(videoInfo.getObject("title")); - if (!isNullOrEmpty(name)) { - return name; - } - throw new ParsingException("Could not get name"); + return getTextFromObject(videoInfo.getObject("title")) + .orElseThrow(() -> new ParsingException("Could not get name")); } @Override @@ -143,38 +141,24 @@ public long getDuration() throws ParsingException { return -1; } - String duration = getTextFromObject(videoInfo.getObject("lengthText")); - - if (isNullOrEmpty(duration)) { - // Available in playlists for videos - duration = videoInfo.getString("lengthSeconds"); - - if (isNullOrEmpty(duration)) { - final JsonObject timeOverlay = videoInfo.getArray("thumbnailOverlays") - .stream() - .filter(JsonObject.class::isInstance) - .map(JsonObject.class::cast) - .filter(thumbnailOverlay -> - thumbnailOverlay.has("thumbnailOverlayTimeStatusRenderer")) + final String duration = getTextFromObject(videoInfo.getObject("lengthText")) + // Available in playlists for videos + .or(() -> Optional.ofNullable(videoInfo.getString("lengthSeconds"))) + .or(() -> videoInfo.getArray("thumbnailOverlays").streamAsJsonObjects() + .map(overlay -> overlay.getObject("thumbnailOverlayTimeStatusRenderer")) + .filter(renderer -> !renderer.isEmpty()) .findFirst() - .orElse(null); + .flatMap(overlay -> getTextFromObject(overlay.getObject("text")))) + .orElse(null); - if (timeOverlay != null) { - duration = getTextFromObject( - timeOverlay.getObject("thumbnailOverlayTimeStatusRenderer") - .getObject("text")); - } + if (isNullOrEmpty(duration)) { + if (isPremiere()) { + // Premieres can be livestreams, so the duration is not available in this + // case + return -1; } - if (isNullOrEmpty(duration)) { - if (isPremiere()) { - // Premieres can be livestreams, so the duration is not available in this - // case - return -1; - } - - throw new ParsingException("Could not get duration"); - } + throw new ParsingException("Could not get duration"); } return YoutubeParsingHelper.parseDurationString(duration); @@ -182,21 +166,10 @@ public long getDuration() throws ParsingException { @Override public String getUploaderName() throws ParsingException { - String name = getTextFromObject(videoInfo.getObject("longBylineText")); - - if (isNullOrEmpty(name)) { - name = getTextFromObject(videoInfo.getObject("ownerText")); - - if (isNullOrEmpty(name)) { - name = getTextFromObject(videoInfo.getObject("shortBylineText")); - - if (isNullOrEmpty(name)) { - throw new ParsingException("Could not get uploader name"); - } - } - } - - return name; + return getTextFromObject(videoInfo.getObject("longBylineText")) + .or(() -> getTextFromObject(videoInfo.getObject("ownerText"))) + .or(() -> getTextFromObject(videoInfo.getObject("shortBylineText"))) + .orElseThrow(() -> new ParsingException("Could not get uploader name")); } @Override @@ -257,19 +230,15 @@ public String getTextualUploadDate() throws ParsingException { return DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm").format(localDateTime); } - String publishedTimeText = getTextFromObject(videoInfo.getObject("publishedTimeText")); - - if (isNullOrEmpty(publishedTimeText) && videoInfo.has("videoInfo")) { - /* - Returned in playlists, in the form: view count separator upload date - */ - publishedTimeText = videoInfo.getObject("videoInfo") - .getArray("runs") - .getObject(2) - .getString("text"); - } - - return isNullOrEmpty(publishedTimeText) ? null : publishedTimeText; + return getTextFromObject(videoInfo.getObject("publishedTimeText")) + .or(() -> { + // Returned in playlists, in the form: view count separator upload date + return Optional.ofNullable(videoInfo.getObject("videoInfo") + .getArray("runs") + .getObject(2) + .getString("text")); + }) + .orElse(null); } @Nullable @@ -303,7 +272,8 @@ public long getViewCount() throws ParsingException { // Ignore all exceptions, as the view count can be hidden by creators, and so cannot be // found in this case - final String viewCountText = getTextFromObject(videoInfo.getObject("viewCountText")); + final String viewCountText = getTextFromObject(videoInfo.getObject("viewCountText")) + .orElse(null); if (!isNullOrEmpty(viewCountText)) { try { return getViewCountFromViewCountText(viewCountText, false); @@ -337,7 +307,8 @@ public long getViewCount() throws ParsingException { // Returned everywhere but in playlists, used by the website to show view counts try { final String shortViewCountText = - getTextFromObject(videoInfo.getObject("shortViewCountText")); + getTextFromObject(videoInfo.getObject("shortViewCountText")) + .orElse(null); if (!isNullOrEmpty(shortViewCountText)) { return getViewCountFromViewCountText(shortViewCountText, true); } @@ -418,18 +389,12 @@ private Instant getInstantFromPremiere() throws ParsingException { @Nullable @Override - public String getShortDescription() throws ParsingException { - if (videoInfo.has("detailedMetadataSnippets")) { - return getTextFromObject(videoInfo.getArray("detailedMetadataSnippets") - .getObject(0) - .getObject("snippetText")); - } - - if (videoInfo.has("descriptionSnippet")) { - return getTextFromObject(videoInfo.getObject("descriptionSnippet")); - } - - return null; + public String getShortDescription() { + return getTextFromObject(videoInfo.getArray("detailedMetadataSnippets") + .getObject(0) + .getObject("snippetText")) + .or(() -> getTextFromObject(videoInfo.getObject("descriptionSnippet"))) + .orElse(null); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeTrendingExtractor.java index 4dd3c04355..4945cc6ab8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeTrendingExtractor.java @@ -23,7 +23,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; @@ -84,19 +83,10 @@ public InfoItemsPage getPage(final Page page) { @Override public String getName() throws ParsingException { final JsonObject header = initialData.getObject("header"); - String name = null; - if (header.has("feedTabbedHeaderRenderer")) { - name = getTextAtKey(header.getObject("feedTabbedHeaderRenderer"), "title"); - } else if (header.has("c4TabbedHeaderRenderer")) { - name = getTextAtKey(header.getObject("c4TabbedHeaderRenderer"), "title"); - } else if (header.has("pageHeaderRenderer")) { - name = getTextAtKey(header.getObject("pageHeaderRenderer"), "pageTitle"); - } - - if (isNullOrEmpty(name)) { - throw new ParsingException("Could not get Trending name"); - } - return name; + return getTextAtKey(header.getObject("feedTabbedHeaderRenderer"), "title") + .or(() -> getTextAtKey(header.getObject("c4TabbedHeaderRenderer"), "title")) + .or(() -> getTextAtKey(header.getObject("pageHeaderRenderer"), "pageTitle")) + .orElseThrow(() -> new ParsingException("Could not get Trending name")); } @Nonnull From 43e9afbb9c141fa19df04e9da36d8b371a55a111 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Sat, 11 Oct 2025 06:17:05 +0530 Subject: [PATCH 2/7] [YouTube] Update getUrlFromNavigationEndpoint to return Optional, add helper method for extracting music URL --- .../youtube/YoutubeDescriptionHelper.java | 3 +- .../youtube/YoutubeMetaInfoHelper.java | 37 ++-- .../youtube/YoutubeParsingHelper.java | 178 ++++++++++-------- .../YoutubeBaseShowInfoItemExtractor.java | 14 +- ...YoutubeMixOrPlaylistInfoItemExtractor.java | 12 +- ...eMixOrPlaylistLockupInfoItemExtractor.java | 5 +- ...MusicAlbumOrPlaylistInfoItemExtractor.java | 38 ++-- .../YoutubeMusicArtistInfoItemExtractor.java | 16 +- ...tubeMusicSongOrVideoInfoItemExtractor.java | 41 +--- .../extractors/YoutubePlaylistExtractor.java | 2 +- .../YoutubeReelInfoItemExtractor.java | 5 +- .../YoutubeStreamInfoItemExtractor.java | 32 ++-- 12 files changed, 173 insertions(+), 210 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDescriptionHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDescriptionHelper.java index afe053a358..7d9d1c0883 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDescriptionHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDescriptionHelper.java @@ -237,7 +237,8 @@ private static void addAllCommandRuns( return; } - final String url = getUrlFromNavigationEndpoint(navigationEndpoint); + final String url = getUrlFromNavigationEndpoint(navigationEndpoint) + .orElse(null); if (url == null) { return; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMetaInfoHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMetaInfoHelper.java index 672c7b4cef..7fc80dc3c0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMetaInfoHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMetaInfoHelper.java @@ -73,7 +73,8 @@ private static MetaInfo getInfoPanelContent(@Nonnull final JsonObject infoPanelC metaInfo.setContent(new Description(description, Description.HTML)); if (infoPanelContentRenderer.has("sourceEndpoint")) { final String metaInfoLinkUrl = getUrlFromNavigationEndpoint( - infoPanelContentRenderer.getObject("sourceEndpoint")); + infoPanelContentRenderer.getObject("sourceEndpoint")) + .orElse(""); try { metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded( metaInfoLinkUrl)))); @@ -81,10 +82,8 @@ private static MetaInfo getInfoPanelContent(@Nonnull final JsonObject infoPanelC throw new ParsingException("Could not get metadata info URL", e); } - final String metaInfoLinkText = getTextFromObject( - infoPanelContentRenderer.getObject("inlineSource")) - .orElseThrow(() -> - new ParsingException("Could not get metadata info link text.")); + final var source = infoPanelContentRenderer.getObject("inlineSource"); + final String metaInfoLinkText = getTextFromObjectOrThrow(source, "metadata info link"); metaInfo.addUrlText(metaInfoLinkText); } @@ -96,12 +95,10 @@ private static MetaInfo getClarificationRenderer( @Nonnull final JsonObject clarificationRenderer) throws ParsingException { final MetaInfo metaInfo = new MetaInfo(); - final String title = getTextFromObject(clarificationRenderer.getObject("contentTitle")) - .orElseThrow(() -> - new ParsingException("Could not extract clarification renderer content")); - final String text = getTextFromObject(clarificationRenderer.getObject("text")) - .orElseThrow(() -> - new ParsingException("Could not extract clarification renderer content")); + final String title = getTextFromObjectOrThrow( + clarificationRenderer.getObject("contentTitle"), "clarification renderer"); + final String text = getTextFromObjectOrThrow( + clarificationRenderer.getObject("text"), "clarification renderer"); metaInfo.setTitle(title); metaInfo.setContent(new Description(text, Description.PLAIN_TEXT)); @@ -110,22 +107,21 @@ private static MetaInfo getClarificationRenderer( .getObject("buttonRenderer"); try { final String url = getUrlFromNavigationEndpoint(actionButton - .getObject("command")); + .getObject("command")).orElse(""); metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded(url)))); } catch (final NullPointerException | MalformedURLException e) { throw new ParsingException("Could not get metadata info URL", e); } - final String metaInfoLinkText = getTextFromObject(actionButton.getObject("text")) - .orElseThrow(() -> - new ParsingException("Could not get metadata info link text.")); - metaInfo.addUrlText(metaInfoLinkText); + final String linkText = getTextFromObjectOrThrow(actionButton.getObject("text"), + "metadata info link"); + metaInfo.addUrlText(linkText); } if (clarificationRenderer.has("secondaryEndpoint") && clarificationRenderer .has("secondarySource")) { final String url = getUrlFromNavigationEndpoint(clarificationRenderer - .getObject("secondaryEndpoint")); + .getObject("secondaryEndpoint")).orElse(null); // Ignore Google URLs, because those point to a Google search about "Covid-19" if (url != null && !isGoogleURL(url)) { try { @@ -193,10 +189,9 @@ private static void getEmergencyOneboxRenderer( metaInfo.addUrlText(urlText); // usually the webpage of the association - final String url = getUrlFromNavigationEndpoint(r.getObject("navigationEndpoint")); - if (url == null) { - throw new ParsingException("Could not extract emergency renderer url"); - } + final String url = getUrlFromNavigationEndpoint(r.getObject("navigationEndpoint")) + .orElseThrow(() -> + new ParsingException("Could not extract emergency renderer url")); try { metaInfo.addUrl(new URL(replaceHttpWithHttps(url))); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index ae949dd361..96c6dc58e2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -687,87 +687,103 @@ public static String getYoutubeMusicClientVersion() return youtubeMusicClientVersion; } - @Nullable - public static String getUrlFromNavigationEndpoint( + @Nonnull + public static Optional getUrlFromNavigationEndpoint( @Nonnull final JsonObject navigationEndpoint) { - if (navigationEndpoint.has("urlEndpoint")) { - String internUrl = navigationEndpoint.getObject("urlEndpoint") - .getString("url"); - if (internUrl.startsWith("https://www.youtube.com/redirect?")) { - // remove https://www.youtube.com part to fall in the next if block - internUrl = internUrl.substring(23); - } - - if (internUrl.startsWith("/redirect?")) { - // q parameter can be the first parameter - internUrl = internUrl.substring(10); - final String[] params = internUrl.split("&"); - for (final String param : params) { - if (param.split("=")[0].equals("q")) { - return Utils.decodeUrlUtf8(param.split("=")[1]); + return Optional.ofNullable(navigationEndpoint.getObject("urlEndpoint") + .getString("url")) + .map(internUrl -> { + if (internUrl.startsWith("https://www.youtube.com/redirect?")) { + // remove https://www.youtube.com part to fall in the next if block + internUrl = internUrl.substring(23); } - } - } else if (internUrl.startsWith("http")) { - return internUrl; - } else if (internUrl.startsWith("/channel") || internUrl.startsWith("/user") - || internUrl.startsWith("/watch")) { - return "https://www.youtube.com" + internUrl; - } - } - - if (navigationEndpoint.has("browseEndpoint")) { - final JsonObject browseEndpoint = navigationEndpoint.getObject("browseEndpoint"); - final String canonicalBaseUrl = browseEndpoint.getString("canonicalBaseUrl"); - final String browseId = browseEndpoint.getString("browseId"); - - if (browseId != null) { - if (browseId.startsWith("UC")) { - // All channel IDs are prefixed with UC - return "https://www.youtube.com/channel/" + browseId; - } else if (browseId.startsWith("VL")) { - // All playlist IDs are prefixed with VL, which needs to be removed from the - // playlist ID - return "https://www.youtube.com/playlist?list=" + browseId.substring(2); - } - } - if (!isNullOrEmpty(canonicalBaseUrl)) { - return "https://www.youtube.com" + canonicalBaseUrl; - } - } - - if (navigationEndpoint.has("watchEndpoint")) { - final StringBuilder url = new StringBuilder(); - url.append("https://www.youtube.com/watch?v=") - .append(navigationEndpoint.getObject("watchEndpoint") - .getString(VIDEO_ID)); - if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) { - url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint") - .getString("playlistId")); - } - if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) { - url.append("&t=") - .append(navigationEndpoint.getObject("watchEndpoint") - .getInt("startTimeSeconds")); - } - return url.toString(); - } - - if (navigationEndpoint.has("watchPlaylistEndpoint")) { - return "https://www.youtube.com/playlist?list=" - + navigationEndpoint.getObject("watchPlaylistEndpoint") - .getString("playlistId"); - } + if (internUrl.startsWith("/redirect?")) { + // q parameter can be the first parameter + internUrl = internUrl.substring(10); + final String[] params = internUrl.split("&"); + for (final String param : params) { + final String[] nameAndValue = param.split("="); + if (nameAndValue[0].equals("q")) { + return Utils.decodeUrlUtf8(nameAndValue[1]); + } + } + } else if (internUrl.startsWith("http")) { + return internUrl; + } else if (internUrl.startsWith("/channel") || internUrl.startsWith("/user") + || internUrl.startsWith("/watch")) { + return "https://www.youtube.com" + internUrl; + } - if (navigationEndpoint.has("commandMetadata")) { - final JsonObject metadata = navigationEndpoint.getObject("commandMetadata") - .getObject("webCommandMetadata"); - if (metadata.has("url")) { - return "https://www.youtube.com" + metadata.getString("url"); - } - } + return null; + }) + .or(() -> { + final var browseEndpoint = navigationEndpoint.getObject("browseEndpoint"); + final var baseUrl = browseEndpoint.getString("canonicalBaseUrl"); + final var browseId = browseEndpoint.getString("browseId"); + + return Optional.ofNullable(browseId) + .map(id -> { + if (id.startsWith("UC")) { + // All channel IDs are prefixed with UC + return "https://www.youtube.com/channel/" + id; + } else if (id.startsWith("VL")) { + // All playlist IDs are prefixed with VL, which needs to be + // removed from the playlist ID + return "https://www.youtube.com/playlist?list=" + + id.substring(2); + } + return null; + }) + .or(() -> { + if (!isNullOrEmpty(baseUrl)) { + return Optional.of("https://www.youtube.com" + baseUrl); + } else { + return Optional.empty(); + } + }); + }) + .or(() -> { + final var watchEndpoint = navigationEndpoint.getObject("watchEndpoint"); + final var videoId = watchEndpoint.getString(VIDEO_ID); + final var playlistId = watchEndpoint.getString("playlistId"); + final var startTime = watchEndpoint.getInt("startTimeSeconds", -1); + final String url = "https://www.youtube.com/watch?v=" + videoId + + (playlistId != null ? "&list=" + playlistId : "") + + (startTime != -1 ? "&t=" + startTime : ""); + return Optional.of(url); + }) + .or(() -> { + final var playlistId = navigationEndpoint.getObject("watchPlaylistEndpoint") + .getString("playlistId"); + return Optional.ofNullable(playlistId) + .map(id -> "https://www.youtube.com/playlist?list=" + id); + }) + .or(() -> { + final var metadata = navigationEndpoint.getObject("commandMetadata") + .getObject("webCommandMetadata"); + return Optional.ofNullable(metadata.getString("url")) + .map(url -> "https://www.youtube.com" + url); + }) + .filter(url -> !url.isEmpty()); + } - return null; + @Nonnull + public static Optional getMusicUploaderUrlFromMenu(@Nonnull final JsonObject object) { + final var items = object.getObject("menu") + .getObject("menuRenderer") + .getArray("items"); + return items.streamAsJsonObjects() + .flatMap(item -> { + final var renderer = item.getObject("menuNavigationItemRenderer"); + final var iconType = renderer.getObject("icon").getString("iconType"); + final var endpoint = renderer.getObject("navigationEndpoint"); + + return "ARTIST".equals(iconType) + ? getUrlFromNavigationEndpoint(endpoint).stream() + : Stream.empty(); + }) + .findFirst(); } /** @@ -790,8 +806,8 @@ public static Optional getTextFromObject(@Nonnull final JsonObject textO if (html) { final String url = getUrlFromNavigationEndpoint( - run.getObject("navigationEndpoint")); - if (!isNullOrEmpty(url)) { + run.getObject("navigationEndpoint")).orElse(null); + if (url != null) { textString = "" + Entities.escape(textString) + ""; } @@ -838,9 +854,9 @@ public static Optional getTextFromObject(@Nonnull final JsonObject textO @Nonnull public static Optional getUrlFromObject(@Nonnull final JsonObject textObject) { return textObject.getArray("runs").streamAsJsonObjects() - .map(textPart -> getUrlFromNavigationEndpoint(textPart - .getObject("navigationEndpoint"))) - .filter(url -> !isNullOrEmpty(url)) + .flatMap(textPart -> getUrlFromNavigationEndpoint(textPart + .getObject("navigationEndpoint")) + .stream()) .findFirst(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBaseShowInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBaseShowInfoItemExtractor.java index a3bdb554e8..8499abe12e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBaseShowInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBaseShowInfoItemExtractor.java @@ -9,7 +9,7 @@ import javax.annotation.Nonnull; import java.util.List; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; @@ -32,7 +32,8 @@ public String getName() throws ParsingException { @Override public String getUrl() throws ParsingException { - return getUrlFromNavigationEndpoint(showRenderer.getObject("navigationEndpoint")); + return getUrlFromNavigationEndpoint(showRenderer.getObject("navigationEndpoint")) + .orElse(null); } @Nonnull @@ -46,11 +47,10 @@ public List getThumbnails() throws ParsingException { public long getStreamCount() throws ParsingException { // The stream count should be always returned in the first text object for English // localizations, but the complete text is parsed for reliability purposes - final String streamCountText = getTextFromObject( - showRenderer.getObject("thumbnailOverlays") - .getObject("thumbnailOverlayBottomPanelRenderer") - .getObject("text")) - .orElseThrow(() -> new ParsingException("Could not get stream count")); + final var textObject = showRenderer.getObject("thumbnailOverlays") + .getObject("thumbnailOverlayBottomPanelRenderer") + .getObject("text"); + final String streamCountText = getTextFromObjectOrThrow(textObject, "stream count"); try { // The data returned could be a human/shortened number, but no show with more than 1000 diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java index 43a0c0b99b..48cc1fab97 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -26,8 +26,7 @@ public YoutubeMixOrPlaylistInfoItemExtractor(final JsonObject mixInfoItem) { @Override public String getName() throws ParsingException { - return getTextFromObject(mixInfoItem.getObject("title")) - .orElseThrow(() -> new ParsingException("Could not get name")); + return getTextFromObjectOrThrow(mixInfoItem.getObject("title"), "name"); } @Override @@ -66,10 +65,9 @@ public boolean isUploaderVerified() throws ParsingException { @Override public long getStreamCount() throws ParsingException { - final String countString = YoutubeParsingHelper.getTextFromObject( - mixInfoItem.getObject("videoCountShortText")) - .orElseThrow(() -> new ParsingException("Could not extract item count for" - + " playlist/mix info item")); + final var textObject = mixInfoItem.getObject("videoCountShortText"); + final String countString = getTextFromObjectOrThrow(textObject, + "item count for playlist/mix info item"); try { return Integer.parseInt(countString); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistLockupInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistLockupInfoItemExtractor.java index 009fc51d0c..bf33c4d38f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistLockupInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistLockupInfoItemExtractor.java @@ -87,7 +87,8 @@ public String getUploaderUrl() throws ParsingException { .getArray("commandRuns") .getObject(0) .getObject("onTap") - .getObject("innertubeCommand")); + .getObject("innertubeCommand")) + .orElse(null); } @Override @@ -159,7 +160,7 @@ public String getUrl() throws ParsingException { return getUrlFromNavigationEndpoint(lockupViewModel.getObject("rendererContext") .getObject("commandContext") .getObject("onTap") - .getObject("innertubeCommand")); + .getObject("innertubeCommand")).orElse(null); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java index 5506bda989..138043f428 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java @@ -11,8 +11,9 @@ import java.util.List; import static org.schabi.newpipe.extractor.ListExtractor.ITEM_COUNT_UNKNOWN; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getMusicUploaderUrlFromMenu; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -49,11 +50,11 @@ public List getThumbnails() throws ParsingException { @Override public String getName() throws ParsingException { - return getTextFromObject(albumOrPlaylistInfoItem.getArray("flexColumns") + final var textObject = albumOrPlaylistInfoItem.getArray("flexColumns") .getObject(0) .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text")) - .orElseThrow(() -> new ParsingException("Could not get name")); + .getObject("text"); + return getTextFromObjectOrThrow(textObject, "name"); } @Override @@ -100,28 +101,13 @@ public String getUploaderName() throws ParsingException { @Override public String getUploaderUrl() throws ParsingException { // first try obtaining the uploader from the menu (will not work for MUSIC_PLAYLISTS though) - final JsonArray items = albumOrPlaylistInfoItem.getObject("menu") - .getObject("menuRenderer") - .getArray("items"); - for (final Object item : items) { - final JsonObject menuNavigationItemRenderer = - ((JsonObject) item).getObject("menuNavigationItemRenderer"); - if (menuNavigationItemRenderer.getObject("icon") - .getString("iconType", "") - .equals("ARTIST")) { - return getUrlFromNavigationEndpoint( - menuNavigationItemRenderer.getObject("navigationEndpoint")); - } - } - - // then try obtaining it from the uploader description element - if (!descriptionElementUploader.has("navigationEndpoint")) { - // if there is no navigationEndpoint for the uploader - // then this playlist/album is likely autogenerated - return null; - } - return getUrlFromNavigationEndpoint( - descriptionElementUploader.getObject("navigationEndpoint")); + return getMusicUploaderUrlFromMenu(albumOrPlaylistInfoItem) + // then try obtaining it from the uploader description element + .or(() -> getUrlFromNavigationEndpoint(descriptionElementUploader + .getObject("navigationEndpoint"))) + // if there is no navigationEndpoint for the uploader + // then this playlist/album is likely autogenerated + .orElse(null); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java index e5afcc3df5..4251fbc5bd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java @@ -12,7 +12,7 @@ import javax.annotation.Nullable; import java.util.List; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -40,21 +40,17 @@ public List getThumbnails() throws ParsingException { @Override public String getName() throws ParsingException { - return getTextFromObject(artistInfoItem.getArray("flexColumns") + final var jsonObject = artistInfoItem.getArray("flexColumns") .getObject(0) .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text")) - .orElseThrow(() -> new ParsingException("Could not get name")); + .getObject("text"); + return getTextFromObjectOrThrow(jsonObject, "name"); } @Override public String getUrl() throws ParsingException { - final String url = getUrlFromNavigationEndpoint( - artistInfoItem.getObject("navigationEndpoint")); - if (!isNullOrEmpty(url)) { - return url; - } - throw new ParsingException("Could not get URL"); + return getUrlFromNavigationEndpoint(artistInfoItem.getObject("navigationEndpoint")) + .orElseThrow(() -> new ParsingException("Could not get URL")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java index 78fdd98fb7..13fc47ca1e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java @@ -14,8 +14,9 @@ import javax.annotation.Nonnull; import java.util.List; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getMusicUploaderUrlFromMenu; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; @@ -46,11 +47,11 @@ public String getUrl() throws ParsingException { @Override public String getName() throws ParsingException { - return getTextFromObject(songOrVideoInfoItem.getArray("flexColumns") + final var textObject = songOrVideoInfoItem.getArray("flexColumns") .getObject(0) .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text")) - .orElseThrow(() -> new ParsingException("Could not get name")); + .getObject("text"); + return getTextFromObjectOrThrow(textObject, "name"); } @Override @@ -85,41 +86,17 @@ public String getUploaderName() throws ParsingException { @Override public String getUploaderUrl() throws ParsingException { if (searchType.equals(MUSIC_VIDEOS)) { - final JsonArray items = songOrVideoInfoItem.getObject("menu") - .getObject("menuRenderer") - .getArray("items"); - for (final Object item : items) { - final JsonObject menuNavigationItemRenderer = - ((JsonObject) item).getObject("menuNavigationItemRenderer"); - if (menuNavigationItemRenderer.getObject("icon") - .getString("iconType", "") - .equals("ARTIST")) { - return getUrlFromNavigationEndpoint( - menuNavigationItemRenderer.getObject("navigationEndpoint")); - } - } - - return null; + return getMusicUploaderUrlFromMenu(songOrVideoInfoItem).orElse(null); } else { - final JsonObject navigationEndpointHolder = songOrVideoInfoItem.getArray("flexColumns") + final JsonObject holder = songOrVideoInfoItem.getArray("flexColumns") .getObject(1) .getObject("musicResponsiveListItemFlexColumnRenderer") .getObject("text") .getArray("runs") .getObject(0); - if (!navigationEndpointHolder.has("navigationEndpoint")) { - return null; - } - - final String url = getUrlFromNavigationEndpoint( - navigationEndpointHolder.getObject("navigationEndpoint")); - - if (!isNullOrEmpty(url)) { - return url; - } - - throw new ParsingException("Could not get uploader URL"); + return getUrlFromNavigationEndpoint(holder.getObject("navigationEndpoint")) + .orElseThrow(() -> new ParsingException("Could not get uploader URL")); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index dbeb15bff4..085548547d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -225,7 +225,7 @@ public String getUploaderUrl() throws ParsingException { .getArray("runs") .getObject(0) .getObject("navigationEndpoint") - : getUploaderInfo().getObject("navigationEndpoint")); + : getUploaderInfo().getObject("navigationEndpoint")).orElse(null); } catch (final Exception e) { throw new ParsingException("Could not get playlist uploader url", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java index dd699d7515..1dcb890183 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; import com.grack.nanojson.JsonObject; @@ -70,8 +71,8 @@ public StreamType getStreamType() throws ParsingException { @Override public long getViewCount() throws ParsingException { - final String viewCountText = getTextFromObject(reelInfo.getObject("viewCountText")) - .orElseThrow(() -> new ParsingException("Could not get short view count")); + final String viewCountText = getTextFromObjectOrThrow( + reelInfo.getObject("viewCountText"), "short view count"); // This approach is language dependent if (viewCountText.toLowerCase().contains("no views")) { return 0; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 65b15df972..9098200788 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -19,9 +19,9 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; @@ -131,8 +131,7 @@ public String getUrl() throws ParsingException { @Override public String getName() throws ParsingException { - return getTextFromObject(videoInfo.getObject("title")) - .orElseThrow(() -> new ParsingException("Could not get name")); + return getTextFromObjectOrThrow(videoInfo.getObject("title"), "name"); } @Override @@ -174,24 +173,17 @@ public String getUploaderName() throws ParsingException { @Override public String getUploaderUrl() throws ParsingException { - String url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText") - .getArray("runs").getObject(0).getObject("navigationEndpoint")); - - if (isNullOrEmpty(url)) { - url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText") - .getArray("runs").getObject(0).getObject("navigationEndpoint")); - - if (isNullOrEmpty(url)) { - url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText") - .getArray("runs").getObject(0).getObject("navigationEndpoint")); - - if (isNullOrEmpty(url)) { - throw new ParsingException("Could not get uploader url"); - } - } - } + return getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText")) + .or(() -> getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText"))) + .or(() -> getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText"))) + .orElseThrow(() -> new ParsingException("Could not get uploader url")); + } - return url; + @Nonnull + private Optional getUrlFromNavigationEndpoint(@Nonnull final JsonObject jsonObject) { + final var endpoint = jsonObject.getArray("runs").getObject(0) + .getObject("navigationEndpoint"); + return YoutubeParsingHelper.getUrlFromNavigationEndpoint(endpoint); } @Nonnull From 4ad744c96660c297e5babff146936fcdb897275c Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Sat, 11 Oct 2025 07:19:43 +0530 Subject: [PATCH 3/7] [YouTube] Reuse string predicate --- .../services/youtube/YoutubeParsingHelper.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 96c6dc58e2..e87aa84ccf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -74,6 +74,7 @@ import java.util.Optional; import java.util.Random; import java.util.Set; +import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -197,6 +198,8 @@ private YoutubeParsingHelper() { private static boolean consentAccepted = false; + private static final Predicate STRING_PREDICATE = text -> !text.isEmpty(); + public static boolean isGoogleURL(final String url) { final String cachedUrl = extractCachedUrlIfNeeded(url); try { @@ -553,8 +556,8 @@ private static String getClientVersionFromServiceTrackingParam( .map(JsonObject.class::cast) .filter(param -> param.getString("key", "") .equals(clientVersionKey)) - .map(param -> param.getString("value")) - .filter(paramValue -> !isNullOrEmpty(paramValue)) + .map(param -> param.getString("value", "")) + .filter(STRING_PREDICATE) .findFirst() .orElse(null); } @@ -765,7 +768,7 @@ public static Optional getUrlFromNavigationEndpoint( return Optional.ofNullable(metadata.getString("url")) .map(url -> "https://www.youtube.com" + url); }) - .filter(url -> !url.isEmpty()); + .filter(STRING_PREDICATE); } @Nonnull @@ -836,7 +839,7 @@ public static Optional getTextFromObject(@Nonnull final JsonObject textO } return Optional.of(string); }) - .filter(text -> !text.isEmpty()); + .filter(STRING_PREDICATE); } @Nonnull @@ -864,7 +867,7 @@ public static Optional getUrlFromObject(@Nonnull final JsonObject textOb public static Optional getTextAtKey(@Nonnull final JsonObject jsonObject, final String theKey) { return Optional.ofNullable(jsonObject.getString(theKey)) - .filter(text -> !text.isEmpty()) + .filter(STRING_PREDICATE) .or(() -> getTextFromObject(jsonObject.getObject(theKey))); } From f42ebf39ca56a6c14fb7681c059b214828ed4907 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Sat, 11 Oct 2025 20:18:45 +0530 Subject: [PATCH 4/7] [YouTube] Fix test failure --- .../youtube/YoutubeParsingHelper.java | 19 +++++++++---------- ...tubeMusicSongOrVideoInfoItemExtractor.java | 15 ++++++++++----- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index e87aa84ccf..a42f42f24f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -746,16 +746,15 @@ public static Optional getUrlFromNavigationEndpoint( } }); }) - .or(() -> { - final var watchEndpoint = navigationEndpoint.getObject("watchEndpoint"); - final var videoId = watchEndpoint.getString(VIDEO_ID); - final var playlistId = watchEndpoint.getString("playlistId"); - final var startTime = watchEndpoint.getInt("startTimeSeconds", -1); - final String url = "https://www.youtube.com/watch?v=" + videoId - + (playlistId != null ? "&list=" + playlistId : "") - + (startTime != -1 ? "&t=" + startTime : ""); - return Optional.of(url); - }) + .or(() -> Optional.ofNullable(navigationEndpoint.getObject("watchEndpoint", null)) + .map(watchEndpoint -> { + final var videoId = watchEndpoint.getString(VIDEO_ID); + final var playlistId = watchEndpoint.getString("playlistId"); + final var startTime = watchEndpoint.getInt("startTimeSeconds", -1); + return "https://www.youtube.com/watch?v=" + videoId + + (playlistId != null ? "&list=" + playlistId : "") + + (startTime != -1 ? "&t=" + startTime : ""); + })) .or(() -> { final var playlistId = navigationEndpoint.getObject("watchPlaylistEndpoint") .getString("playlistId"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java index 13fc47ca1e..35917776c4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java @@ -88,15 +88,20 @@ public String getUploaderUrl() throws ParsingException { if (searchType.equals(MUSIC_VIDEOS)) { return getMusicUploaderUrlFromMenu(songOrVideoInfoItem).orElse(null); } else { - final JsonObject holder = songOrVideoInfoItem.getArray("flexColumns") + final var endpoint = songOrVideoInfoItem.getArray("flexColumns") .getObject(1) .getObject("musicResponsiveListItemFlexColumnRenderer") .getObject("text") .getArray("runs") - .getObject(0); - - return getUrlFromNavigationEndpoint(holder.getObject("navigationEndpoint")) - .orElseThrow(() -> new ParsingException("Could not get uploader URL")); + .getObject(0) + .getObject("navigationEndpoint"); + + if (!endpoint.isEmpty()) { + return getUrlFromNavigationEndpoint(endpoint) + .orElseThrow(() -> new ParsingException("Could not get uploader URL")); + } else { + return null; + } } } From 30ba0c20571e85f6a115625616d887e46fb9d825 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Sat, 11 Oct 2025 20:35:36 +0530 Subject: [PATCH 5/7] [YouTube] Reduce lambda creation in getUrlFromNavigationEndpoint --- .../youtube/YoutubeParsingHelper.java | 37 ++++++++----------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index a42f42f24f..f2b6ba5a3b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -720,32 +720,27 @@ public static Optional getUrlFromNavigationEndpoint( return null; }) - .or(() -> { - final var browseEndpoint = navigationEndpoint.getObject("browseEndpoint"); - final var baseUrl = browseEndpoint.getString("canonicalBaseUrl"); - final var browseId = browseEndpoint.getString("browseId"); - - return Optional.ofNullable(browseId) - .map(id -> { - if (id.startsWith("UC")) { + .or(() -> Optional.ofNullable(navigationEndpoint.getObject("browseEndpoint", null)) + .map(browseEndpoint -> { + final var canonicalBaseUrl = + browseEndpoint.getString("canonicalBaseUrl"); + final var browseId = browseEndpoint.getString("browseId"); + + if (browseId != null) { + if (browseId.startsWith("UC")) { // All channel IDs are prefixed with UC - return "https://www.youtube.com/channel/" + id; - } else if (id.startsWith("VL")) { + return "https://www.youtube.com/channel/" + browseId; + } else if (browseId.startsWith("VL")) { // All playlist IDs are prefixed with VL, which needs to be // removed from the playlist ID return "https://www.youtube.com/playlist?list=" - + id.substring(2); - } - return null; - }) - .or(() -> { - if (!isNullOrEmpty(baseUrl)) { - return Optional.of("https://www.youtube.com" + baseUrl); - } else { - return Optional.empty(); + + browseId.substring(2); } - }); - }) + } else if (!isNullOrEmpty(canonicalBaseUrl)) { + return "https://www.youtube.com" + canonicalBaseUrl; + } + return null; + })) .or(() -> Optional.ofNullable(navigationEndpoint.getObject("watchEndpoint", null)) .map(watchEndpoint -> { final var videoId = watchEndpoint.getString(VIDEO_ID); From 9dab638036e0406f27b1071a6713d85d153cc2c1 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Mon, 13 Oct 2025 05:18:11 +0530 Subject: [PATCH 6/7] [YouTube] Use Optional chaining in YoutubePlaylistExtractor --- .../youtube/YoutubeParsingHelper.java | 2 +- .../extractors/YoutubePlaylistExtractor.java | 66 ++++++------------- 2 files changed, 21 insertions(+), 47 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index f2b6ba5a3b..dcbbaa90f2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -826,7 +826,7 @@ public static Optional getTextFromObject(@Nonnull final JsonObject textO final String string; if (html) { - string = text.replaceAll("\\n", "
") + string = text.replace("\\n", "
") .replaceAll(" {2}", "  "); } else { string = text; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 085548547d..210a82a464 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -37,6 +37,7 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.List; +import java.util.Optional; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -266,52 +267,25 @@ public boolean isUploaderVerified() throws ParsingException { @Override public long getStreamCount() throws ParsingException { - if (isNewPlaylistInterface) { - final String numVideosText = - getTextFromObject(getPlaylistHeader().getObject("numVideosText")) - .orElse(null); - if (numVideosText != null) { - try { - return Long.parseLong(Utils.removeNonDigitCharacters(numVideosText)); - } catch (final NumberFormatException ignored) { - } - } - - final String firstByLineRendererText = getTextFromObject( - getPlaylistHeader().getArray("byline") - .getObject(0) - .getObject("text")) - .orElse(null); - - if (firstByLineRendererText != null) { - try { - return Long.parseLong(Utils.removeNonDigitCharacters(firstByLineRendererText)); - } catch (final NumberFormatException ignored) { - } - } - } - - // These data structures are returned in both layouts - final JsonArray briefStats = - (isNewPlaylistInterface ? getPlaylistHeader() : getPlaylistInfo()) - .getArray("briefStats"); - if (!briefStats.isEmpty()) { - final var briefsStatsText = getTextFromObject(briefStats.getObject(0)).orElse(null); - if (briefsStatsText != null) { - return Long.parseLong(Utils.removeNonDigitCharacters(briefsStatsText)); - } - } - - final JsonArray stats = (isNewPlaylistInterface ? getPlaylistHeader() : getPlaylistInfo()) - .getArray("stats"); - if (!stats.isEmpty()) { - final var statsText = getTextFromObject(stats.getObject(0)).orElse(null); - if (statsText != null) { - return Long.parseLong(Utils.removeNonDigitCharacters(statsText)); - } - } - - return ITEM_COUNT_UNKNOWN; + final var header = getPlaylistHeader(); + final Optional count = isNewPlaylistInterface + ? getTextFromObject(header.getObject("numVideosText")) + .or(() -> getTextFromObject(header.getArray("byline") + .getObject(0).getObject("text"))) + : Optional.empty(); + final var playlist = isNewPlaylistInterface ? header : getPlaylistInfo(); + + // "briefStats" and "stats" are returned in both layouts + return count.or(() -> getTextFromObject(playlist.getArray("briefStats").getObject(0))) + .or(() -> getTextFromObject(playlist.getArray("stats").getObject(0))) + .map(numText -> { + try { + return Long.parseLong(Utils.removeNonDigitCharacters(numText)); + } catch (final NumberFormatException e) { + return null; + } + }) + .orElse(ITEM_COUNT_UNKNOWN); } @Nonnull From f5c682dccdd81adfd1649365f63b5e7fd5dd7f75 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Sun, 9 Nov 2025 12:04:52 +0530 Subject: [PATCH 7/7] Fix tests --- .../youtube/extractors/YoutubeStreamExtractor.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 89858928e0..7cf5c89921 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -536,8 +536,8 @@ public List getUploaderAvatars() throws ParsingException { @Override public long getUploaderSubscriberCount() throws ParsingException { - final var videoOwnerRenderer = videoSecondaryInfoRenderer.getObject("owner") - .getObject("videoOwnerRenderer"); + final var videoOwnerRenderer = JsonUtils.getObject(videoSecondaryInfoRenderer, + "owner.videoOwnerRenderer"); final String subscriberCountText = getTextFromObject(videoOwnerRenderer.getObject("subscriberCountText")) .or(() -> YoutubeParsingHelper.getFirstCollaborator(videoOwnerRenderer) @@ -547,12 +547,9 @@ public long getUploaderSubscriberCount() throws ParsingException { .filter(YoutubeParsingHelper.STRING_PREDICATE) .orElse(null); - if (subscriberCountText == null) { - return UNKNOWN_SUBSCRIBER_COUNT; - } - try { - return Utils.mixedNumberWordToLong(subscriberCountText); + return subscriberCountText != null ? Utils.mixedNumberWordToLong(subscriberCountText) + : UNKNOWN_SUBSCRIBER_COUNT; } catch (final NumberFormatException e) { throw new ParsingException("Could not get uploader subscriber count", e); }