diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java index c3faf673f8..a66ffaeccc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java @@ -519,22 +519,23 @@ public static String getChannelName(@Nullable final ChannelHeader channelHeader, } return Optional.ofNullable(channelHeader) - .map(header -> { + .flatMap(header -> { final JsonObject channelJson = header.json; switch (header.headerType) { case PAGE: - return channelJson.getObject(CONTENT) + final String pageTitle = channelJson.getObject(CONTENT) .getObject(PAGE_HEADER_VIEW_MODEL) .getObject(TITLE) .getObject("dynamicTextViewModel") .getObject("text") .getString(CONTENT, channelJson.getString("pageTitle")); + return Optional.ofNullable(pageTitle); case CAROUSEL: case INTERACTIVE_TABBED: return getTextFromObject(channelJson.getObject(TITLE)); case C4_TABBED: default: - return channelJson.getString(TITLE); + return Optional.ofNullable(channelJson.getString(TITLE)); } }) // The channel name from a microformatDataRenderer may be different from the one diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDescriptionHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDescriptionHelper.java index afe053a358..7d9d1c0883 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDescriptionHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDescriptionHelper.java @@ -237,7 +237,8 @@ private static void addAllCommandRuns( return; } - final String url = getUrlFromNavigationEndpoint(navigationEndpoint); + final String url = getUrlFromNavigationEndpoint(navigationEndpoint) + .orElse(null); if (url == null) { return; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMetaInfoHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMetaInfoHelper.java index 214a13c405..7fc80dc3c0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMetaInfoHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMetaInfoHelper.java @@ -5,7 +5,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isGoogleURL; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; import com.grack.nanojson.JsonArray; @@ -67,17 +66,15 @@ public static List getMetaInfo(@Nonnull final JsonArray contents) private static MetaInfo getInfoPanelContent(@Nonnull final JsonObject infoPanelContentRenderer) throws ParsingException { final MetaInfo metaInfo = new MetaInfo(); - final StringBuilder sb = new StringBuilder(); - for (final Object paragraph : infoPanelContentRenderer.getArray("paragraphs")) { - if (sb.length() != 0) { - sb.append("
"); - } - sb.append(getTextFromObject((JsonObject) paragraph)); - } - metaInfo.setContent(new Description(sb.toString(), Description.HTML)); + final String description = infoPanelContentRenderer.getArray("paragraphs") + .streamAsJsonObjects() + .map(paragraph -> getTextFromObject(paragraph).orElse("")) + .collect(Collectors.joining("
")); + metaInfo.setContent(new Description(description, Description.HTML)); if (infoPanelContentRenderer.has("sourceEndpoint")) { final String metaInfoLinkUrl = getUrlFromNavigationEndpoint( - infoPanelContentRenderer.getObject("sourceEndpoint")); + infoPanelContentRenderer.getObject("sourceEndpoint")) + .orElse(""); try { metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded( metaInfoLinkUrl)))); @@ -85,11 +82,8 @@ private static MetaInfo getInfoPanelContent(@Nonnull final JsonObject infoPanelC throw new ParsingException("Could not get metadata info URL", e); } - final String metaInfoLinkText = getTextFromObject( - infoPanelContentRenderer.getObject("inlineSource")); - if (isNullOrEmpty(metaInfoLinkText)) { - throw new ParsingException("Could not get metadata info link text."); - } + final var source = infoPanelContentRenderer.getObject("inlineSource"); + final String metaInfoLinkText = getTextFromObjectOrThrow(source, "metadata info link"); metaInfo.addUrlText(metaInfoLinkText); } @@ -101,13 +95,10 @@ private static MetaInfo getClarificationRenderer( @Nonnull final JsonObject clarificationRenderer) throws ParsingException { final MetaInfo metaInfo = new MetaInfo(); - final String title = getTextFromObject(clarificationRenderer - .getObject("contentTitle")); - final String text = getTextFromObject(clarificationRenderer - .getObject("text")); - if (title == null || text == null) { - throw new ParsingException("Could not extract clarification renderer content"); - } + final String title = getTextFromObjectOrThrow( + clarificationRenderer.getObject("contentTitle"), "clarification renderer"); + final String text = getTextFromObjectOrThrow( + clarificationRenderer.getObject("text"), "clarification renderer"); metaInfo.setTitle(title); metaInfo.setContent(new Description(text, Description.PLAIN_TEXT)); @@ -116,31 +107,28 @@ private static MetaInfo getClarificationRenderer( .getObject("buttonRenderer"); try { final String url = getUrlFromNavigationEndpoint(actionButton - .getObject("command")); + .getObject("command")).orElse(""); metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded(url)))); } catch (final NullPointerException | MalformedURLException e) { throw new ParsingException("Could not get metadata info URL", e); } - final String metaInfoLinkText = getTextFromObject( - actionButton.getObject("text")); - if (isNullOrEmpty(metaInfoLinkText)) { - throw new ParsingException("Could not get metadata info link text."); - } - metaInfo.addUrlText(metaInfoLinkText); + final String linkText = getTextFromObjectOrThrow(actionButton.getObject("text"), + "metadata info link"); + metaInfo.addUrlText(linkText); } if (clarificationRenderer.has("secondaryEndpoint") && clarificationRenderer .has("secondarySource")) { final String url = getUrlFromNavigationEndpoint(clarificationRenderer - .getObject("secondaryEndpoint")); + .getObject("secondaryEndpoint")).orElse(null); // Ignore Google URLs, because those point to a Google search about "Covid-19" if (url != null && !isGoogleURL(url)) { try { metaInfo.addUrl(new URL(url)); - final String description = getTextFromObject(clarificationRenderer - .getObject("secondarySource")); - metaInfo.addUrlText(description == null ? url : description); + final String urlText = getTextFromObject(clarificationRenderer + .getObject("secondarySource")).orElse(url); + metaInfo.addUrlText(urlText); } catch (final MalformedURLException e) { throw new ParsingException("Could not get metadata info secondary URL", e); } @@ -201,10 +189,9 @@ private static void getEmergencyOneboxRenderer( metaInfo.addUrlText(urlText); // usually the webpage of the association - final String url = getUrlFromNavigationEndpoint(r.getObject("navigationEndpoint")); - if (url == null) { - throw new ParsingException("Could not extract emergency renderer url"); - } + final String url = getUrlFromNavigationEndpoint(r.getObject("navigationEndpoint")) + .orElseThrow(() -> + new ParsingException("Could not extract emergency renderer url")); try { metaInfo.addUrl(new URL(replaceHttpWithHttps(url))); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index b54441db5a..a8596811db 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -20,31 +20,12 @@ package org.schabi.newpipe.extractor.services.youtube; -import static org.schabi.newpipe.extractor.NewPipe.getDownloader; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_USER_AGENT_VERSION; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_USER_AGENT; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_HARDCODED_CLIENT_VERSION; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_CLIENT_ID; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_CLIENT_NAME; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_HARDCODED_CLIENT_VERSION; -import static org.schabi.newpipe.extractor.utils.Utils.HTTP; -import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; -import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonBuilder; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; - import org.jsoup.nodes.Entities; import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.Image.ResolutionLevel; @@ -63,6 +44,8 @@ import org.schabi.newpipe.extractor.utils.RandomStringFromAlphabetGenerator; import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -74,12 +57,28 @@ import java.util.Optional; import java.util.Random; import java.util.Set; +import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.NewPipe.getDownloader; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_USER_AGENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_USER_AGENT; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_HARDCODED_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_HARDCODED_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.utils.Utils.HTTP; +import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; +import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public final class YoutubeParsingHelper { @@ -197,6 +196,8 @@ private YoutubeParsingHelper() { private static boolean consentAccepted = false; + public static final Predicate STRING_PREDICATE = text -> !text.isBlank(); + public static boolean isGoogleURL(final String url) { final String cachedUrl = extractCachedUrlIfNeeded(url); try { @@ -553,8 +554,8 @@ private static String getClientVersionFromServiceTrackingParam( .map(JsonObject.class::cast) .filter(param -> param.getString("key", "") .equals(clientVersionKey)) - .map(param -> param.getString("value")) - .filter(paramValue -> !isNullOrEmpty(paramValue)) + .map(param -> param.getString("value", "")) + .filter(STRING_PREDICATE) .findFirst() .orElse(null); } @@ -687,103 +688,106 @@ public static String getYoutubeMusicClientVersion() return youtubeMusicClientVersion; } - @Nullable - public static String getUrlFromNavigationEndpoint( + @Nonnull + public static Optional getUrlFromNavigationEndpoint( @Nonnull final JsonObject navigationEndpoint) { - if (navigationEndpoint.has("urlEndpoint")) { - String internUrl = navigationEndpoint.getObject("urlEndpoint") - .getString("url"); - if (internUrl.startsWith("https://www.youtube.com/redirect?")) { - // remove https://www.youtube.com part to fall in the next if block - internUrl = internUrl.substring(23); - } - - if (internUrl.startsWith("/redirect?")) { - // q parameter can be the first parameter - internUrl = internUrl.substring(10); - final String[] params = internUrl.split("&"); - for (final String param : params) { - if (param.split("=")[0].equals("q")) { - return Utils.decodeUrlUtf8(param.split("=")[1]); + return Optional.ofNullable(navigationEndpoint.getObject("urlEndpoint") + .getString("url")) + .map(internUrl -> { + if (internUrl.startsWith("https://www.youtube.com/redirect?")) { + // remove https://www.youtube.com part to fall in the next if block + internUrl = internUrl.substring(23); } - } - } else if (internUrl.startsWith("http")) { - return internUrl; - } else if (internUrl.startsWith("/channel") || internUrl.startsWith("/user") - || internUrl.startsWith("/watch")) { - return "https://www.youtube.com" + internUrl; - } - } - - if (navigationEndpoint.has("browseEndpoint")) { - final JsonObject browseEndpoint = navigationEndpoint.getObject("browseEndpoint"); - final String canonicalBaseUrl = browseEndpoint.getString("canonicalBaseUrl"); - final String browseId = browseEndpoint.getString("browseId"); - - if (browseId != null) { - if (browseId.startsWith("UC")) { - // All channel IDs are prefixed with UC - return "https://www.youtube.com/channel/" + browseId; - } else if (browseId.startsWith("VL")) { - // All playlist IDs are prefixed with VL, which needs to be removed from the - // playlist ID - return "https://www.youtube.com/playlist?list=" + browseId.substring(2); - } - } - - if (!isNullOrEmpty(canonicalBaseUrl)) { - return "https://www.youtube.com" + canonicalBaseUrl; - } - } - - if (navigationEndpoint.has("watchEndpoint")) { - final StringBuilder url = new StringBuilder(); - url.append("https://www.youtube.com/watch?v=") - .append(navigationEndpoint.getObject("watchEndpoint") - .getString(VIDEO_ID)); - if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) { - url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint") - .getString("playlistId")); - } - if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) { - url.append("&t=") - .append(navigationEndpoint.getObject("watchEndpoint") - .getInt("startTimeSeconds")); - } - return url.toString(); - } - - if (navigationEndpoint.has("watchPlaylistEndpoint")) { - return "https://www.youtube.com/playlist?list=" - + navigationEndpoint.getObject("watchPlaylistEndpoint") - .getString("playlistId"); - } - - if (navigationEndpoint.has("showDialogCommand")) { - try { - final JsonArray listItems = JsonUtils.getArray(navigationEndpoint, - "showDialogCommand.panelLoadingStrategy.inlineContent.dialogViewModel" - + ".customContent.listViewModel.listItems"); - - // the first item seems to always be the channel that actually uploaded the video, - // i.e. it appears in their video feed - final JsonObject command = JsonUtils.getObject(listItems.getObject(0), - "listItemViewModel.rendererContext.commandContext.onTap.innertubeCommand"); - return getUrlFromNavigationEndpoint(command); - } catch (final ParsingException p) { - } - } + if (internUrl.startsWith("/redirect?")) { + // q parameter can be the first parameter + internUrl = internUrl.substring(10); + final String[] params = internUrl.split("&"); + for (final String param : params) { + final String[] nameAndValue = param.split("="); + if (nameAndValue[0].equals("q")) { + return Utils.decodeUrlUtf8(nameAndValue[1]); + } + } + } else if (internUrl.startsWith("http")) { + return internUrl; + } else if (internUrl.startsWith("/channel") || internUrl.startsWith("/user") + || internUrl.startsWith("/watch")) { + return "https://www.youtube.com" + internUrl; + } - if (navigationEndpoint.has("commandMetadata")) { - final JsonObject metadata = navigationEndpoint.getObject("commandMetadata") - .getObject("webCommandMetadata"); - if (metadata.has("url")) { - return "https://www.youtube.com" + metadata.getString("url"); - } - } + return null; + }) + .or(() -> Optional.ofNullable(navigationEndpoint.getObject("browseEndpoint", null)) + .map(browseEndpoint -> { + final var canonicalBaseUrl = + browseEndpoint.getString("canonicalBaseUrl"); + final var browseId = browseEndpoint.getString("browseId"); + + if (browseId != null) { + if (browseId.startsWith("UC")) { + // All channel IDs are prefixed with UC + return "https://www.youtube.com/channel/" + browseId; + } else if (browseId.startsWith("VL")) { + // All playlist IDs are prefixed with VL, which needs to be + // removed from the playlist ID + return "https://www.youtube.com/playlist?list=" + + browseId.substring(2); + } + } else if (!isNullOrEmpty(canonicalBaseUrl)) { + return "https://www.youtube.com" + canonicalBaseUrl; + } + return null; + })) + .or(() -> Optional.ofNullable(navigationEndpoint.getObject("watchEndpoint", null)) + .map(watchEndpoint -> { + final var videoId = watchEndpoint.getString(VIDEO_ID); + final var playlistId = watchEndpoint.getString("playlistId"); + final var startTime = watchEndpoint.getInt("startTimeSeconds", -1); + return "https://www.youtube.com/watch?v=" + videoId + + (playlistId != null ? "&list=" + playlistId : "") + + (startTime != -1 ? "&t=" + startTime : ""); + })) + .or(() -> { + final var playlistId = navigationEndpoint.getObject("watchPlaylistEndpoint") + .getString("playlistId"); + return Optional.ofNullable(playlistId) + .map(id -> "https://www.youtube.com/playlist?list=" + id); + }) + .or(() -> { + final var listItems = navigationEndpoint.getObject("showDialogCommand") + .getObject("panelLoadingStrategy").getObject("inlineContent") + .getObject("dialogViewModel").getObject("customContent") + .getObject("listViewModel") + .getArray("listItems"); + + // the first item seems to always be the channel that actually uploaded the + // video, i.e. it appears in their video feed + final var command = listItems.getObject(0).getObject("listItemViewModel") + .getObject("rendererContext").getObject("commandContext") + .getObject("onTap").getObject("innertubeCommand", null); + return Optional.ofNullable(command) + .flatMap(YoutubeParsingHelper::getUrlFromNavigationEndpoint); + }) + .filter(STRING_PREDICATE); + } - return null; + @Nonnull + public static Optional getMusicUploaderUrlFromMenu(@Nonnull final JsonObject object) { + final var items = object.getObject("menu") + .getObject("menuRenderer") + .getArray("items"); + return items.streamAsJsonObjects() + .flatMap(item -> { + final var renderer = item.getObject("menuNavigationItemRenderer"); + final var iconType = renderer.getObject("icon").getString("iconType"); + final var endpoint = renderer.getObject("navigationEndpoint"); + + return "ARTIST".equals(iconType) + ? getUrlFromNavigationEndpoint(endpoint).stream() + : Stream.empty(); + }) + .findFirst(); } /** @@ -792,125 +796,80 @@ public static String getUrlFromNavigationEndpoint( * * @param textObject JSON object to get the text from * @param html whether to return HTML, by parsing the {@code navigationEndpoint} - * @return text in the JSON object or {@code null} + * @return text in the JSON object as an {@link Optional} */ - @Nullable - public static String getTextFromObject(final JsonObject textObject, final boolean html) { - if (isNullOrEmpty(textObject)) { - return null; - } - - if (textObject.has("simpleText")) { - return textObject.getString("simpleText"); - } - - final JsonArray runs = textObject.getArray("runs"); - if (runs.isEmpty()) { - return null; - } - - final StringBuilder textBuilder = new StringBuilder(); - for (final Object o : runs) { - final JsonObject run = (JsonObject) o; - String text = run.getString("text"); - - if (html) { - if (run.has("navigationEndpoint")) { - final String url = getUrlFromNavigationEndpoint( - run.getObject("navigationEndpoint")); - if (!isNullOrEmpty(url)) { - text = "" + Entities.escape(text) - + ""; + @Nonnull + public static Optional getTextFromObject(@Nonnull final JsonObject textObject, + final boolean html) { + return Optional.ofNullable(textObject.getString("simpleText")) + .or(() -> { + final var runs = textObject.getArray("runs"); + final String text = runs.streamAsJsonObjects() + .map(run -> { + String textString = run.getString("text"); + + if (html) { + final String url = getUrlFromNavigationEndpoint( + run.getObject("navigationEndpoint")).orElse(null); + if (url != null) { + textString = "" + + Entities.escape(textString) + ""; + } + + if (run.getBoolean("strikethrough")) { + textString = "" + textString + ""; + } + if (run.getBoolean("italics")) { + textString = "" + textString + ""; + } + if (run.getBoolean("bold")) { + textString = "" + textString + ""; + } + } + + return textString; + }) + .collect(Collectors.joining()); + + final String string; + if (html) { + string = text.replace("\\n", "
") + .replaceAll(" {2}", "  "); + } else { + string = text; } - } - - final boolean bold = run.has("bold") - && run.getBoolean("bold"); - final boolean italic = run.has("italics") - && run.getBoolean("italics"); - final boolean strikethrough = run.has("strikethrough") - && run.getBoolean("strikethrough"); - - if (bold) { - textBuilder.append(""); - } - if (italic) { - textBuilder.append(""); - } - if (strikethrough) { - textBuilder.append(""); - } - - textBuilder.append(text); - - if (strikethrough) { - textBuilder.append(""); - } - if (italic) { - textBuilder.append(""); - } - if (bold) { - textBuilder.append(""); - } - } else { - textBuilder.append(text); - } - } - - String text = textBuilder.toString(); - - if (html) { - text = text.replaceAll("\\n", "
"); - text = text.replaceAll(" {2}", "  "); - } - - return text; + return Optional.of(string); + }) + .filter(STRING_PREDICATE); } @Nonnull public static String getTextFromObjectOrThrow(final JsonObject textObject, final String error) throws ParsingException { - final String result = getTextFromObject(textObject); - if (result == null) { - throw new ParsingException("Could not extract text: " + error); - } - return result; + return getTextFromObject(textObject) + .orElseThrow(() -> new ParsingException("Could not extract text: " + error)); } - @Nullable - public static String getTextFromObject(final JsonObject textObject) { + @Nonnull + public static Optional getTextFromObject(@Nonnull final JsonObject textObject) { return getTextFromObject(textObject, false); } - @Nullable - public static String getUrlFromObject(final JsonObject textObject) { - if (isNullOrEmpty(textObject)) { - return null; - } - - final JsonArray runs = textObject.getArray("runs"); - if (runs.isEmpty()) { - return null; - } - - for (final Object textPart : runs) { - final String url = getUrlFromNavigationEndpoint(((JsonObject) textPart) - .getObject("navigationEndpoint")); - if (!isNullOrEmpty(url)) { - return url; - } - } - - return null; + @Nonnull + public static Optional getUrlFromObject(@Nonnull final JsonObject textObject) { + return textObject.getArray("runs").streamAsJsonObjects() + .flatMap(textPart -> getUrlFromNavigationEndpoint(textPart + .getObject("navigationEndpoint")) + .stream()) + .findFirst(); } - @Nullable - public static String getTextAtKey(@Nonnull final JsonObject jsonObject, final String theKey) { - if (jsonObject.isString(theKey)) { - return jsonObject.getString(theKey); - } else { - return getTextFromObject(jsonObject.getObject(theKey)); - } + @Nonnull + public static Optional getTextAtKey(@Nonnull final JsonObject jsonObject, + final String theKey) { + return Optional.ofNullable(jsonObject.getString(theKey)) + .filter(STRING_PREDICATE) + .or(() -> getTextFromObject(jsonObject.getObject(theKey))); } public static String fixThumbnailUrl(@Nonnull final String thumbnailUrl) { @@ -1238,7 +1197,8 @@ public static void defaultAlertsCheck(@Nonnull final JsonObject initialData) final JsonArray alerts = initialData.getArray("alerts"); if (!isNullOrEmpty(alerts)) { final JsonObject alertRenderer = alerts.getObject(0).getObject("alertRenderer"); - final String alertText = getTextFromObject(alertRenderer.getObject("text")); + final String alertText = getTextFromObject(alertRenderer.getObject("text")) + .orElse(null); final String alertType = alertRenderer.getString("type", ""); if (alertType.equalsIgnoreCase("ERROR")) { if (alertText != null @@ -1295,36 +1255,25 @@ public static String extractCachedUrlIfNeeded(final String url) { return url; } - public static boolean isVerified(final JsonArray badges) { - if (Utils.isNullOrEmpty(badges)) { - return false; - } - - for (final Object badge : badges) { - final String style = ((JsonObject) badge).getObject("metadataBadgeRenderer") - .getString("style"); - if (style != null && (style.equals("BADGE_STYLE_TYPE_VERIFIED") - || style.equals("BADGE_STYLE_TYPE_VERIFIED_ARTIST"))) { - return true; - } - } - - return false; + public static boolean isVerified(@Nonnull final JsonArray badges) { + return badges.streamAsJsonObjects() + .anyMatch(badge -> { + final String style = badge.getObject("metadataBadgeRenderer") + .getString("style"); + return "BADGE_STYLE_TYPE_VERIFIED".equals(style) + || "BADGE_STYLE_TYPE_VERIFIED_ARTIST".equals(style); + }); } public static boolean hasArtistOrVerifiedIconBadgeAttachment( @Nonnull final JsonArray attachmentRuns) { - return attachmentRuns.stream() - .filter(JsonObject.class::isInstance) - .map(JsonObject.class::cast) + return attachmentRuns.streamAsJsonObjects() .anyMatch(attachmentRun -> attachmentRun.getObject("element") .getObject("type") .getObject("imageType") .getObject("image") .getArray("sources") - .stream() - .filter(JsonObject.class::isInstance) - .map(JsonObject.class::cast) + .streamAsJsonObjects() .anyMatch(source -> { final String imageName = source.getObject("clientResource") .getString("imageName"); @@ -1593,19 +1542,17 @@ public static JsonBuilder prepareJsonBuilder( * Gets the first collaborator, which is the channel that owns the video, * i.e. the video is displayed on their channel page. * - * @param navigationEndpoint JSON object for the navigationEndpoint - * @return The first collaborator in the JSON object or {@code null} + * @param renderer JSON object for the video renderer + * @return An {@link Optional} containing the first collaborator, if one is present */ - @Nullable - public static JsonObject getFirstCollaborator(final JsonObject navigationEndpoint) - throws ParsingException { - try { - // CHECKSTYLE:OFF - final JsonArray listItems = JsonUtils.getArray(navigationEndpoint, "showDialogCommand.panelLoadingStrategy.inlineContent.dialogViewModel.customContent.listViewModel.listItems"); - // CHECKSTYLE:ON - return listItems.getObject(0).getObject("listItemViewModel"); - } catch (final ParsingException e) { - return null; - } + @Nonnull + public static Optional getFirstCollaborator(final JsonObject renderer) { + final JsonArray listItems = renderer.getObject("navigationEndpoint") + .getObject("showDialogCommand").getObject("panelLoadingStrategy") + .getObject("inlineContent").getObject("dialogViewModel") + .getObject("customContent").getObject("listViewModel") + .getArray("listItems"); + return Optional.ofNullable(listItems.getObject(0) + .getObject("listItemViewModel", null)); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBaseShowInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBaseShowInfoItemExtractor.java index 67254302fc..8499abe12e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBaseShowInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBaseShowInfoItemExtractor.java @@ -9,7 +9,7 @@ import javax.annotation.Nonnull; import java.util.List; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; @@ -32,7 +32,8 @@ public String getName() throws ParsingException { @Override public String getUrl() throws ParsingException { - return getUrlFromNavigationEndpoint(showRenderer.getObject("navigationEndpoint")); + return getUrlFromNavigationEndpoint(showRenderer.getObject("navigationEndpoint")) + .orElse(null); } @Nonnull @@ -46,13 +47,10 @@ public List getThumbnails() throws ParsingException { public long getStreamCount() throws ParsingException { // The stream count should be always returned in the first text object for English // localizations, but the complete text is parsed for reliability purposes - final String streamCountText = getTextFromObject( - showRenderer.getObject("thumbnailOverlays") - .getObject("thumbnailOverlayBottomPanelRenderer") - .getObject("text")); - if (streamCountText == null) { - throw new ParsingException("Could not get stream count"); - } + final var textObject = showRenderer.getObject("thumbnailOverlays") + .getObject("thumbnailOverlayBottomPanelRenderer") + .getObject("text"); + final String streamCountText = getTextFromObjectOrThrow(textObject, "stream count"); try { // The data returned could be a human/shortened number, but no show with more than 1000 diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 147345526a..92fe80428d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -257,7 +257,7 @@ public long getSubscriberCount() throws ParsingException { if (textObject != null) { try { - return Utils.mixedNumberWordToLong(getTextFromObject(textObject)); + return Utils.mixedNumberWordToLong(getTextFromObject(textObject).orElse("")); } catch (final NumberFormatException e) { throw new ParsingException("Could not get subscriber count", e); } @@ -321,7 +321,8 @@ public String getDescription() throws ParsingException { The description extracted is incomplete and the original one can be only accessed from the About tab */ - return getTextFromObject(channelHeader.json.getObject("description")); + return getTextFromObject(channelHeader.json.getObject("description")) + .orElse(null); } return jsonResponse.getObject(METADATA) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 38861f4e51..cc22b92275 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -47,14 +47,9 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor public YoutubeChannelInfoItemExtractor(final JsonObject channelInfoItem) { this.channelInfoItem = channelInfoItem; - - boolean wHandle = false; - final String subscriberCountText = getTextFromObject( - channelInfoItem.getObject("subscriberCountText")); - if (subscriberCountText != null) { - wHandle = subscriberCountText.startsWith("@"); - } - this.withHandle = wHandle; + this.withHandle = getTextFromObject(channelInfoItem.getObject("subscriberCountText")) + .map(text -> text.startsWith("@")) + .orElse(false); } @Nonnull @@ -70,7 +65,7 @@ public List getThumbnails() throws ParsingException { @Override public String getName() throws ParsingException { try { - return getTextFromObject(channelInfoItem.getObject("title")); + return getTextFromObject(channelInfoItem.getObject("title")).orElse(null); } catch (final Exception e) { throw new ParsingException("Could not get name", e); } @@ -97,14 +92,14 @@ public long getSubscriberCount() throws ParsingException { if (withHandle) { if (channelInfoItem.has("videoCountText")) { return Utils.mixedNumberWordToLong(getTextFromObject( - channelInfoItem.getObject("videoCountText"))); + channelInfoItem.getObject("videoCountText")).orElse("")); } else { return -1; } } return Utils.mixedNumberWordToLong(getTextFromObject( - channelInfoItem.getObject("subscriberCountText"))); + channelInfoItem.getObject("subscriberCountText")).orElse("")); } catch (final Exception e) { throw new ParsingException("Could not get subscriber count", e); } @@ -120,7 +115,7 @@ public long getStreamCount() throws ParsingException { } return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject( - channelInfoItem.getObject("videoCountText")))); + channelInfoItem.getObject("videoCountText")).orElse(""))); } catch (final Exception e) { throw new ParsingException("Could not get stream count", e); } @@ -139,7 +134,8 @@ public String getDescription() throws ParsingException { return null; } - return getTextFromObject(channelInfoItem.getObject("descriptionSnippet")); + return getTextFromObject(channelInfoItem.getObject("descriptionSnippet")) + .orElse(null); } catch (final Exception e) { throw new ParsingException("Could not get description", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 8667768a4b..c7fbffce3e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -393,7 +393,7 @@ public int getCommentsCount() throws ExtractionException { try { return Integer.parseInt( - Utils.removeNonDigitCharacters(getTextFromObject(countText)) + Utils.removeNonDigitCharacters(getTextFromObject(countText).orElse("")) ); } catch (final Exception e) { throw new ExtractionException("Unable to get comments count", e); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java index ddc7b7bcc0..45371f9035 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java @@ -66,7 +66,8 @@ public List getThumbnails() throws ParsingException { @Override public String getName() throws ParsingException { try { - return getTextFromObject(JsonUtils.getObject(commentRenderer, "authorText")); + return getTextFromObject(JsonUtils.getObject(commentRenderer, "authorText")) + .orElse(""); } catch (final Exception e) { return ""; } @@ -75,8 +76,8 @@ public String getName() throws ParsingException { @Override public String getTextualUploadDate() throws ParsingException { try { - return getTextFromObject(JsonUtils.getObject(commentRenderer, - "publishedTimeText")); + return getTextFromObject(JsonUtils.getObject(commentRenderer, "publishedTimeText")) + .orElse(null); } catch (final Exception e) { throw new ParsingException("Could not get publishedTimeText", e); } @@ -170,10 +171,7 @@ public String getTextualLikeCount() throws ParsingException { } final JsonObject voteCountObj = JsonUtils.getObject(commentRenderer, "voteCount"); - if (voteCountObj.isEmpty()) { - return ""; - } - return getTextFromObject(voteCountObj); + return getTextFromObject(voteCountObj).orElse(""); } catch (final Exception e) { throw new ParsingException("Could not get the vote count", e); } @@ -189,7 +187,7 @@ public Description getCommentText() throws ParsingException { // https://github.com/TeamNewPipe/NewPipeExtractor/issues/380#issuecomment-668808584 return Description.EMPTY_DESCRIPTION; } - final String commentText = getTextFromObject(contentText, true); + final String commentText = getTextFromObject(contentText, true).orElse(""); // YouTube adds U+FEFF in some comments. // eg. https://www.youtube.com/watch?v=Nj4F63E59io final String commentTextBomRemoved = Utils.removeUTF8BOM(commentText); @@ -235,7 +233,8 @@ public boolean isUploaderVerified() throws ParsingException { @Override public String getUploaderName() throws ParsingException { try { - return getTextFromObject(JsonUtils.getObject(commentRenderer, "authorText")); + return getTextFromObject(JsonUtils.getObject(commentRenderer, "authorText")) + .orElse(""); } catch (final Exception e) { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java index 16c7a3e3ea..48cc1fab97 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistInfoItemExtractor.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -26,11 +26,7 @@ public YoutubeMixOrPlaylistInfoItemExtractor(final JsonObject mixInfoItem) { @Override public String getName() throws ParsingException { - final String name = getTextFromObject(mixInfoItem.getObject("title")); - if (isNullOrEmpty(name)) { - throw new ParsingException("Could not get name"); - } - return name; + return getTextFromObjectOrThrow(mixInfoItem.getObject("title"), "name"); } @Override @@ -51,7 +47,8 @@ public List getThumbnails() throws ParsingException { @Override public String getUploaderName() throws ParsingException { // this will be a list of uploaders for mixes - return YoutubeParsingHelper.getTextFromObject(mixInfoItem.getObject("longBylineText")); + return YoutubeParsingHelper.getTextFromObject(mixInfoItem.getObject("longBylineText")) + .orElse(null); } @Override @@ -68,11 +65,9 @@ public boolean isUploaderVerified() throws ParsingException { @Override public long getStreamCount() throws ParsingException { - final String countString = YoutubeParsingHelper.getTextFromObject( - mixInfoItem.getObject("videoCountShortText")); - if (countString == null) { - throw new ParsingException("Could not extract item count for playlist/mix info item"); - } + final var textObject = mixInfoItem.getObject("videoCountShortText"); + final String countString = getTextFromObjectOrThrow(textObject, + "item count for playlist/mix info item"); try { return Integer.parseInt(countString); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistLockupInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistLockupInfoItemExtractor.java index 009fc51d0c..bf33c4d38f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistLockupInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixOrPlaylistLockupInfoItemExtractor.java @@ -87,7 +87,8 @@ public String getUploaderUrl() throws ParsingException { .getArray("commandRuns") .getObject(0) .getObject("onTap") - .getObject("innertubeCommand")); + .getObject("innertubeCommand")) + .orElse(null); } @Override @@ -159,7 +160,7 @@ public String getUrl() throws ParsingException { return getUrlFromNavigationEndpoint(lockupViewModel.getObject("rendererContext") .getObject("commandContext") .getObject("onTap") - .getObject("innertubeCommand")); + .getObject("innertubeCommand")).orElse(null); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 1df90f967f..7f6c40c94a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -126,11 +126,8 @@ public void onFetchPage(@Nonnull final Downloader downloader) @Nonnull @Override public String getName() throws ParsingException { - final String name = YoutubeParsingHelper.getTextAtKey(playlistData, "title"); - if (isNullOrEmpty(name)) { - throw new ParsingException("Could not get playlist name"); - } - return name; + return YoutubeParsingHelper.getTextAtKey(playlistData, "title") + .orElseThrow(() -> new ParsingException("Could not get playlist name")); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java index 1b3fccfb27..138043f428 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicAlbumOrPlaylistInfoItemExtractor.java @@ -11,8 +11,9 @@ import java.util.List; import static org.schabi.newpipe.extractor.ListExtractor.ITEM_COUNT_UNKNOWN; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getMusicUploaderUrlFromMenu; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -49,16 +50,11 @@ public List getThumbnails() throws ParsingException { @Override public String getName() throws ParsingException { - final String name = getTextFromObject(albumOrPlaylistInfoItem.getArray("flexColumns") + final var textObject = albumOrPlaylistInfoItem.getArray("flexColumns") .getObject(0) .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text")); - - if (!isNullOrEmpty(name)) { - return name; - } - - throw new ParsingException("Could not get name"); + .getObject("text"); + return getTextFromObjectOrThrow(textObject, "name"); } @Override @@ -105,28 +101,13 @@ public String getUploaderName() throws ParsingException { @Override public String getUploaderUrl() throws ParsingException { // first try obtaining the uploader from the menu (will not work for MUSIC_PLAYLISTS though) - final JsonArray items = albumOrPlaylistInfoItem.getObject("menu") - .getObject("menuRenderer") - .getArray("items"); - for (final Object item : items) { - final JsonObject menuNavigationItemRenderer = - ((JsonObject) item).getObject("menuNavigationItemRenderer"); - if (menuNavigationItemRenderer.getObject("icon") - .getString("iconType", "") - .equals("ARTIST")) { - return getUrlFromNavigationEndpoint( - menuNavigationItemRenderer.getObject("navigationEndpoint")); - } - } - - // then try obtaining it from the uploader description element - if (!descriptionElementUploader.has("navigationEndpoint")) { - // if there is no navigationEndpoint for the uploader - // then this playlist/album is likely autogenerated - return null; - } - return getUrlFromNavigationEndpoint( - descriptionElementUploader.getObject("navigationEndpoint")); + return getMusicUploaderUrlFromMenu(albumOrPlaylistInfoItem) + // then try obtaining it from the uploader description element + .or(() -> getUrlFromNavigationEndpoint(descriptionElementUploader + .getObject("navigationEndpoint"))) + // if there is no navigationEndpoint for the uploader + // then this playlist/album is likely autogenerated + .orElse(null); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java index 4fe2dc666f..4251fbc5bd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java @@ -12,7 +12,7 @@ import javax.annotation.Nullable; import java.util.List; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -40,24 +40,17 @@ public List getThumbnails() throws ParsingException { @Override public String getName() throws ParsingException { - final String name = getTextFromObject(artistInfoItem.getArray("flexColumns") + final var jsonObject = artistInfoItem.getArray("flexColumns") .getObject(0) .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text")); - if (!isNullOrEmpty(name)) { - return name; - } - throw new ParsingException("Could not get name"); + .getObject("text"); + return getTextFromObjectOrThrow(jsonObject, "name"); } @Override public String getUrl() throws ParsingException { - final String url = getUrlFromNavigationEndpoint( - artistInfoItem.getObject("navigationEndpoint")); - if (!isNullOrEmpty(url)) { - return url; - } - throw new ParsingException("Could not get URL"); + return getUrlFromNavigationEndpoint(artistInfoItem.getObject("navigationEndpoint")) + .orElseThrow(() -> new ParsingException("Could not get URL")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index a116eb312e..e6a4079a7e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -151,7 +151,8 @@ public String getSearchSuggestion() throws ParsingException { .getObject("didYouMeanRenderer"); if (!didYouMeanRenderer.isEmpty()) { - return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); + return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")) + .orElse(""); } // NOTE: As of 2025-07 "showing results for ..." doesn't seem to be returned by diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java index 11b220288c..35917776c4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSongOrVideoInfoItemExtractor.java @@ -14,8 +14,9 @@ import javax.annotation.Nonnull; import java.util.List; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getMusicUploaderUrlFromMenu; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; @@ -46,14 +47,11 @@ public String getUrl() throws ParsingException { @Override public String getName() throws ParsingException { - final String name = getTextFromObject(songOrVideoInfoItem.getArray("flexColumns") + final var textObject = songOrVideoInfoItem.getArray("flexColumns") .getObject(0) .getObject("musicResponsiveListItemFlexColumnRenderer") - .getObject("text")); - if (!isNullOrEmpty(name)) { - return name; - } - throw new ParsingException("Could not get name"); + .getObject("text"); + return getTextFromObjectOrThrow(textObject, "name"); } @Override @@ -88,41 +86,22 @@ public String getUploaderName() throws ParsingException { @Override public String getUploaderUrl() throws ParsingException { if (searchType.equals(MUSIC_VIDEOS)) { - final JsonArray items = songOrVideoInfoItem.getObject("menu") - .getObject("menuRenderer") - .getArray("items"); - for (final Object item : items) { - final JsonObject menuNavigationItemRenderer = - ((JsonObject) item).getObject("menuNavigationItemRenderer"); - if (menuNavigationItemRenderer.getObject("icon") - .getString("iconType", "") - .equals("ARTIST")) { - return getUrlFromNavigationEndpoint( - menuNavigationItemRenderer.getObject("navigationEndpoint")); - } - } - - return null; + return getMusicUploaderUrlFromMenu(songOrVideoInfoItem).orElse(null); } else { - final JsonObject navigationEndpointHolder = songOrVideoInfoItem.getArray("flexColumns") + final var endpoint = songOrVideoInfoItem.getArray("flexColumns") .getObject(1) .getObject("musicResponsiveListItemFlexColumnRenderer") .getObject("text") .getArray("runs") - .getObject(0); + .getObject(0) + .getObject("navigationEndpoint"); - if (!navigationEndpointHolder.has("navigationEndpoint")) { + if (!endpoint.isEmpty()) { + return getUrlFromNavigationEndpoint(endpoint) + .orElseThrow(() -> new ParsingException("Could not get uploader URL")); + } else { return null; } - - final String url = getUrlFromNavigationEndpoint( - navigationEndpointHolder.getObject("navigationEndpoint")); - - if (!isNullOrEmpty(url)) { - return url; - } - - throw new ParsingException("Could not get uploader URL"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 3e2fd89d5e..210a82a464 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -37,6 +37,7 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.List; +import java.util.Optional; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -178,14 +179,10 @@ private JsonObject getPlaylistHeader() { @Nonnull @Override public String getName() throws ParsingException { - final String name = getTextFromObject(getPlaylistInfo().getObject("title")); - if (!isNullOrEmpty(name)) { - return name; - } - - return browseMetadataResponse.getObject(MICROFORMAT) - .getObject("microformatDataRenderer") - .getString("title"); + return getTextFromObject(getPlaylistInfo().getObject("title")) + .orElseGet(() -> browseMetadataResponse.getObject(MICROFORMAT) + .getObject("microformatDataRenderer") + .getString("title")); } @Nonnull @@ -229,7 +226,7 @@ public String getUploaderUrl() throws ParsingException { .getArray("runs") .getObject(0) .getObject("navigationEndpoint") - : getUploaderInfo().getObject("navigationEndpoint")); + : getUploaderInfo().getObject("navigationEndpoint")).orElse(null); } catch (final Exception e) { throw new ParsingException("Could not get playlist uploader url", e); } @@ -240,7 +237,7 @@ public String getUploaderName() throws ParsingException { try { return getTextFromObject(isNewPlaylistInterface ? getPlaylistHeader().getObject("ownerText") - : getUploaderInfo().getObject("title")); + : getUploaderInfo().getObject("title")).orElse(null); } catch (final Exception e) { throw new ParsingException("Could not get playlist uploader name", e); } @@ -270,60 +267,32 @@ public boolean isUploaderVerified() throws ParsingException { @Override public long getStreamCount() throws ParsingException { - if (isNewPlaylistInterface) { - final String numVideosText = - getTextFromObject(getPlaylistHeader().getObject("numVideosText")); - if (numVideosText != null) { - try { - return Long.parseLong(Utils.removeNonDigitCharacters(numVideosText)); - } catch (final NumberFormatException ignored) { - } - } - - final String firstByLineRendererText = getTextFromObject( - getPlaylistHeader().getArray("byline") - .getObject(0) - .getObject("text")); - - if (firstByLineRendererText != null) { - try { - return Long.parseLong(Utils.removeNonDigitCharacters(firstByLineRendererText)); - } catch (final NumberFormatException ignored) { - } - } - } - - // These data structures are returned in both layouts - final JsonArray briefStats = - (isNewPlaylistInterface ? getPlaylistHeader() : getPlaylistInfo()) - .getArray("briefStats"); - if (!briefStats.isEmpty()) { - final String briefsStatsText = getTextFromObject(briefStats.getObject(0)); - if (briefsStatsText != null) { - return Long.parseLong(Utils.removeNonDigitCharacters(briefsStatsText)); - } - } - - final JsonArray stats = (isNewPlaylistInterface ? getPlaylistHeader() : getPlaylistInfo()) - .getArray("stats"); - if (!stats.isEmpty()) { - final String statsText = getTextFromObject(stats.getObject(0)); - if (statsText != null) { - return Long.parseLong(Utils.removeNonDigitCharacters(statsText)); - } - } - - return ITEM_COUNT_UNKNOWN; + final var header = getPlaylistHeader(); + final Optional count = isNewPlaylistInterface + ? getTextFromObject(header.getObject("numVideosText")) + .or(() -> getTextFromObject(header.getArray("byline") + .getObject(0).getObject("text"))) + : Optional.empty(); + final var playlist = isNewPlaylistInterface ? header : getPlaylistInfo(); + + // "briefStats" and "stats" are returned in both layouts + return count.or(() -> getTextFromObject(playlist.getArray("briefStats").getObject(0))) + .or(() -> getTextFromObject(playlist.getArray("stats").getObject(0))) + .map(numText -> { + try { + return Long.parseLong(Utils.removeNonDigitCharacters(numText)); + } catch (final NumberFormatException e) { + return null; + } + }) + .orElse(ITEM_COUNT_UNKNOWN); } @Nonnull @Override public Description getDescription() throws ParsingException { - final String description = getTextFromObject( - getPlaylistInfo().getObject("description"), - true - ); - + final var descriptionObj = getPlaylistInfo().getObject("description"); + final String description = getTextFromObject(descriptionObj, true).orElse(null); return new Description(description, Description.HTML); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java index a0584a20fb..3392bdc129 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java @@ -12,6 +12,7 @@ import javax.annotation.Nonnull; import java.util.List; +import java.util.Optional; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; @@ -45,7 +46,7 @@ public List getThumbnails() throws ParsingException { @Override public String getName() throws ParsingException { try { - return getTextFromObject(playlistInfoItem.getObject("title")); + return getTextFromObject(playlistInfoItem.getObject("title")).orElse(null); } catch (final Exception e) { throw new ParsingException("Could not get name", e); } @@ -64,7 +65,7 @@ public String getUrl() throws ParsingException { @Override public String getUploaderName() throws ParsingException { try { - return getTextFromObject(playlistInfoItem.getObject("longBylineText")); + return getTextFromObject(playlistInfoItem.getObject("longBylineText")).orElse(null); } catch (final Exception e) { throw new ParsingException("Could not get uploader name", e); } @@ -73,7 +74,7 @@ public String getUploaderName() throws ParsingException { @Override public String getUploaderUrl() throws ParsingException { try { - return getUrlFromObject(playlistInfoItem.getObject("longBylineText")); + return getUrlFromObject(playlistInfoItem.getObject("longBylineText")).orElse(null); } catch (final Exception e) { throw new ParsingException("Could not get uploader url", e); } @@ -90,18 +91,10 @@ public boolean isUploaderVerified() throws ParsingException { @Override public long getStreamCount() throws ParsingException { - String videoCountText = playlistInfoItem.getString("videoCount"); - if (videoCountText == null) { - videoCountText = getTextFromObject(playlistInfoItem.getObject("videoCountText")); - } - - if (videoCountText == null) { - videoCountText = getTextFromObject(playlistInfoItem.getObject("videoCountShortText")); - } - - if (videoCountText == null) { - throw new ParsingException("Could not get stream count"); - } + final var videoCountText = Optional.ofNullable(playlistInfoItem.getString("videoCount")) + .or(() -> getTextFromObject(playlistInfoItem.getObject("videoCountText"))) + .or(() -> getTextFromObject(playlistInfoItem.getObject("videoCountShortText"))) + .orElseThrow(() -> new ParsingException("Could not get stream count")); try { return Long.parseLong(Utils.removeNonDigitCharacters(videoCountText)); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java index 71c7440b42..1dcb890183 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeReelInfoItemExtractor.java @@ -1,8 +1,8 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonObject; @@ -45,7 +45,7 @@ public YoutubeReelInfoItemExtractor(@Nonnull final JsonObject reelInfo) { @Override public String getName() throws ParsingException { - return getTextFromObject(reelInfo.getObject("headline")); + return getTextFromObject(reelInfo.getObject("headline")).orElse(null); } @Override @@ -71,17 +71,13 @@ public StreamType getStreamType() throws ParsingException { @Override public long getViewCount() throws ParsingException { - final String viewCountText = getTextFromObject(reelInfo.getObject("viewCountText")); - if (!isNullOrEmpty(viewCountText)) { - // This approach is language dependent - if (viewCountText.toLowerCase().contains("no views")) { - return 0; - } - - return Utils.mixedNumberWordToLong(viewCountText); + final String viewCountText = getTextFromObjectOrThrow( + reelInfo.getObject("viewCountText"), "short view count"); + // This approach is language dependent + if (viewCountText.toLowerCase().contains("no views")) { + return 0; } - - throw new ParsingException("Could not get short view count"); + return Utils.mixedNumberWordToLong(viewCountText); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index e888787cf9..ec12b8820c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -35,7 +35,6 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.Objects; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -130,11 +129,11 @@ public String getSearchSuggestion() throws ParsingException { "correctedQueryEndpoint.searchEndpoint.query"); } - return Objects.requireNonNullElse( - getTextFromObject(itemSectionRenderer.getArray("contents") - .getObject(0) - .getObject("showingResultsForRenderer") - .getObject("correctedQuery")), ""); + final var query = itemSectionRenderer.getArray("contents") + .getObject(0) + .getObject("showingResultsForRenderer") + .getObject("correctedQuery"); + return getTextFromObject(query).orElse(""); } @Override @@ -231,7 +230,7 @@ private void collectStreamsFrom(final MultiInfoItemsCollector collector, if (item.has("backgroundPromoRenderer")) { throw new NothingFoundException( getTextFromObject(item.getObject("backgroundPromoRenderer") - .getObject("bodyText"))); + .getObject("bodyText")).orElse("")); } else if (item.has("videoRenderer") && extractVideoResults) { collector.commit(new YoutubeStreamInfoItemExtractor( item.getObject("videoRenderer"), timeAgoParser)); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeShowRendererInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeShowRendererInfoItemExtractor.java index c7119907e0..0c2f02c5d8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeShowRendererInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeShowRendererInfoItemExtractor.java @@ -7,7 +7,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromObject; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; /** * A {@link YoutubeBaseShowInfoItemExtractor} implementation for {@code showRenderer}s. @@ -27,26 +26,16 @@ class YoutubeShowRendererInfoItemExtractor extends YoutubeBaseShowInfoItemExtrac @Override public String getUploaderName() throws ParsingException { - String name = getTextFromObject(longBylineText); - if (isNullOrEmpty(name)) { - name = getTextFromObject(shortBylineText); - if (isNullOrEmpty(name)) { - throw new ParsingException("Could not get uploader name"); - } - } - return name; + return getTextFromObject(longBylineText) + .or(() -> getTextFromObject(shortBylineText)) + .orElseThrow(() -> new ParsingException("Could not get uploader name")); } @Override public String getUploaderUrl() throws ParsingException { - String uploaderUrl = getUrlFromObject(longBylineText); - if (uploaderUrl == null) { - uploaderUrl = getUrlFromObject(shortBylineText); - if (uploaderUrl == null) { - throw new ParsingException("Could not get uploader URL"); - } - } - return uploaderUrl; + return getUrlFromObject(longBylineText) + .or(() -> getUrlFromObject(shortBylineText)) + .orElseThrow(() -> new ParsingException("Could not get uploader URL")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index ba23eb6463..7cf5c89921 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -20,25 +20,9 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import static org.schabi.newpipe.extractor.services.youtube.ItagItem.APPROX_DURATION_MS_UNKNOWN; -import static org.schabi.newpipe.extractor.services.youtube.ItagItem.CONTENT_LENGTH_UNKNOWN; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeDescriptionHelper.attributedDescriptionToHtml; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONTENT_CHECK_OK; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateContentPlaybackNonce; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; - import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.MetaInfo; @@ -52,8 +36,8 @@ import org.schabi.newpipe.extractor.exceptions.PaidContentException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.PrivateContentException; -import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException; import org.schabi.newpipe.extractor.exceptions.SignInConfirmNotBotException; +import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.DateWrapper; @@ -84,6 +68,8 @@ import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.LocalDate; @@ -98,8 +84,20 @@ import java.util.Optional; import java.util.stream.Collectors; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.services.youtube.ItagItem.APPROX_DURATION_MS_UNKNOWN; +import static org.schabi.newpipe.extractor.services.youtube.ItagItem.CONTENT_LENGTH_UNKNOWN; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeDescriptionHelper.attributedDescriptionToHtml; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONTENT_CHECK_OK; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateContentPlaybackNonce; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class YoutubeStreamExtractor extends StreamExtractor { private static final String PREMIERED = "Premiered "; @@ -153,20 +151,10 @@ public YoutubeStreamExtractor(final StreamingService service, final LinkHandler @Override public String getName() throws ParsingException { assertPageFetched(); - String title; - // Try to get the video's original title, which is untranslated - title = playerResponse.getObject("videoDetails").getString("title"); - - if (isNullOrEmpty(title)) { - title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title")); - - if (isNullOrEmpty(title)) { - throw new ParsingException("Could not get name"); - } - } - - return title; + return Optional.ofNullable(playerResponse.getObject("videoDetails").getString("title")) + .or(() -> getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title"))) + .orElseThrow(() -> new ParsingException("Could not get name")); } @Nullable @@ -193,18 +181,19 @@ public String getTextualUploadDate() { return null; } - final var textObject = getVideoPrimaryInfoRenderer().getObject("dateText"); - final String rendererDateText = getTextFromObject(textObject); - if (rendererDateText == null) { - return null; - } else if (rendererDateText.startsWith(PREMIERED_ON)) { // Premiered on 21 Feb 2020 - return rendererDateText.substring(PREMIERED_ON.length()); - } else if (rendererDateText.startsWith(PREMIERED)) { - // Premiered 20 hours ago / Premiered Feb 21, 2020 - return rendererDateText.substring(PREMIERED.length()); - } else { - return rendererDateText; - } + return getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")) + .map(rendererDateText -> { + if (rendererDateText.startsWith(PREMIERED_ON)) { + // Premiered on 21 Feb 2020 + return rendererDateText.substring(PREMIERED_ON.length()); + } else if (rendererDateText.startsWith(PREMIERED)) { + // Premiered 20 hours ago / Premiered Feb 21, 2020 + return rendererDateText.substring(PREMIERED.length()); + } else { + return rendererDateText; + } + }) + .orElse(null); } @Override @@ -256,31 +245,26 @@ public List getThumbnails() throws ParsingException { @Nonnull @Override - public Description getDescription() throws ParsingException { + public Description getDescription() { assertPageFetched(); // Description with more info on links - final String videoSecondaryInfoRendererDescription = getTextFromObject( - getVideoSecondaryInfoRenderer().getObject("description"), - true); - if (!isNullOrEmpty(videoSecondaryInfoRendererDescription)) { - return new Description(videoSecondaryInfoRendererDescription, Description.HTML); - } - - final String attributedDescription = attributedDescriptionToHtml( - getVideoSecondaryInfoRenderer().getObject("attributedDescription")); - if (!isNullOrEmpty(attributedDescription)) { - return new Description(attributedDescription, Description.HTML); - } - - String description = playerResponse.getObject("videoDetails") - .getString("shortDescription"); - if (description == null) { - final JsonObject descriptionObject = playerMicroFormatRenderer.getObject("description"); - description = getTextFromObject(descriptionObject); - } - - // Raw non-html description - return new Description(description, Description.PLAIN_TEXT); + final var renderer = getVideoSecondaryInfoRenderer(); + return getTextFromObject(renderer.getObject("description"), true) + .or(() -> { + final var description = renderer.getObject("attributedDescription"); + return Optional.ofNullable(attributedDescriptionToHtml(description)); + }) + .map(description -> new Description(description, Description.HTML)) + .orElseGet(() -> { + final var shortDescription = playerResponse.getObject("videoDetails") + .getString("shortDescription"); + final String description = Optional.ofNullable(shortDescription) + .or(() -> getTextFromObject(playerMicroFormatRenderer + .getObject("description"))) + .orElse(null); + // Raw non-html description + return new Description(description, Description.PLAIN_TEXT); + }); } @Override @@ -370,16 +354,11 @@ public long getTimeStamp() throws ParsingException { @Override public long getViewCount() throws ParsingException { - String views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount") - .getObject("videoViewCountRenderer").getObject("viewCount")); - - if (isNullOrEmpty(views)) { - views = playerResponse.getObject("videoDetails").getString("viewCount"); - - if (isNullOrEmpty(views)) { - throw new ParsingException("Could not get view count"); - } - } + final var views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount") + .getObject("videoViewCountRenderer").getObject("viewCount")) + .or(() -> Optional.ofNullable(playerResponse.getObject("videoDetails") + .getString("viewCount"))) + .orElseThrow(() -> new ParsingException("Could not get view count")); if (views.toLowerCase().contains("no views")) { return 0; @@ -402,109 +381,79 @@ public long getLikeCount() throws ParsingException { .getObject("menuRenderer") .getArray("topLevelButtons"); - try { - return parseLikeCountFromLikeButtonViewModel(topLevelButtons); - } catch (final ParsingException ignored) { - // A segmentedLikeDislikeButtonRenderer could be returned instead of a - // segmentedLikeDislikeButtonViewModel, so ignore extraction errors relative to - // segmentedLikeDislikeButtonViewModel object - } - - try { - return parseLikeCountFromLikeButtonRenderer(topLevelButtons); - } catch (final ParsingException e) { - throw new ParsingException("Could not get like count", e); - } + return parseLikeCountFromLikeButtonViewModel(topLevelButtons) + // A segmentedLikeDislikeButtonRenderer could be returned instead of a + // segmentedLikeDislikeButtonViewModel, so ignore extraction errors relative to + // segmentedLikeDislikeButtonViewModel object + .or(() -> parseLikeCountFromLikeButtonRenderer(topLevelButtons)) + .orElseThrow(() -> new ParsingException("Could not get like count")); } - private static long parseLikeCountFromLikeButtonRenderer( - @Nonnull final JsonArray topLevelButtons) throws ParsingException { - String likesString = null; - final JsonObject likeToggleButtonRenderer = topLevelButtons.stream() - .filter(JsonObject.class::isInstance) - .map(JsonObject.class::cast) + private static Optional parseLikeCountFromLikeButtonRenderer( + @Nonnull final JsonArray topLevelButtons) { + return topLevelButtons.streamAsJsonObjects() .map(button -> button.getObject("segmentedLikeDislikeButtonRenderer") .getObject("likeButton") - .getObject("toggleButtonRenderer")) - .filter(toggleButtonRenderer -> !isNullOrEmpty(toggleButtonRenderer)) + .getObject("toggleButtonRenderer", null)) + .filter(Objects::nonNull) .findFirst() - .orElse(null); - - if (likeToggleButtonRenderer != null) { - // Use one of the accessibility strings available (this one has the same path as the - // one used for comments' like count extraction) - likesString = likeToggleButtonRenderer.getObject("accessibilityData") - .getObject("accessibilityData") - .getString("label"); - - // Use the other accessibility string available which contains the exact like count - if (likesString == null) { - likesString = likeToggleButtonRenderer.getObject("accessibility") - .getString("label"); - } - - // Last method: use the defaultText's accessibility data, which contains the exact like - // count too, except when it is equal to 0, where a localized string is returned instead - if (likesString == null) { - likesString = likeToggleButtonRenderer.getObject("defaultText") - .getObject("accessibility") - .getObject("accessibilityData") - .getString("label"); - } - - // This check only works with English localizations! - if (likesString != null && likesString.toLowerCase().contains("no likes")) { - return 0; - } - } - - // If ratings are allowed and the likes string is null, it means that we couldn't extract - // the full like count from accessibility data - if (likesString == null) { - throw new ParsingException("Could not get like count from accessibility data"); - } - - try { - return Long.parseLong(Utils.removeNonDigitCharacters(likesString)); - } catch (final NumberFormatException e) { - throw new ParsingException("Could not parse \"" + likesString + "\" as a long", e); - } + .flatMap(toggleButtonRenderer -> { + // Use one of the accessibility strings available (this one has the same path + // as the one used for comments' like count extraction) + return Optional.ofNullable(toggleButtonRenderer.getObject("accessibilityData") + .getObject("accessibilityData") + .getString("label")) + + // Use the other accessibility string available which contains the exact + // like count + .or(() -> Optional.ofNullable(toggleButtonRenderer + .getObject("accessibility").getString("label"))) + + // Last method: use the defaultText's accessibility data, which contains + // the exact like count too, except when it is equal to 0, where a + // localized string is returned instead + .or(() -> Optional.ofNullable( + toggleButtonRenderer.getObject("defaultText") + .getObject("accessibility") + .getObject("accessibilityData") + .getString("label"))); + }) + .map(likesString -> { + if (likesString.toLowerCase().contains("no likes")) { + return 0L; + } else { + try { + return Long.parseLong(Utils.removeNonDigitCharacters(likesString)); + } catch (final NumberFormatException e) { + return null; + } + } + }); } - private static long parseLikeCountFromLikeButtonViewModel( - @Nonnull final JsonArray topLevelButtons) throws ParsingException { + private static Optional parseLikeCountFromLikeButtonViewModel( + @Nonnull final JsonArray topLevelButtons) { // Try first with the current video actions buttons data structure - final JsonObject likeToggleButtonViewModel = topLevelButtons.stream() - .filter(JsonObject.class::isInstance) - .map(JsonObject.class::cast) + return topLevelButtons.streamAsJsonObjects() .map(button -> button.getObject("segmentedLikeDislikeButtonViewModel") .getObject("likeButtonViewModel") .getObject("likeButtonViewModel") .getObject("toggleButtonViewModel") .getObject("toggleButtonViewModel") .getObject("defaultButtonViewModel") - .getObject("buttonViewModel")) - .filter(buttonViewModel -> !isNullOrEmpty(buttonViewModel)) + .getObject("buttonViewModel") + .getString("accessibilityText")) + .filter(Objects::nonNull) .findFirst() - .orElse(null); - - if (likeToggleButtonViewModel == null) { - throw new ParsingException("Could not find buttonViewModel object"); - } - - final String accessibilityText = likeToggleButtonViewModel.getString("accessibilityText"); - if (accessibilityText == null) { - throw new ParsingException("Could not find buttonViewModel's accessibilityText string"); - } - - // The like count is always returned as a number in this element, even for videos with no - // likes - try { - return Long.parseLong(Utils.removeNonDigitCharacters(accessibilityText)); - } catch (final NumberFormatException e) { - throw new ParsingException( - "Could not parse \"" + accessibilityText + "\" as a long", e); - } + .map(accessibilityText -> { + try { + // The like count is always returned as a number in this element, even for + // videos with no likes + return Long.parseLong(Utils.removeNonDigitCharacters(accessibilityText)); + } catch (final NumberFormatException e) { + return null; + } + }); } @Nonnull @@ -540,25 +489,20 @@ public String getUploaderName() throws ParsingException { } @Override - public boolean isUploaderVerified() throws ParsingException { - final JsonObject videoOwnerRenderer = getVideoSecondaryInfoRenderer() - .getObject("owner") - .getObject("videoOwnerRenderer"); - - if (videoOwnerRenderer.has("badges")) { - return YoutubeParsingHelper.isVerified(videoOwnerRenderer - .getArray("badges")); - } - - - final JsonObject channel = YoutubeParsingHelper.getFirstCollaborator( - videoOwnerRenderer.getObject("navigationEndpoint")); - if (channel == null) { - return false; - } - - return YoutubeParsingHelper.hasArtistOrVerifiedIconBadgeAttachment( - channel.getObject("title").getArray("attachmentRuns")); + public boolean isUploaderVerified() { + final var videoOwnerRenderer = getVideoSecondaryInfoRenderer().getObject("owner") + .getObject("videoOwnerRenderer"); + + return Optional.ofNullable(videoOwnerRenderer.getArray("badges", null)) + .map(YoutubeParsingHelper::isVerified) + .or(() -> YoutubeParsingHelper.getFirstCollaborator(videoOwnerRenderer) + .map(channel -> { + final var attachmentRuns = channel.getObject("title") + .getArray("attachmentRuns"); + return YoutubeParsingHelper + .hasArtistOrVerifiedIconBadgeAttachment(attachmentRuns); + })) + .orElse(false); } @Nonnull @@ -592,26 +536,20 @@ public List getUploaderAvatars() throws ParsingException { @Override public long getUploaderSubscriberCount() throws ParsingException { - final JsonObject videoOwnerRenderer = JsonUtils.getObject(videoSecondaryInfoRenderer, + final var videoOwnerRenderer = JsonUtils.getObject(videoSecondaryInfoRenderer, "owner.videoOwnerRenderer"); - - String subscriberCountText = null; - if (videoOwnerRenderer.has("subscriberCountText")) { - subscriberCountText = getTextFromObject(videoOwnerRenderer - .getObject("subscriberCountText")); - } else { - final String content = YoutubeParsingHelper.getFirstCollaborator( - videoOwnerRenderer.getObject("navigationEndpoint") - ).getObject("subtitle").getString("content"); - subscriberCountText = content.split("•")[1]; - } - - if (isNullOrEmpty(subscriberCountText)) { - return UNKNOWN_SUBSCRIBER_COUNT; - } + final String subscriberCountText = + getTextFromObject(videoOwnerRenderer.getObject("subscriberCountText")) + .or(() -> YoutubeParsingHelper.getFirstCollaborator(videoOwnerRenderer) + .map(collaborator -> collaborator.getObject("content") + .getString("content")) + .map(content -> content.split("•")[1])) + .filter(YoutubeParsingHelper.STRING_PREDICATE) + .orElse(null); try { - return Utils.mixedNumberWordToLong(subscriberCountText); + return subscriberCountText != null ? Utils.mixedNumberWordToLong(subscriberCountText) + : UNKNOWN_SUBSCRIBER_COUNT; } catch (final NumberFormatException e) { throw new ParsingException("Could not get uploader subscriber count", e); } @@ -820,13 +758,9 @@ public MultiInfoItemsCollector getRelatedItems() throws ExtractionException { */ @Override public String getErrorMessage() { - try { - return getTextFromObject(playerResponse.getObject(PLAYABILITY_STATUS) - .getObject("errorScreen").getObject("playerErrorMessageRenderer") - .getObject("reason")); - } catch (final NullPointerException e) { - return null; // No error message - } + return getTextFromObject(playerResponse.getObject(PLAYABILITY_STATUS) + .getObject("errorScreen").getObject("playerErrorMessageRenderer") + .getObject("reason")).orElse(null); } /*////////////////////////////////////////////////////////////////////////// @@ -917,17 +851,20 @@ private static void checkPlayabilityStatus(@Nonnull final JsonObject playability } if (reason.contains("unavailable")) { - final String detailedErrorMessage = getTextFromObject(playabilityStatus + final var subreason = playabilityStatus .getObject("errorScreen") .getObject("playerErrorMessageRenderer") - .getObject("subreason")); - if (detailedErrorMessage != null && detailedErrorMessage.contains("country")) { - throw new GeographicRestrictionException( - "This video is not available in client's country."); - } else { - throw new ContentNotAvailableException( - Objects.requireNonNullElse(detailedErrorMessage, reason)); - } + .getObject("subreason"); + throw getTextFromObject(subreason) + .map(message -> { + if (message.contains("country")) { + return new GeographicRestrictionException("This video is not " + + "available in client's country."); + } else { + return new ContentNotAvailableException(message); + } + }) + .orElse(new ContentNotAvailableException(reason)); } if (reason.contains("age-restricted")) { @@ -1572,19 +1509,17 @@ public String getCategory() { @Nonnull @Override public String getLicence() throws ParsingException { - final JsonObject metadataRowRenderer = getVideoSecondaryInfoRenderer() + final var metadataRowRenderer = getVideoSecondaryInfoRenderer() .getObject("metadataRowContainer") .getObject("metadataRowContainerRenderer") .getArray("rows") .getObject(0) .getObject("metadataRowRenderer"); - - final JsonArray contents = metadataRowRenderer.getArray("contents"); - final String license = getTextFromObject(contents.getObject(0)); - return license != null - && "Licence".equals(getTextFromObject(metadataRowRenderer.getObject("title"))) - ? license - : "YouTube licence"; + final var contents = metadataRowRenderer.getArray("contents"); + final var title = getTextFromObject(metadataRowRenderer.getObject("title")).orElse(null); + return getTextFromObject(contents.getObject(0)) + .filter(license -> "Licence".equals(title)) + .orElse("YouTube licence"); } @Override @@ -1649,10 +1584,8 @@ public List getStreamSegments() throws ParsingException { break; } - final String title = getTextFromObject(segmentJson.getObject("title")); - if (isNullOrEmpty(title)) { - throw new ParsingException("Could not get stream segment title."); - } + final String title = getTextFromObject(segmentJson.getObject("title")) + .orElseThrow(() -> new ParsingException("Could not get stream segment title.")); final StreamSegment segment = new StreamSegment(title, startTimeSeconds); segment.setUrl(getUrl() + "?t=" + startTimeSeconds); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 7b4deaaa58..9098200788 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -19,9 +19,9 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObjectOrThrow; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailsFromInfoItem; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; @@ -48,6 +48,7 @@ import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.List; +import java.util.Optional; import java.util.regex.Pattern; public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @@ -130,11 +131,7 @@ public String getUrl() throws ParsingException { @Override public String getName() throws ParsingException { - final String name = getTextFromObject(videoInfo.getObject("title")); - if (!isNullOrEmpty(name)) { - return name; - } - throw new ParsingException("Could not get name"); + return getTextFromObjectOrThrow(videoInfo.getObject("title"), "name"); } @Override @@ -143,38 +140,24 @@ public long getDuration() throws ParsingException { return -1; } - String duration = getTextFromObject(videoInfo.getObject("lengthText")); - - if (isNullOrEmpty(duration)) { - // Available in playlists for videos - duration = videoInfo.getString("lengthSeconds"); - - if (isNullOrEmpty(duration)) { - final JsonObject timeOverlay = videoInfo.getArray("thumbnailOverlays") - .stream() - .filter(JsonObject.class::isInstance) - .map(JsonObject.class::cast) - .filter(thumbnailOverlay -> - thumbnailOverlay.has("thumbnailOverlayTimeStatusRenderer")) + final String duration = getTextFromObject(videoInfo.getObject("lengthText")) + // Available in playlists for videos + .or(() -> Optional.ofNullable(videoInfo.getString("lengthSeconds"))) + .or(() -> videoInfo.getArray("thumbnailOverlays").streamAsJsonObjects() + .map(overlay -> overlay.getObject("thumbnailOverlayTimeStatusRenderer")) + .filter(renderer -> !renderer.isEmpty()) .findFirst() - .orElse(null); + .flatMap(overlay -> getTextFromObject(overlay.getObject("text")))) + .orElse(null); - if (timeOverlay != null) { - duration = getTextFromObject( - timeOverlay.getObject("thumbnailOverlayTimeStatusRenderer") - .getObject("text")); - } + if (isNullOrEmpty(duration)) { + if (isPremiere()) { + // Premieres can be livestreams, so the duration is not available in this + // case + return -1; } - if (isNullOrEmpty(duration)) { - if (isPremiere()) { - // Premieres can be livestreams, so the duration is not available in this - // case - return -1; - } - - throw new ParsingException("Could not get duration"); - } + throw new ParsingException("Could not get duration"); } return YoutubeParsingHelper.parseDurationString(duration); @@ -182,43 +165,25 @@ public long getDuration() throws ParsingException { @Override public String getUploaderName() throws ParsingException { - String name = getTextFromObject(videoInfo.getObject("longBylineText")); - - if (isNullOrEmpty(name)) { - name = getTextFromObject(videoInfo.getObject("ownerText")); - - if (isNullOrEmpty(name)) { - name = getTextFromObject(videoInfo.getObject("shortBylineText")); - - if (isNullOrEmpty(name)) { - throw new ParsingException("Could not get uploader name"); - } - } - } - - return name; + return getTextFromObject(videoInfo.getObject("longBylineText")) + .or(() -> getTextFromObject(videoInfo.getObject("ownerText"))) + .or(() -> getTextFromObject(videoInfo.getObject("shortBylineText"))) + .orElseThrow(() -> new ParsingException("Could not get uploader name")); } @Override public String getUploaderUrl() throws ParsingException { - String url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText") - .getArray("runs").getObject(0).getObject("navigationEndpoint")); - - if (isNullOrEmpty(url)) { - url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText") - .getArray("runs").getObject(0).getObject("navigationEndpoint")); - - if (isNullOrEmpty(url)) { - url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText") - .getArray("runs").getObject(0).getObject("navigationEndpoint")); - - if (isNullOrEmpty(url)) { - throw new ParsingException("Could not get uploader url"); - } - } - } + return getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText")) + .or(() -> getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText"))) + .or(() -> getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText"))) + .orElseThrow(() -> new ParsingException("Could not get uploader url")); + } - return url; + @Nonnull + private Optional getUrlFromNavigationEndpoint(@Nonnull final JsonObject jsonObject) { + final var endpoint = jsonObject.getArray("runs").getObject(0) + .getObject("navigationEndpoint"); + return YoutubeParsingHelper.getUrlFromNavigationEndpoint(endpoint); } @Nonnull @@ -257,19 +222,15 @@ public String getTextualUploadDate() throws ParsingException { return DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm").format(localDateTime); } - String publishedTimeText = getTextFromObject(videoInfo.getObject("publishedTimeText")); - - if (isNullOrEmpty(publishedTimeText) && videoInfo.has("videoInfo")) { - /* - Returned in playlists, in the form: view count separator upload date - */ - publishedTimeText = videoInfo.getObject("videoInfo") - .getArray("runs") - .getObject(2) - .getString("text"); - } - - return isNullOrEmpty(publishedTimeText) ? null : publishedTimeText; + return getTextFromObject(videoInfo.getObject("publishedTimeText")) + .or(() -> { + // Returned in playlists, in the form: view count separator upload date + return Optional.ofNullable(videoInfo.getObject("videoInfo") + .getArray("runs") + .getObject(2) + .getString("text")); + }) + .orElse(null); } @Nullable @@ -303,7 +264,8 @@ public long getViewCount() throws ParsingException { // Ignore all exceptions, as the view count can be hidden by creators, and so cannot be // found in this case - final String viewCountText = getTextFromObject(videoInfo.getObject("viewCountText")); + final String viewCountText = getTextFromObject(videoInfo.getObject("viewCountText")) + .orElse(null); if (!isNullOrEmpty(viewCountText)) { try { return getViewCountFromViewCountText(viewCountText, false); @@ -337,7 +299,8 @@ public long getViewCount() throws ParsingException { // Returned everywhere but in playlists, used by the website to show view counts try { final String shortViewCountText = - getTextFromObject(videoInfo.getObject("shortViewCountText")); + getTextFromObject(videoInfo.getObject("shortViewCountText")) + .orElse(null); if (!isNullOrEmpty(shortViewCountText)) { return getViewCountFromViewCountText(shortViewCountText, true); } @@ -418,18 +381,12 @@ private Instant getInstantFromPremiere() throws ParsingException { @Nullable @Override - public String getShortDescription() throws ParsingException { - if (videoInfo.has("detailedMetadataSnippets")) { - return getTextFromObject(videoInfo.getArray("detailedMetadataSnippets") - .getObject(0) - .getObject("snippetText")); - } - - if (videoInfo.has("descriptionSnippet")) { - return getTextFromObject(videoInfo.getObject("descriptionSnippet")); - } - - return null; + public String getShortDescription() { + return getTextFromObject(videoInfo.getArray("detailedMetadataSnippets") + .getObject(0) + .getObject("snippetText")) + .or(() -> getTextFromObject(videoInfo.getObject("descriptionSnippet"))) + .orElse(null); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeTrendingExtractor.java index 4dd3c04355..4945cc6ab8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeTrendingExtractor.java @@ -23,7 +23,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; @@ -84,19 +83,10 @@ public InfoItemsPage getPage(final Page page) { @Override public String getName() throws ParsingException { final JsonObject header = initialData.getObject("header"); - String name = null; - if (header.has("feedTabbedHeaderRenderer")) { - name = getTextAtKey(header.getObject("feedTabbedHeaderRenderer"), "title"); - } else if (header.has("c4TabbedHeaderRenderer")) { - name = getTextAtKey(header.getObject("c4TabbedHeaderRenderer"), "title"); - } else if (header.has("pageHeaderRenderer")) { - name = getTextAtKey(header.getObject("pageHeaderRenderer"), "pageTitle"); - } - - if (isNullOrEmpty(name)) { - throw new ParsingException("Could not get Trending name"); - } - return name; + return getTextAtKey(header.getObject("feedTabbedHeaderRenderer"), "title") + .or(() -> getTextAtKey(header.getObject("c4TabbedHeaderRenderer"), "title")) + .or(() -> getTextAtKey(header.getObject("pageHeaderRenderer"), "pageTitle")) + .orElseThrow(() -> new ParsingException("Could not get Trending name")); } @Nonnull