diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Info.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Info.java index 78a15553b1..83a5707243 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Info.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Info.java @@ -42,11 +42,11 @@ public void addAllErrors(final Collection throwables) { this.errors.addAll(throwables); } - public Info(final int serviceId, - final String id, - final String url, - final String originalUrl, - final String name) { + protected Info(final int serviceId, + final String id, + final String url, + final String originalUrl, + final String name) { this.serviceId = serviceId; this.id = id; this.url = url; @@ -54,7 +54,7 @@ public Info(final int serviceId, this.name = name; } - public Info(final int serviceId, final LinkHandler linkHandler, final String name) { + protected Info(final int serviceId, final LinkHandler linkHandler, final String name) { this(serviceId, linkHandler.getId(), linkHandler.getUrl(), diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java deleted file mode 100644 index 4423068488..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java +++ /dev/null @@ -1,168 +0,0 @@ -package org.schabi.newpipe.extractor; - -/* - * Created by Adam Howard on 08/11/15. - * - * Copyright (c) Christian Schabesberger - * and Adam Howard 2015 - * - * MediaFormat.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -import java.util.Arrays; -import java.util.function.Function; - -/** - * Static data about various media formats support by NewPipe, eg mime type, extension - */ - -@SuppressWarnings("MethodParamPad") // we want the media format table below to be aligned -public enum MediaFormat { - // @formatter:off - //video and audio combined formats - // id name suffix mimeType - MPEG_4 (0x0, "MPEG-4", "mp4", "video/mp4"), - v3GPP (0x10, "3GPP", "3gp", "video/3gpp"), - WEBM (0x20, "WebM", "webm", "video/webm"), - // audio formats - M4A (0x100, "m4a", "m4a", "audio/mp4"), - WEBMA (0x200, "WebM", "webm", "audio/webm"), - MP3 (0x300, "MP3", "mp3", "audio/mpeg"), - OPUS (0x400, "opus", "opus", "audio/opus"), - OGG (0x500, "ogg", "ogg", "audio/ogg"), - WEBMA_OPUS(0x200, "WebM Opus", "webm", "audio/webm"), - // subtitles formats - VTT (0x1000, "WebVTT", "vtt", "text/vtt"), - TTML (0x2000, "Timed Text Markup Language", "ttml", "application/ttml+xml"), - TRANSCRIPT1(0x3000, "TranScript v1", "srv1", "text/xml"), - TRANSCRIPT2(0x4000, "TranScript v2", "srv2", "text/xml"), - TRANSCRIPT3(0x5000, "TranScript v3", "srv3", "text/xml"), - SRT (0x6000, "SubRip file format", "srt", "text/srt"); - // @formatter:on - - public final int id; - public final String name; - public final String suffix; - public final String mimeType; - - MediaFormat(final int id, final String name, final String suffix, final String mimeType) { - this.id = id; - this.name = name; - this.suffix = suffix; - this.mimeType = mimeType; - } - - private static T getById(final int id, - final Function field, - final T orElse) { - return Arrays.stream(MediaFormat.values()) - .filter(mediaFormat -> mediaFormat.id == id) - .map(field) - .findFirst() - .orElse(orElse); - } - - /** - * Return the friendly name of the media format with the supplied id - * - * @param id the id of the media format. Currently an arbitrary, NewPipe-specific number. - * @return the friendly name of the MediaFormat associated with this ids, - * or an empty String if none match it. - */ - public static String getNameById(final int id) { - return getById(id, MediaFormat::getName, ""); - } - - /** - * Return the file extension of the media format with the supplied id - * - * @param id the id of the media format. Currently an arbitrary, NewPipe-specific number. - * @return the file extension of the MediaFormat associated with this ids, - * or an empty String if none match it. - */ - public static String getSuffixById(final int id) { - return getById(id, MediaFormat::getSuffix, ""); - } - - /** - * Return the MIME type of the media format with the supplied id - * - * @param id the id of the media format. Currently an arbitrary, NewPipe-specific number. - * @return the MIME type of the MediaFormat associated with this ids, - * or an empty String if none match it. - */ - public static String getMimeById(final int id) { - return getById(id, MediaFormat::getMimeType, null); - } - - /** - * Return the MediaFormat with the supplied mime type - * - * @return MediaFormat associated with this mime type, - * or null if none match it. - */ - public static MediaFormat getFromMimeType(final String mimeType) { - return Arrays.stream(MediaFormat.values()) - .filter(mediaFormat -> mediaFormat.mimeType.equals(mimeType)) - .findFirst() - .orElse(null); - } - - /** - * Get the media format by its id. - * - * @param id the id - * @return the id of the media format or null. - */ - public static MediaFormat getFormatById(final int id) { - return getById(id, mediaFormat -> mediaFormat, null); - } - - public static MediaFormat getFromSuffix(final String suffix) { - return Arrays.stream(MediaFormat.values()) - .filter(mediaFormat -> mediaFormat.suffix.equals(suffix)) - .findFirst() - .orElse(null); - } - - /** - * Get the name of the format - * - * @return the name of the format - */ - public String getName() { - return name; - } - - /** - * Get the filename extension - * - * @return the filename extension - */ - public String getSuffix() { - return suffix; - } - - /** - * Get the mime type - * - * @return the mime type - */ - public String getMimeType() { - return mimeType; - } - -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java index 75d0bbf805..18017ded2a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java @@ -4,12 +4,13 @@ import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.localization.Localization; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.io.IOException; import java.util.List; import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + /** * A base for downloader implementations that NewPipe will use * to download needed resources during extraction. @@ -148,8 +149,27 @@ public Response post(final String url, /** * Do a request using the specified {@link Request} object. * + * @param request The request to process * @return the result of the request */ public abstract Response execute(@Nonnull Request request) throws IOException, ReCaptchaException; + + /** + * Get the size of the content that the url is pointing by firing a HEAD request. + * + * @param url an url pointing to the content + * @return the size of the content, in bytes or -1 if unknown + */ + public long getContentLength(final String url) { + try { + final String contentLengthHeader = head(url).getHeader("Content-Length"); + if (contentLengthHeader == null) { + return -1; + } + return Long.parseLong(contentLengthHeader); + } catch (final Exception e) { + return -1; + } + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java index be6ba0b7ad..9992d985c0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java @@ -2,17 +2,17 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; + import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; import javax.annotation.Nullable; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL; -import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; - public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { private final JsonObject show; @@ -58,8 +58,8 @@ public String getThumbnailUrl() { } @Override - public StreamType getStreamType() { - return StreamType.AUDIO_STREAM; + public boolean isAudioOnly() { + return true; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java index 03a9c2222b..dcc2815274 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java @@ -11,7 +11,6 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Element; -import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -21,9 +20,12 @@ import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector; -import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamSegment; +import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleProgressiveHTTPDeliveryDataImpl; +import org.schabi.newpipe.extractor.streamdata.format.registry.AudioFormatRegistry; +import org.schabi.newpipe.extractor.streamdata.stream.AudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.simpleimpl.SimpleAudioStreamImpl; import java.io.IOException; import java.util.ArrayList; @@ -120,24 +122,23 @@ public long getLength() { @Override public List getAudioStreams() { - final List audioStreams = new ArrayList<>(); final JsonObject streams = showInfo.getObject("audio_stream"); + final List audioStreams = new ArrayList<>(); if (streams.has(MP3_128)) { - audioStreams.add(new AudioStream.Builder() - .setId(MP3_128) - .setContent(streams.getString(MP3_128), true) - .setMediaFormat(MediaFormat.MP3) - .setAverageBitrate(128) - .build()); + audioStreams.add(new SimpleAudioStreamImpl( + AudioFormatRegistry.MP3, + new SimpleProgressiveHTTPDeliveryDataImpl(streams.getString(MP3_128)), + 128 + )); } if (streams.has(OPUS_LO)) { - audioStreams.add(new AudioStream.Builder() - .setId(OPUS_LO) - .setContent(streams.getString(OPUS_LO), true) - .setMediaFormat(MediaFormat.OPUS) - .setAverageBitrate(100).build()); + audioStreams.add(new SimpleAudioStreamImpl( + AudioFormatRegistry.OPUS, + new SimpleProgressiveHTTPDeliveryDataImpl(streams.getString(OPUS_LO)), + 100 + )); } return audioStreams; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index e4120bed89..714dca714b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -11,7 +11,6 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -19,11 +18,12 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector; -import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleProgressiveHTTPDeliveryDataImpl; +import org.schabi.newpipe.extractor.streamdata.format.registry.AudioFormatRegistry; +import org.schabi.newpipe.extractor.streamdata.stream.AudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.simpleimpl.SimpleAudioStreamImpl; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; @@ -143,36 +143,28 @@ public Description getDescription() { @Override public List getAudioStreams() { - return Collections.singletonList(new AudioStream.Builder() - .setId("mp3-128") - .setContent(albumJson.getArray("trackinfo") - .getObject(0) - .getObject("file") - .getString("mp3-128"), true) - .setMediaFormat(MediaFormat.MP3) - .setAverageBitrate(128) - .build()); + return Collections.singletonList( + new SimpleAudioStreamImpl( + AudioFormatRegistry.MP3, + new SimpleProgressiveHTTPDeliveryDataImpl(albumJson + .getArray("trackinfo") + .getObject(0) + .getObject("file") + .getString("mp3-128")), + 128 + ) + ); } @Override - public long getLength() throws ParsingException { - return (long) albumJson.getArray("trackinfo").getObject(0) - .getDouble("duration"); - } - - @Override - public List getVideoStreams() { - return Collections.emptyList(); + public boolean isAudioOnly() { + return true; } @Override - public List getVideoOnlyStreams() { - return Collections.emptyList(); - } - - @Override - public StreamType getStreamType() { - return StreamType.AUDIO_STREAM; + public long getLength() throws ParsingException { + return (long) albumJson.getArray("trackinfo").getObject(0) + .getDouble("duration"); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java index 5e585e7e00..d1a55b33a1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java @@ -22,11 +22,7 @@ public BandcampSearchStreamInfoItemExtractor(final Element searchResult, public String getUploaderName() { final String subhead = resultInfo.getElementsByClass("subhead").text(); final String[] splitBy = subhead.split("by "); - if (splitBy.length > 1) { - return splitBy[1]; - } else { - return splitBy[0]; - } + return splitBy[splitBy.length > 1 ? 1 : 0]; } @Nullable diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java index fee96fcd97..07ca6d7ed9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java @@ -3,7 +3,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; import javax.annotation.Nullable; @@ -13,13 +12,13 @@ public abstract class BandcampStreamInfoItemExtractor implements StreamInfoItemExtractor { private final String uploaderUrl; - public BandcampStreamInfoItemExtractor(final String uploaderUrl) { + protected BandcampStreamInfoItemExtractor(final String uploaderUrl) { this.uploaderUrl = uploaderUrl; } @Override - public StreamType getStreamType() { - return StreamType.AUDIO_STREAM; + public boolean isAudioOnly() { + return true; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java index 9271bbc322..7322f75405 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java @@ -1,30 +1,32 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors; -import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; -import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN; - import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; -import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.Description; -import org.schabi.newpipe.extractor.stream.Stream; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.streamdata.delivery.DeliveryData; +import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleDASHUrlDeliveryDataImpl; +import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleHLSDeliveryDataImpl; +import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleProgressiveHTTPDeliveryDataImpl; +import org.schabi.newpipe.extractor.streamdata.format.registry.AudioFormatRegistry; +import org.schabi.newpipe.extractor.streamdata.format.registry.VideoAudioFormatRegistry; +import org.schabi.newpipe.extractor.streamdata.stream.AudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.VideoAudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.quality.VideoQualityData; +import org.schabi.newpipe.extractor.streamdata.stream.simpleimpl.SimpleAudioStreamImpl; +import org.schabi.newpipe.extractor.streamdata.stream.simpleimpl.SimpleVideoAudioStreamImpl; import java.io.IOException; -import java.util.Collections; import java.util.List; -import java.util.function.Function; +import java.util.Objects; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -45,7 +47,8 @@ public MediaCCCLiveStreamExtractor(final StreamingService service, @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { - final JsonArray doc = MediaCCCParsingHelper.getLiveStreams(downloader, + final JsonArray doc = MediaCCCParsingHelper.getLiveStreams( + downloader, getExtractorLocalization()); // Find the correct room for (int c = 0; c < doc.size(); c++) { @@ -137,7 +140,7 @@ public String getDashMpdUrl() throws ParsingException { */ @Nonnull @Override - public String getHlsUrl() { + public String getHlsMasterPlaylistUrl() { return getManifestOfDeliveryMethodWanted("hls"); } @@ -155,77 +158,56 @@ private String getManifestOfDeliveryMethodWanted(@Nonnull final String deliveryM @Override public List getAudioStreams() throws IOException, ExtractionException { - return getStreams("audio", - dto -> { - final AudioStream.Builder builder = new AudioStream.Builder() - .setId(dto.urlValue.getString("tech", ID_UNKNOWN)) - .setContent(dto.urlValue.getString(URL), true) - .setAverageBitrate(UNKNOWN_BITRATE); - - if ("hls".equals(dto.urlKey)) { - // We don't know with the type string what media format will - // have HLS streams. - // However, the tech string may contain some information - // about the media format used. - return builder.setDeliveryMethod(DeliveryMethod.HLS) - .build(); + return getStreamDTOs("audio") + .map(dto -> { + try { + return new SimpleAudioStreamImpl( + new AudioFormatRegistry().getFromSuffixOrThrow(dto.getUrlKey()), + buildDeliveryData(dto) + ); + } catch (final Exception ignored) { + return null; } - - return builder.setMediaFormat(MediaFormat.getFromSuffix(dto.urlKey)) - .build(); - }); + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); } @Override - public List getVideoStreams() throws IOException, ExtractionException { - return getStreams("video", - dto -> { - final JsonArray videoSize = dto.streamJsonObj.getArray("videoSize"); - - final VideoStream.Builder builder = new VideoStream.Builder() - .setId(dto.urlValue.getString("tech", ID_UNKNOWN)) - .setContent(dto.urlValue.getString(URL), true) - .setIsVideoOnly(false) - .setResolution(videoSize.getInt(0) + "x" + videoSize.getInt(1)); - - if ("hls".equals(dto.urlKey)) { - // We don't know with the type string what media format will - // have HLS streams. - // However, the tech string may contain some information - // about the media format used. - return builder.setDeliveryMethod(DeliveryMethod.HLS) - .build(); + public List getVideoStreams() throws IOException, ExtractionException { + return getStreamDTOs("video") + .map(dto -> { + try { + final JsonArray videoSize = + dto.getStreamJsonObj().getArray("videoSize"); + + return new SimpleVideoAudioStreamImpl( + new VideoAudioFormatRegistry() + .getFromSuffixOrThrow(dto.getUrlKey()), + buildDeliveryData(dto), + VideoQualityData.fromHeightWidth( + videoSize.getInt(1, VideoQualityData.UNKNOWN), + videoSize.getInt(0, VideoQualityData.UNKNOWN)) + ); + } catch (final Exception ignored) { + return null; } - - return builder.setMediaFormat(MediaFormat.getFromSuffix(dto.urlKey)) - .build(); - }); + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); } - - /** - * This is just an internal class used in {@link #getStreams(String, Function)} to tie together - * the stream json object, its URL key and its URL value. An object of this class would be - * temporary and the three values it holds would be converted to a proper {@link Stream} - * object based on the wanted stream type. - */ - private static final class MediaCCCLiveStreamMapperDTO { - final JsonObject streamJsonObj; - final String urlKey; - final JsonObject urlValue; - - MediaCCCLiveStreamMapperDTO(final JsonObject streamJsonObj, - final String urlKey, - final JsonObject urlValue) { - this.streamJsonObj = streamJsonObj; - this.urlKey = urlKey; - this.urlValue = urlValue; + private DeliveryData buildDeliveryData(final MediaCCCLiveStreamMapperDTO dto) { + final String url = dto.getUrlValue().getString(URL); + if ("hls".equals(dto.getUrlKey())) { + return new SimpleHLSDeliveryDataImpl(url); + } else if ("dash".equals(dto.getUrlKey())) { + return new SimpleDASHUrlDeliveryDataImpl(url); } + return new SimpleProgressiveHTTPDeliveryDataImpl(url); } - private List getStreams( - @Nonnull final String streamType, - @Nonnull final Function converter) { + private Stream getStreamDTOs(@Nonnull final String streamType) { return room.getArray(STREAMS).stream() // Ensure that we use only process JsonObjects .filter(JsonObject.class::isInstance) @@ -238,22 +220,12 @@ private List getStreams( .map(e -> new MediaCCCLiveStreamMapperDTO( streamJsonObj, e.getKey(), - (JsonObject) e.getValue()))) - // The DASH manifest will be extracted with getDashMpdUrl - .filter(dto -> !"dash".equals(dto.urlKey)) - // Convert - .map(converter) - .collect(Collectors.toList()); + (JsonObject) e.getValue()))); } @Override - public List getVideoOnlyStreams() { - return Collections.emptyList(); - } - - @Override - public StreamType getStreamType() throws ParsingException { - return StreamType.LIVE_STREAM; + public boolean isLive() { + return true; } @Nonnull @@ -261,4 +233,31 @@ public StreamType getStreamType() throws ParsingException { public String getCategory() { return group; } + + static final class MediaCCCLiveStreamMapperDTO { + private final JsonObject streamJsonObj; + private final String urlKey; + private final JsonObject urlValue; + + MediaCCCLiveStreamMapperDTO(final JsonObject streamJsonObj, + final String urlKey, + final JsonObject urlValue) { + this.streamJsonObj = streamJsonObj; + this.urlKey = urlKey; + this.urlValue = urlValue; + } + + JsonObject getStreamJsonObj() { + return streamJsonObj; + } + + String getUrlKey() { + return urlKey; + } + + JsonObject getUrlValue() { + return urlValue; + } + } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java index 2b311fb352..69b78dc791 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java @@ -1,10 +1,10 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors; import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; import javax.annotation.Nullable; @@ -38,15 +38,16 @@ public String getThumbnailUrl() throws ParsingException { } @Override - public StreamType getStreamType() throws ParsingException { - boolean isVideo = false; - for (final Object stream : roomInfo.getArray("streams")) { - if ("video".equals(((JsonObject) stream).getString("type"))) { - isVideo = true; - break; - } - } - return isVideo ? StreamType.LIVE_STREAM : StreamType.AUDIO_LIVE_STREAM; + public boolean isLive() { + return true; + } + + @Override + public boolean isAudioOnly() { + return roomInfo.getArray("streams").stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .noneMatch(s -> "video".equalsIgnoreCase(s.getString("type"))); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java index ad07c4720d..84b62e225e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java @@ -3,6 +3,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -43,13 +44,21 @@ public static boolean isLiveStreamId(final String id) { /** * Get currently available live streams from * - * https://streaming.media.ccc.de/streams/v2.json. + * https://streaming.media.ccc.de/streams/v2.json. + *

* Use this method to cache requests, because they can get quite big. + *

+ * + *

+ * For more information see also: + * https://github.com/voc/streaming-website#json-api. + *

* TODO: implement better caching policy (max-age: 3 min) - * @param downloader The downloader to use for making the request + * + * @param downloader The downloader to use for making the request * @param localization The localization to be used. Will most likely be ignored. * @return {@link JsonArray} containing current conferences and info about their rooms and - * streams. + * streams. * @throws ExtractionException if the data could not be fetched or the retrieved data could not * be parsed to a {@link JsonArray} */ @@ -58,13 +67,14 @@ public static JsonArray getLiveStreams(final Downloader downloader, throws ExtractionException { if (liveStreams == null) { try { - final String site = downloader.get("https://streaming.media.ccc.de/streams/v2.json", - localization).responseBody(); + final String site = downloader + .get("https://streaming.media.ccc.de/streams/v2.json", localization) + .responseBody(); liveStreams = JsonParser.array().from(site); } catch (final IOException | ReCaptchaException e) { - throw new ExtractionException("Could not get live stream JSON.", e); + throw new ExtractionException("Could not get live stream JSON", e); } catch (final JsonParserException e) { - throw new ExtractionException("Could not parse JSON.", e); + throw new ExtractionException("Could not parse JSON", e); } } return liveStreams; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java index 1e18f8da13..386d6c2ce2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java @@ -6,7 +6,6 @@ import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; @@ -36,11 +35,6 @@ public String getThumbnailUrl() throws ParsingException { return event.getString("thumb_url"); } - @Override - public StreamType getStreamType() throws ParsingException { - return StreamType.VIDEO_STREAM; - } - @Override public boolean isAd() { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 0a086fcc6b..9c99ac1191 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -1,14 +1,9 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors; -import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; -import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN; - -import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -18,18 +13,23 @@ import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory; -import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleProgressiveHTTPDeliveryDataImpl; +import org.schabi.newpipe.extractor.streamdata.format.registry.AudioFormatRegistry; +import org.schabi.newpipe.extractor.streamdata.format.registry.VideoAudioFormatRegistry; +import org.schabi.newpipe.extractor.streamdata.stream.AudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.VideoAudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.quality.VideoQualityData; +import org.schabi.newpipe.extractor.streamdata.stream.simpleimpl.SimpleAudioStreamImpl; +import org.schabi.newpipe.extractor.streamdata.stream.simpleimpl.SimpleVideoAudioStreamImpl; import org.schabi.newpipe.extractor.utils.JsonUtils; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -96,80 +96,34 @@ public String getUploaderAvatarUrl() { @Override public List getAudioStreams() throws ExtractionException { - final JsonArray recordings = data.getArray("recordings"); - final List audioStreams = new ArrayList<>(); - for (int i = 0; i < recordings.size(); i++) { - final JsonObject recording = recordings.getObject(i); - final String mimeType = recording.getString("mime_type"); - if (mimeType.startsWith("audio")) { - // First we need to resolve the actual video data from the CDN - final MediaFormat mediaFormat; - if (mimeType.endsWith("opus")) { - mediaFormat = MediaFormat.OPUS; - } else if (mimeType.endsWith("mpeg")) { - mediaFormat = MediaFormat.MP3; - } else if (mimeType.endsWith("ogg")) { - mediaFormat = MediaFormat.OGG; - } else { - mediaFormat = null; - } - - // Not checking containsSimilarStream here, since MediaCCC does not provide enough - // information to decide whether two streams are similar. Hence that method would - // always return false, e.g. even for different language variations. - audioStreams.add(new AudioStream.Builder() - .setId(recording.getString("filename", ID_UNKNOWN)) - .setContent(recording.getString("recording_url"), true) - .setMediaFormat(mediaFormat) - .setAverageBitrate(UNKNOWN_BITRATE) - .build()); - } - } - return audioStreams; - } - - @Override - public List getVideoStreams() throws ExtractionException { - final JsonArray recordings = data.getArray("recordings"); - final List videoStreams = new ArrayList<>(); - for (int i = 0; i < recordings.size(); i++) { - final JsonObject recording = recordings.getObject(i); - final String mimeType = recording.getString("mime_type"); - if (mimeType.startsWith("video")) { - // First we need to resolve the actual video data from the CDN - final MediaFormat mediaFormat; - if (mimeType.endsWith("webm")) { - mediaFormat = MediaFormat.WEBM; - } else if (mimeType.endsWith("mp4")) { - mediaFormat = MediaFormat.MPEG_4; - } else { - mediaFormat = null; - } - - // Not checking containsSimilarStream here, since MediaCCC does not provide enough - // information to decide whether two streams are similar. Hence that method would - // always return false, e.g. even for different language variations. - videoStreams.add(new VideoStream.Builder() - .setId(recording.getString("filename", ID_UNKNOWN)) - .setContent(recording.getString("recording_url"), true) - .setIsVideoOnly(false) - .setMediaFormat(mediaFormat) - .setResolution(recording.getInt("height") + "p") - .build()); - } - } - - return videoStreams; + return getRecordingsByMimeType("audio") + .map(o -> new SimpleAudioStreamImpl( + new AudioFormatRegistry().getFromMimeTypeOrThrow(o.getString("mime_type")), + new SimpleProgressiveHTTPDeliveryDataImpl(o.getString("recording_url")) + )) + .collect(Collectors.toList()); } @Override - public List getVideoOnlyStreams() { - return Collections.emptyList(); + public List getVideoStreams() throws ExtractionException { + return getRecordingsByMimeType("video") + .map(o -> new SimpleVideoAudioStreamImpl( + new VideoAudioFormatRegistry() + .getFromMimeTypeOrThrow(o.getString("mime_type")), + new SimpleProgressiveHTTPDeliveryDataImpl(o.getString("recording_url")), + VideoQualityData.fromHeightWidth( + o.getInt("height", VideoQualityData.UNKNOWN), + o.getInt("width", VideoQualityData.UNKNOWN)) + )) + .collect(Collectors.toList()); } - @Override - public StreamType getStreamType() { - return StreamType.VIDEO_STREAM; + private Stream getRecordingsByMimeType(final String startsWithMimeType) { + return data.getArray("recordings").stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(rec -> rec.getString("mime_type", "") + .startsWith(startsWithMimeType)); } @Override @@ -181,8 +135,7 @@ public void onFetchPage(@Nonnull final Downloader downloader) conferenceData = JsonParser.object() .from(downloader.get(data.getString("conference_url")).responseBody()); } catch (final JsonParserException jpe) { - throw new ExtractionException("Could not parse json returned by URL: " + videoUrl, - jpe); + throw new ExtractionException("Could not parse json returned by URL: " + videoUrl, jpe); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java index 92f0894b9f..293d825f33 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java @@ -1,11 +1,11 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems; import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; import javax.annotation.Nullable; @@ -16,11 +16,6 @@ public MediaCCCStreamInfoItemExtractor(final JsonObject event) { this.event = event; } - @Override - public StreamType getStreamType() { - return StreamType.VIDEO_STREAM; - } - @Override public boolean isAd() { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index d70abf11d1..991d30d53f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -1,6 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -9,7 +8,6 @@ import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -22,15 +20,25 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; -import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.Description; -import org.schabi.newpipe.extractor.stream.Stream; +import org.schabi.newpipe.extractor.stream.Privacy; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.stream.SubtitlesStream; -import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.streamdata.delivery.DeliveryData; +import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleHLSDeliveryDataImpl; +import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleProgressiveHTTPDeliveryDataImpl; +import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleTorrentDeliveryDataImpl; +import org.schabi.newpipe.extractor.streamdata.format.registry.AudioFormatRegistry; +import org.schabi.newpipe.extractor.streamdata.format.registry.SubtitleFormatRegistry; +import org.schabi.newpipe.extractor.streamdata.format.registry.VideoAudioFormatRegistry; +import org.schabi.newpipe.extractor.streamdata.stream.AudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.Stream; +import org.schabi.newpipe.extractor.streamdata.stream.SubtitleStream; +import org.schabi.newpipe.extractor.streamdata.stream.VideoAudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.quality.VideoQualityData; +import org.schabi.newpipe.extractor.streamdata.stream.simpleimpl.SimpleAudioStreamImpl; +import org.schabi.newpipe.extractor.streamdata.stream.simpleimpl.SimpleSubtitleStreamImpl; +import org.schabi.newpipe.extractor.streamdata.stream.simpleimpl.SimpleVideoAudioStreamImpl; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; @@ -41,6 +49,9 @@ import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.Objects; +import java.util.function.BiFunction; +import java.util.function.Consumer; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -53,15 +64,14 @@ public class PeertubeStreamExtractor extends StreamExtractor { private static final String FILE_DOWNLOAD_URL = "fileDownloadUrl"; private static final String FILE_URL = "fileUrl"; private static final String PLAYLIST_URL = "playlistUrl"; - private static final String RESOLUTION_ID = "resolution.id"; private static final String STREAMING_PLAYLISTS = "streamingPlaylists"; private final String baseUrl; private JsonObject json; - private final List subtitles = new ArrayList<>(); - private final List audioStreams = new ArrayList<>(); - private final List videoStreams = new ArrayList<>(); + private List subtitles = null; + private List audioStreams = null; + private List videoStreams = null; public PeertubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler) throws ParsingException { @@ -118,12 +128,7 @@ public Description getDescription() throws ParsingException { @Override public int getAgeLimit() throws ParsingException { - final boolean isNSFW = JsonUtils.getBoolean(json, "nsfw"); - if (isNSFW) { - return 18; - } else { - return NO_AGE_LIMIT; - } + return JsonUtils.getBoolean(json, "nsfw") ? 18 : NO_AGE_LIMIT; } @Override @@ -136,12 +141,9 @@ public long getTimeStamp() throws ParsingException { final long timestamp = getTimestampSeconds( "((#|&|\\?)start=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)"); - if (timestamp == -2) { - // regex for timestamp was not found - return 0; - } else { - return timestamp; - } + return (timestamp == -2) + ? 0 // regex for timestamp was not found + : timestamp; } @Override @@ -212,15 +214,16 @@ public String getSubChannelAvatarUrl() { @Nonnull @Override - public String getHlsUrl() { + public String getHlsMasterPlaylistUrl() throws ParsingException { assertPageFetched(); - if (getStreamType() == StreamType.VIDEO_STREAM - && !isNullOrEmpty(json.getObject(FILES))) { + if (!isLive() && !isNullOrEmpty(json.getObject(FILES))) { return json.getObject(FILES).getString(PLAYLIST_URL, ""); } - return json.getArray(STREAMING_PLAYLISTS).getObject(0).getString(PLAYLIST_URL, ""); + return json.getArray(STREAMING_PLAYLISTS) + .getObject(0) + .getString(PLAYLIST_URL, ""); } @Override @@ -234,51 +237,30 @@ public List getAudioStreams() throws ParsingException { That's why the extraction of audio streams is only run when there are video streams extracted and when the content is not a livestream. */ - if (audioStreams.isEmpty() && videoStreams.isEmpty() - && getStreamType() == StreamType.VIDEO_STREAM) { - getStreams(); - } + tryExtractStreams(); return audioStreams; } @Override - public List getVideoStreams() throws ExtractionException { + public List getVideoStreams() throws ExtractionException { assertPageFetched(); - if (videoStreams.isEmpty()) { - if (getStreamType() == StreamType.VIDEO_STREAM) { - getStreams(); - } else { - extractLiveVideoStreams(); - } - } + tryExtractStreams(); return videoStreams; } - @Override - public List getVideoOnlyStreams() { - return Collections.emptyList(); - } - @Nonnull @Override - public List getSubtitlesDefault() { + public List getSubtitles() { + assertPageFetched(); return subtitles; } - @Nonnull @Override - public List getSubtitles(final MediaFormat format) { - return subtitles.stream() - .filter(sub -> sub.getFormat() == format) - .collect(Collectors.toList()); - } - - @Override - public StreamType getStreamType() { - return json.getBoolean("isLive") ? StreamType.LIVE_STREAM : StreamType.VIDEO_STREAM; + public boolean isLive() { + return json.getBoolean("isLive"); } @Nullable @@ -383,7 +365,7 @@ public void onFetchPage(@Nonnull final Downloader downloader) throw new ExtractionException("Could not extract PeerTube channel data"); } - loadSubtitles(); + tryExtractSubtitles(); } private void setInitialData(final String responseBody) throws ExtractionException { @@ -398,246 +380,166 @@ private void setInitialData(final String responseBody) throws ExtractionExceptio PeertubeParsingHelper.validate(json); } - private void loadSubtitles() { - if (subtitles.isEmpty()) { - try { - final Response response = getDownloader().get(baseUrl - + PeertubeStreamLinkHandlerFactory.VIDEO_API_ENDPOINT - + getId() + "/captions"); - final JsonObject captionsJson = JsonParser.object().from(response.responseBody()); - final JsonArray captions = JsonUtils.getArray(captionsJson, "data"); - for (final Object c : captions) { - if (c instanceof JsonObject) { - final JsonObject caption = (JsonObject) c; - final String url = baseUrl + JsonUtils.getString(caption, "captionPath"); - final String languageCode = JsonUtils.getString(caption, "language.id"); - final String ext = url.substring(url.lastIndexOf(".") + 1); - final MediaFormat fmt = MediaFormat.getFromSuffix(ext); - if (fmt != null && !isNullOrEmpty(languageCode)) { - subtitles.add(new SubtitlesStream.Builder() - .setContent(url, true) - .setMediaFormat(fmt) - .setLanguageCode(languageCode) - .setAutoGenerated(false) - .build()); - } - } - } - } catch (final Exception ignored) { - // Ignore all exceptions - } + private void tryExtractSubtitles() { + if (subtitles != null) { + return; } - } - - private void extractLiveVideoStreams() throws ParsingException { try { - final JsonArray streamingPlaylists = json.getArray(STREAMING_PLAYLISTS); - streamingPlaylists.stream() + final Response response = getDownloader().get(baseUrl + + PeertubeStreamLinkHandlerFactory.VIDEO_API_ENDPOINT + + getId() + "/captions"); + final JsonObject captionsJson = JsonParser.object().from(response.responseBody()); + final JsonArray captions = JsonUtils.getArray(captionsJson, "data"); + + subtitles = captions.stream() .filter(JsonObject.class::isInstance) .map(JsonObject.class::cast) - .map(stream -> new VideoStream.Builder() - .setId(String.valueOf(stream.getInt("id", -1))) - .setContent(stream.getString(PLAYLIST_URL, ""), true) - .setIsVideoOnly(false) - .setResolution("") - .setMediaFormat(MediaFormat.MPEG_4) - .setDeliveryMethod(DeliveryMethod.HLS) - .build()) - // Don't use the containsSimilarStream method because it will always return - // false so if there are multiples HLS URLs returned, only the first will be - // extracted in this case. - .forEachOrdered(videoStreams::add); - } catch (final Exception e) { - throw new ParsingException("Could not get video streams", e); + .map(caption -> { + try { + final String url = + baseUrl + JsonUtils.getString(caption, "captionPath"); + + return new SimpleSubtitleStreamImpl( + new SubtitleFormatRegistry() + .getFromSuffixOrThrow( + url.substring(url.lastIndexOf(".") + 1)), + new SimpleProgressiveHTTPDeliveryDataImpl(url), + false, + JsonUtils.getString(caption, "language.id") + ); + } catch (final Exception ignored) { + return null; + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } catch (final Exception ignored) { + subtitles = Collections.emptyList(); } } - private void getStreams() throws ParsingException { - // Progressive streams - getStreamsFromArray(json.getArray(FILES), ""); + private void tryExtractStreams() throws ParsingException { + if (audioStreams != null && videoStreams != null) { + return; + } - // HLS streams + // Initialize + audioStreams = new ArrayList<>(); + videoStreams = new ArrayList<>(); + + // Progressive streams try { - for (final JsonObject playlist : json.getArray(STREAMING_PLAYLISTS).stream() - .filter(JsonObject.class::isInstance) - .map(JsonObject.class::cast) - .collect(Collectors.toList())) { - getStreamsFromArray(playlist.getArray(FILES), playlist.getString(PLAYLIST_URL)); - } + addStreamsFromArray( + json.getArray(FILES), + null); } catch (final Exception e) { - throw new ParsingException("Could not get streams", e); + throw new ParsingException("Could not add HLS streams", e); } - } - private void getStreamsFromArray(@Nonnull final JsonArray streams, - final String playlistUrl) throws ParsingException { + // HLS streams try { - /* - Starting with version 3.4.0 of PeerTube, the HLS playlist of stream resolutions - contains the UUID of the streams, so we can't use the same method to get the URL of - the HLS playlist without fetching the master playlist. - These UUIDs are the same as the ones returned into the fileUrl and fileDownloadUrl - strings. - */ - final boolean isInstanceUsingRandomUuidsForHlsStreams = !isNullOrEmpty(playlistUrl) - && playlistUrl.endsWith("-master.m3u8"); - - for (final JsonObject stream : streams.stream() + json.getArray(STREAMING_PLAYLISTS).stream() .filter(JsonObject.class::isInstance) .map(JsonObject.class::cast) - .collect(Collectors.toList())) { - - // Extract stream version of streams first - final String url = JsonUtils.getString(stream, - stream.has(FILE_URL) ? FILE_URL : FILE_DOWNLOAD_URL); - if (isNullOrEmpty(url)) { - // Not a valid stream URL - return; - } - - final String resolution = JsonUtils.getString(stream, "resolution.label"); - final String idSuffix = stream.has(FILE_URL) ? FILE_URL : FILE_DOWNLOAD_URL; - - if (resolution.toLowerCase().contains("audio")) { - // An audio stream - addNewAudioStream(stream, isInstanceUsingRandomUuidsForHlsStreams, resolution, - idSuffix, url, playlistUrl); - } else { - // A video stream - addNewVideoStream(stream, isInstanceUsingRandomUuidsForHlsStreams, resolution, - idSuffix, url, playlistUrl); - } - } + .forEach(playlist -> addStreamsFromArray( + playlist.getArray(FILES), + playlist.getString(PLAYLIST_URL))); } catch (final Exception e) { - throw new ParsingException("Could not get streams from array", e); + throw new ParsingException("Could not add HLS streams", e); } } - @Nonnull - private String getHlsPlaylistUrlFromFragmentedFileUrl( - @Nonnull final JsonObject streamJsonObject, - @Nonnull final String idSuffix, - @Nonnull final String format, - @Nonnull final String url) throws ParsingException { - final String streamUrl = FILE_DOWNLOAD_URL.equals(idSuffix) - ? JsonUtils.getString(streamJsonObject, FILE_URL) - : url; - return streamUrl.replace("-fragmented." + format, ".m3u8"); + private void addStreamsFromArray( + @Nonnull final JsonArray streams, + final String playlistUrl + ) { + streams.stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(stream -> !isNullOrEmpty(getUrlFromStream(stream))) + .forEach(stream -> { + final JsonObject resJson = stream.getObject("resolution"); + if (resJson.getString("label", "") + .toLowerCase() + .contains("audio")) { + // An audio stream + addNewStreams( + this.audioStreams, + stream, + playlistUrl, + (s, dd) -> new SimpleAudioStreamImpl( + new AudioFormatRegistry() + .getFromSuffixOrThrow(getExtensionFromStream(s)), + dd + ) + ); + + } else { + // A video stream + addNewStreams( + this.videoStreams, + stream, + playlistUrl, + (s, dd) -> new SimpleVideoAudioStreamImpl( + new VideoAudioFormatRegistry() + .getFromSuffixOrThrow(getExtensionFromStream(s)), + dd, + VideoQualityData.fromHeightFps( + resJson.getInt("id", VideoQualityData.UNKNOWN), + stream.getInt("fps", VideoQualityData.UNKNOWN)) + ) + ); + } + }); } - @Nonnull - private String getHlsPlaylistUrlFromMasterPlaylist(@Nonnull final JsonObject streamJsonObject, - @Nonnull final String playlistUrl) - throws ParsingException { - return playlistUrl.replace("master", JsonUtils.getNumber(streamJsonObject, - RESOLUTION_ID).toString()); - } - - private void addNewAudioStream(@Nonnull final JsonObject streamJsonObject, - final boolean isInstanceUsingRandomUuidsForHlsStreams, - @Nonnull final String resolution, - @Nonnull final String idSuffix, - @Nonnull final String url, - @Nullable final String playlistUrl) throws ParsingException { - final String extension = url.substring(url.lastIndexOf(".") + 1); - final MediaFormat format = MediaFormat.getFromSuffix(extension); - final String id = resolution + "-" + extension; - - // Add progressive HTTP streams first - audioStreams.add(new AudioStream.Builder() - .setId(id + "-" + idSuffix + "-" + DeliveryMethod.PROGRESSIVE_HTTP) - .setContent(url, true) - .setMediaFormat(format) - .setAverageBitrate(UNKNOWN_BITRATE) - .build()); - - // Then add HLS streams - if (!isNullOrEmpty(playlistUrl)) { - final String hlsStreamUrl; - if (isInstanceUsingRandomUuidsForHlsStreams) { - hlsStreamUrl = getHlsPlaylistUrlFromFragmentedFileUrl(streamJsonObject, idSuffix, - extension, url); - - } else { - hlsStreamUrl = getHlsPlaylistUrlFromMasterPlaylist(streamJsonObject, playlistUrl); - } - final AudioStream audioStream = new AudioStream.Builder() - .setId(id + "-" + DeliveryMethod.HLS) - .setContent(hlsStreamUrl, true) - .setDeliveryMethod(DeliveryMethod.HLS) - .setMediaFormat(format) - .setAverageBitrate(UNKNOWN_BITRATE) - .setManifestUrl(playlistUrl) - .build(); - if (!Stream.containSimilarStream(audioStream, audioStreams)) { - audioStreams.add(audioStream); - } - } - // Finally, add torrent URLs - final String torrentUrl = JsonUtils.getString(streamJsonObject, "torrentUrl"); - if (!isNullOrEmpty(torrentUrl)) { - audioStreams.add(new AudioStream.Builder() - .setId(id + "-" + idSuffix + "-" + DeliveryMethod.TORRENT) - .setContent(torrentUrl, true) - .setDeliveryMethod(DeliveryMethod.TORRENT) - .setMediaFormat(format) - .setAverageBitrate(UNKNOWN_BITRATE) - .build()); - } + private static String getStreamUrlKeyFromStream(@Nonnull final JsonObject stream) { + return stream.has(FILE_URL) ? FILE_URL : FILE_DOWNLOAD_URL; } - private void addNewVideoStream(@Nonnull final JsonObject streamJsonObject, - final boolean isInstanceUsingRandomUuidsForHlsStreams, - @Nonnull final String resolution, - @Nonnull final String idSuffix, - @Nonnull final String url, - @Nullable final String playlistUrl) throws ParsingException { - final String extension = url.substring(url.lastIndexOf(".") + 1); - final MediaFormat format = MediaFormat.getFromSuffix(extension); - final String id = resolution + "-" + extension; - - // Add progressive HTTP streams first - videoStreams.add(new VideoStream.Builder() - .setId(id + "-" + idSuffix + "-" + DeliveryMethod.PROGRESSIVE_HTTP) - .setContent(url, true) - .setIsVideoOnly(false) - .setResolution(resolution) - .setMediaFormat(format) - .build()); - - // Then add HLS streams - if (!isNullOrEmpty(playlistUrl)) { - final String hlsStreamUrl = isInstanceUsingRandomUuidsForHlsStreams - ? getHlsPlaylistUrlFromFragmentedFileUrl(streamJsonObject, idSuffix, extension, - url) - : getHlsPlaylistUrlFromMasterPlaylist(streamJsonObject, playlistUrl); - - final VideoStream videoStream = new VideoStream.Builder() - .setId(id + "-" + DeliveryMethod.HLS) - .setContent(hlsStreamUrl, true) - .setIsVideoOnly(false) - .setDeliveryMethod(DeliveryMethod.HLS) - .setResolution(resolution) - .setMediaFormat(format) - .setManifestUrl(playlistUrl) - .build(); - if (!Stream.containSimilarStream(videoStream, videoStreams)) { - videoStreams.add(videoStream); - } + private static String getUrlFromStream(@Nonnull final JsonObject stream) { + return stream.getString(getStreamUrlKeyFromStream(stream)); + } + + private static String getExtensionFromStream(@Nonnull final JsonObject stream) { + final String url = stream.getString(getStreamUrlKeyFromStream(stream)); + return url.substring(url.lastIndexOf(".") + 1); + } + + private void addNewStreams( + final List streams, + @Nonnull final JsonObject stream, + final String playlistUrl, + @Nonnull final BiFunction buildStream + ) { + final Consumer addDeliveryDataToStream = + dd -> { + try { + streams.add(buildStream.apply(stream, dd)); + } catch (final Exception ignored) { + // Ignore exception when a single stream couldn't be added + } + }; + + // Add Progressive HTTP (this is also done for HLS streams because the source file can + // also be streamed over progressive HTTP) + addDeliveryDataToStream.accept( + new SimpleProgressiveHTTPDeliveryDataImpl(getUrlFromStream(stream))); + + // Add HLS (only for PeerTube 3.4+) + if (!isNullOrEmpty(playlistUrl) + && playlistUrl.endsWith("-master.m3u8") + && !isNullOrEmpty(stream.getString(FILE_URL)) + ) { + addDeliveryDataToStream.accept(new SimpleHLSDeliveryDataImpl(stream.getString(FILE_URL) + .replace("-fragmented." + getExtensionFromStream(stream), ".m3u8"))); } - // Add finally torrent URLs - final String torrentUrl = JsonUtils.getString(streamJsonObject, "torrentUrl"); - if (!isNullOrEmpty(torrentUrl)) { - videoStreams.add(new VideoStream.Builder() - .setId(id + "-" + idSuffix + "-" + DeliveryMethod.TORRENT) - .setContent(torrentUrl, true) - .setIsVideoOnly(false) - .setDeliveryMethod(DeliveryMethod.TORRENT) - .setResolution(resolution) - .setMediaFormat(format) - .build()); + // Add torrent + if (!isNullOrEmpty(stream.getString("torrentUrl"))) { + addDeliveryDataToStream.accept( + new SimpleTorrentDeliveryDataImpl(stream.getString("torrentUrl"))); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java index b9c7f4b13e..9cc6afeb80 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -1,12 +1,12 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.JsonUtils; import javax.annotation.Nullable; @@ -93,8 +93,8 @@ public DateWrapper getUploadDate() throws ParsingException { } @Override - public StreamType getStreamType() { - return item.getBoolean("isLive") ? StreamType.LIVE_STREAM : StreamType.VIDEO_STREAM; + public boolean isLive() { + return item.getBoolean("isLive"); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 0d28ed9544..049552b314 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -2,8 +2,6 @@ import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.clientId; -import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; -import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN; import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -12,26 +10,29 @@ import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.SoundCloudGoPlusContentException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; -import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.Description; -import org.schabi.newpipe.extractor.stream.Stream; +import org.schabi.newpipe.extractor.stream.Privacy; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleHLSDeliveryDataImpl; +import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleProgressiveHTTPDeliveryDataImpl; +import org.schabi.newpipe.extractor.streamdata.format.AudioMediaFormat; +import org.schabi.newpipe.extractor.streamdata.format.registry.AudioFormatRegistry; +import org.schabi.newpipe.extractor.streamdata.stream.AudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.simpleimpl.SimpleAudioStreamImpl; import java.io.IOException; import java.io.UnsupportedEncodingException; @@ -39,6 +40,9 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -162,38 +166,22 @@ public String getUploaderAvatarUrl() { @Override public List getAudioStreams() throws ExtractionException { - final List audioStreams = new ArrayList<>(); // Streams can be streamable and downloadable - or explicitly not. // For playing the track, it is only necessary to have a streamable track. // If this is not the case, this track might not be published yet. + // If audio streams were calculated, return the calculated result if (!track.getBoolean("streamable") || !isAvailable) { - return audioStreams; + return Collections.emptyList(); } try { - final JsonArray transcodings = track.getObject("media").getArray("transcodings"); - if (!isNullOrEmpty(transcodings)) { - // Get information about what stream formats are available - extractAudioStreams(transcodings, checkMp3ProgressivePresence(transcodings), - audioStreams); - } - - extractDownloadableFileIfAvailable(audioStreams); + final List audioStreams = new ArrayList<>(extractAudioStreams()); + extractDownloadableFileIfAvailable().ifPresent(audioStreams::add); + return audioStreams; } catch (final NullPointerException e) { - throw new ExtractionException("Could not get audio streams", e); + throw new ExtractionException("Could not get SoundCloud's tracks audio URL", e); } - - return audioStreams; - } - - private static boolean checkMp3ProgressivePresence(@Nonnull final JsonArray transcodings) { - return transcodings.stream() - .filter(JsonObject.class::isInstance) - .map(JsonObject.class::cast) - .anyMatch(transcodingJsonObject -> transcodingJsonObject.getString("preset") - .contains("mp3") && transcodingJsonObject.getObject("format") - .getString("protocol").equals("progressive")); } @Nonnull @@ -230,59 +218,54 @@ private String getDownloadUrl(@Nonnull final String trackId) return null; } - private void extractAudioStreams(@Nonnull final JsonArray transcodings, - final boolean mp3ProgressiveInStreams, - final List audioStreams) { - transcodings.stream() + private List extractAudioStreams() { + final JsonArray transcodings = track.getObject("media").getArray("transcodings"); + if (isNullOrEmpty(transcodings)) { + return Collections.emptyList(); + } + + return transcodings.stream() .filter(JsonObject.class::isInstance) .map(JsonObject.class::cast) - .forEachOrdered(transcoding -> { - final String url = transcoding.getString("url"); - if (isNullOrEmpty(url)) { - return; + .filter(transcoding -> !isNullOrEmpty(transcoding.getString("url"))) + .map(transcoding -> { + final String protocol = transcoding + .getObject("format") + .getString("protocol"); + final String mediaUrl; + try { + mediaUrl = getTranscodingUrl(transcoding.getString("url")); + } catch (final Exception e) { + return null; // Abort if something went wrong + } + if (isNullOrEmpty(mediaUrl)) { + return null; // Ignore invalid urls } - try { - final String preset = transcoding.getString("preset", ID_UNKNOWN); - final String protocol = transcoding.getObject("format") - .getString("protocol"); - final AudioStream.Builder builder = new AudioStream.Builder() - .setId(preset); - - final boolean isHls = protocol.equals("hls"); - if (isHls) { - builder.setDeliveryMethod(DeliveryMethod.HLS); - } - - builder.setContent(getTranscodingUrl(url), true); - - if (preset.contains("mp3")) { - // Don't add the MP3 HLS stream if there is a progressive stream - // present because both have the same bitrate - if (mp3ProgressiveInStreams && isHls) { - return; - } - - builder.setMediaFormat(MediaFormat.MP3); - builder.setAverageBitrate(128); - } else if (preset.contains("opus")) { - builder.setMediaFormat(MediaFormat.OPUS); - builder.setAverageBitrate(64); - builder.setDeliveryMethod(DeliveryMethod.HLS); - } else { - // Unknown format, skip to the next audio stream - return; - } - - final AudioStream audioStream = builder.build(); - if (!Stream.containSimilarStream(audioStream, audioStreams)) { - audioStreams.add(audioStream); - } - } catch (final ExtractionException | IOException ignored) { - // Something went wrong when trying to get and add this audio stream, - // skip to the next one + final String preset = transcoding.getString("preset", ""); + + final AudioMediaFormat mediaFormat; + final int averageBitrate; + if (preset.contains("mp3")) { + mediaFormat = AudioFormatRegistry.MP3; + averageBitrate = 128; + } else if (preset.contains("opus")) { + mediaFormat = AudioFormatRegistry.OPUS; + averageBitrate = 64; + } else { + return null; } - }); + + return new SimpleAudioStreamImpl( + mediaFormat, + "hls".equals(protocol) + ? new SimpleHLSDeliveryDataImpl(mediaUrl) + : new SimpleProgressiveHTTPDeliveryDataImpl(mediaUrl), + averageBitrate + ); + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); } /** @@ -298,47 +281,73 @@ private void extractAudioStreams(@Nonnull final JsonArray transcodings, * downloaded, and not otherwise. *

* - * @param audioStreams the audio streams to which the downloadable file is added + * @return An {@link Optional} that may contain the extracted audio stream */ - public void extractDownloadableFileIfAvailable(final List audioStreams) { - if (track.getBoolean("downloadable") && track.getBoolean("has_downloads_left")) { - try { - final String downloadUrl = getDownloadUrl(getId()); - if (!isNullOrEmpty(downloadUrl)) { - audioStreams.add(new AudioStream.Builder() - .setId("original-format") - .setContent(downloadUrl, true) - .setAverageBitrate(UNKNOWN_BITRATE) - .build()); - } - } catch (final Exception ignored) { - // If something went wrong when trying to get the download URL, ignore the - // exception throw because this "stream" is not necessary to play the track - } + @Nonnull + private Optional extractDownloadableFileIfAvailable() { + if (!track.getBoolean("downloadable") || !track.getBoolean("has_downloads_left")) { + return Optional.empty(); } - } - private static String urlEncode(final String value) { try { - return URLEncoder.encode(value, UTF_8); - } catch (final UnsupportedEncodingException e) { - throw new IllegalStateException(e); + final String downloadUrl = getDownloadUrl(getId()); + if (isNullOrEmpty(downloadUrl)) { + return Optional.empty(); + } + + // Find out what type of file is served + final String fileType = determineFileTypeFromDownloadUrl(downloadUrl); + + // No fileType found -> ignore it + if (isNullOrEmpty(fileType)) { + return Optional.empty(); + } + + return Optional.of(new SimpleAudioStreamImpl( + new AudioFormatRegistry().getFromSuffixOrThrow(fileType), + new SimpleProgressiveHTTPDeliveryDataImpl(downloadUrl) + )); + } catch (final Exception ignored) { + // If something went wrong when trying to get the download URL, ignore the + // exception throw because this "stream" is not necessary to play the track + return Optional.empty(); } } - @Override - public List getVideoStreams() { - return Collections.emptyList(); - } + /** + * Determines the file type/extension of the download url. + *

+ * Note: Uses HTTP FETCH for inspection. + *

+ */ + @Nullable + private String determineFileTypeFromDownloadUrl(final String downloadUrl) + throws IOException, ReCaptchaException { - @Override - public List getVideoOnlyStreams() { - return Collections.emptyList(); + final Response response = NewPipe.getDownloader().head(downloadUrl); + + // As of 2022-06 Soundcloud uses AWS S3 + // Use the AWS header to identify the filetype first because it's simpler + final String amzMetaFileType = response.getHeader("x-amz-meta-file-type"); + if (!isNullOrEmpty(amzMetaFileType)) { + return amzMetaFileType; + } + + // If the AWS header was not present try extract the filetype + // by inspecting the download file name + // Example-Value: + // attachment;filename="SoundCloud%20Download"; filename*=utf-8''song.mp3 + final String contentDisp = response.getHeader("Content-Disposition"); + if (!isNullOrEmpty(contentDisp) && contentDisp.contains(".")) { + return contentDisp.substring(contentDisp.lastIndexOf(".") + 1); + } + + return null; } @Override - public StreamType getStreamType() { - return StreamType.AUDIO_STREAM; + public boolean isAudioOnly() { + return true; } @Nullable @@ -396,4 +405,13 @@ public List getTags() { } return tags; } + + + private static String urlEncode(final String value) { + try { + return URLEncoder.encode(value, UTF_8); + } catch (final UnsupportedEncodingException e) { + throw new IllegalStateException(e); + } + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java index 3d265e4e41..8e9e5878bb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java @@ -8,7 +8,6 @@ import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; import javax.annotation.Nullable; @@ -20,6 +19,11 @@ public SoundcloudStreamInfoItemExtractor(final JsonObject itemObject) { this.itemObject = itemObject; } + @Override + public boolean isAudioOnly() { + return true; + } + @Override public String getUrl() { return replaceHttpWithHttps(itemObject.getString("permalink_url")); @@ -80,11 +84,6 @@ public String getThumbnailUrl() { return artworkUrl.replace("large.jpg", "crop.jpg"); } - @Override - public StreamType getStreamType() { - return StreamType.AUDIO_STREAM; - } - @Override public boolean isAd() { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/DeliveryType.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/DeliveryType.java deleted file mode 100644 index 17833dc5fc..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/DeliveryType.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube; - -/** - * Streaming format types used by YouTube in their streams. - * - *

- * It is different from {@link org.schabi.newpipe.extractor.stream.DeliveryMethod delivery methods}! - *

- */ -public enum DeliveryType { - - /** - * YouTube's progressive delivery method, which works with HTTP range headers. - * (Note that official clients use the corresponding parameter instead.) - * - *

- * Initialization and index ranges are available to get metadata (the corresponding values - * are returned in the player response). - *

- */ - PROGRESSIVE, - - /** - * YouTube's OTF delivery method which uses a sequence parameter to get segments of - * streams. - * - *

- * The first sequence (which can be fetched with the {@code &sq=0} parameter) contains all the - * metadata needed to build the stream source (sidx boxes, segment length, segment count, - * duration, ...). - *

- * - *

- * Only used for videos; mostly those with a small amount of views, or ended livestreams - * which have just been re-encoded as normal videos. - *

- */ - OTF, - - /** - * YouTube's delivery method for livestreams which uses a sequence parameter to get - * segments of streams. - * - *

- * Each sequence (which can be fetched with the {@code &sq=0} parameter) contains its own - * metadata (sidx boxes, segment length, ...), which make no need of an initialization - * segment. - *

- * - *

- * Only used for livestreams (ended or running). - *

- */ - LIVE -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java deleted file mode 100644 index e0ff09a6f7..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java +++ /dev/null @@ -1,542 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube; - -import static org.schabi.newpipe.extractor.MediaFormat.M4A; -import static org.schabi.newpipe.extractor.MediaFormat.MPEG_4; -import static org.schabi.newpipe.extractor.MediaFormat.WEBM; -import static org.schabi.newpipe.extractor.MediaFormat.WEBMA; -import static org.schabi.newpipe.extractor.MediaFormat.WEBMA_OPUS; -import static org.schabi.newpipe.extractor.MediaFormat.v3GPP; -import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.AUDIO; -import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.VIDEO; -import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.VIDEO_ONLY; - -import org.schabi.newpipe.extractor.MediaFormat; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -import java.io.Serializable; - -public class ItagItem implements Serializable { - - /** - * List can be found here: - * https://github.com/ytdl-org/youtube-dl/blob/e988fa4/youtube_dl/extractor/youtube.py#L1195 - */ - private static final ItagItem[] ITAG_LIST = { - ///////////////////////////////////////////////////// - // VIDEO ID Type Format Resolution FPS //// - ///////////////////////////////////////////////////// - new ItagItem(17, VIDEO, v3GPP, "144p"), - new ItagItem(36, VIDEO, v3GPP, "240p"), - - new ItagItem(18, VIDEO, MPEG_4, "360p"), - new ItagItem(34, VIDEO, MPEG_4, "360p"), - new ItagItem(35, VIDEO, MPEG_4, "480p"), - new ItagItem(59, VIDEO, MPEG_4, "480p"), - new ItagItem(78, VIDEO, MPEG_4, "480p"), - new ItagItem(22, VIDEO, MPEG_4, "720p"), - new ItagItem(37, VIDEO, MPEG_4, "1080p"), - new ItagItem(38, VIDEO, MPEG_4, "1080p"), - - new ItagItem(43, VIDEO, WEBM, "360p"), - new ItagItem(44, VIDEO, WEBM, "480p"), - new ItagItem(45, VIDEO, WEBM, "720p"), - new ItagItem(46, VIDEO, WEBM, "1080p"), - - ////////////////////////////////////////////////////////////////// - // AUDIO ID ItagType Format Bitrate // - ////////////////////////////////////////////////////////////////// - new ItagItem(171, AUDIO, WEBMA, 128), - new ItagItem(172, AUDIO, WEBMA, 256), - new ItagItem(139, AUDIO, M4A, 48), - new ItagItem(140, AUDIO, M4A, 128), - new ItagItem(141, AUDIO, M4A, 256), - new ItagItem(249, AUDIO, WEBMA_OPUS, 50), - new ItagItem(250, AUDIO, WEBMA_OPUS, 70), - new ItagItem(251, AUDIO, WEBMA_OPUS, 160), - - /// VIDEO ONLY //////////////////////////////////////////// - // ID Type Format Resolution FPS //// - /////////////////////////////////////////////////////////// - new ItagItem(160, VIDEO_ONLY, MPEG_4, "144p"), - new ItagItem(133, VIDEO_ONLY, MPEG_4, "240p"), - new ItagItem(134, VIDEO_ONLY, MPEG_4, "360p"), - new ItagItem(135, VIDEO_ONLY, MPEG_4, "480p"), - new ItagItem(212, VIDEO_ONLY, MPEG_4, "480p"), - new ItagItem(136, VIDEO_ONLY, MPEG_4, "720p"), - new ItagItem(298, VIDEO_ONLY, MPEG_4, "720p60", 60), - new ItagItem(137, VIDEO_ONLY, MPEG_4, "1080p"), - new ItagItem(299, VIDEO_ONLY, MPEG_4, "1080p60", 60), - new ItagItem(266, VIDEO_ONLY, MPEG_4, "2160p"), - - new ItagItem(278, VIDEO_ONLY, WEBM, "144p"), - new ItagItem(242, VIDEO_ONLY, WEBM, "240p"), - new ItagItem(243, VIDEO_ONLY, WEBM, "360p"), - new ItagItem(244, VIDEO_ONLY, WEBM, "480p"), - new ItagItem(245, VIDEO_ONLY, WEBM, "480p"), - new ItagItem(246, VIDEO_ONLY, WEBM, "480p"), - new ItagItem(247, VIDEO_ONLY, WEBM, "720p"), - new ItagItem(248, VIDEO_ONLY, WEBM, "1080p"), - new ItagItem(271, VIDEO_ONLY, WEBM, "1440p"), - // #272 is either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug) - new ItagItem(272, VIDEO_ONLY, WEBM, "2160p"), - new ItagItem(302, VIDEO_ONLY, WEBM, "720p60", 60), - new ItagItem(303, VIDEO_ONLY, WEBM, "1080p60", 60), - new ItagItem(308, VIDEO_ONLY, WEBM, "1440p60", 60), - new ItagItem(313, VIDEO_ONLY, WEBM, "2160p"), - new ItagItem(315, VIDEO_ONLY, WEBM, "2160p60", 60) - }; - - /*////////////////////////////////////////////////////////////////////////// - // Utils - //////////////////////////////////////////////////////////////////////////*/ - - public static boolean isSupported(final int itag) { - for (final ItagItem item : ITAG_LIST) { - if (itag == item.id) { - return true; - } - } - return false; - } - - @Nonnull - public static ItagItem getItag(final int itagId) throws ParsingException { - for (final ItagItem item : ITAG_LIST) { - if (itagId == item.id) { - return new ItagItem(item); - } - } - throw new ParsingException("itag " + itagId + " is not supported"); - } - - /*////////////////////////////////////////////////////////////////////////// - // Static constants - //////////////////////////////////////////////////////////////////////////*/ - - public static final int AVERAGE_BITRATE_UNKNOWN = -1; - public static final int SAMPLE_RATE_UNKNOWN = -1; - public static final int FPS_NOT_APPLICABLE_OR_UNKNOWN = -1; - public static final int TARGET_DURATION_SEC_UNKNOWN = -1; - public static final int AUDIO_CHANNELS_NOT_APPLICABLE_OR_UNKNOWN = -1; - public static final long CONTENT_LENGTH_UNKNOWN = -1; - public static final long APPROX_DURATION_MS_UNKNOWN = -1; - - /*////////////////////////////////////////////////////////////////////////// - // Constructors and misc - //////////////////////////////////////////////////////////////////////////*/ - - public enum ItagType { - AUDIO, - VIDEO, - VIDEO_ONLY - } - - /** - * Call {@link #ItagItem(int, ItagType, MediaFormat, String, int)} with the fps set to 30. - */ - public ItagItem(final int id, - final ItagType type, - final MediaFormat format, - final String resolution) { - this.id = id; - this.itagType = type; - this.mediaFormat = format; - this.resolutionString = resolution; - this.fps = 30; - } - - /** - * Constructor for videos. - */ - public ItagItem(final int id, - final ItagType type, - final MediaFormat format, - final String resolution, - final int fps) { - this.id = id; - this.itagType = type; - this.mediaFormat = format; - this.resolutionString = resolution; - this.fps = fps; - } - - public ItagItem(final int id, - final ItagType type, - final MediaFormat format, - final int avgBitrate) { - this.id = id; - this.itagType = type; - this.mediaFormat = format; - this.avgBitrate = avgBitrate; - } - - /** - * Copy constructor of the {@link ItagItem} class. - * - * @param itagItem the {@link ItagItem} to copy its properties into a new {@link ItagItem} - */ - public ItagItem(@Nonnull final ItagItem itagItem) { - this.mediaFormat = itagItem.mediaFormat; - this.id = itagItem.id; - this.itagType = itagItem.itagType; - this.avgBitrate = itagItem.avgBitrate; - this.sampleRate = itagItem.sampleRate; - this.audioChannels = itagItem.audioChannels; - this.resolutionString = itagItem.resolutionString; - this.fps = itagItem.fps; - this.bitrate = itagItem.bitrate; - this.width = itagItem.width; - this.height = itagItem.height; - this.initStart = itagItem.initStart; - this.initEnd = itagItem.initEnd; - this.indexStart = itagItem.indexStart; - this.indexEnd = itagItem.indexEnd; - this.quality = itagItem.quality; - this.codec = itagItem.codec; - this.targetDurationSec = itagItem.targetDurationSec; - this.approxDurationMs = itagItem.approxDurationMs; - this.contentLength = itagItem.contentLength; - } - - public MediaFormat getMediaFormat() { - return mediaFormat; - } - - private final MediaFormat mediaFormat; - - public final int id; - public final ItagType itagType; - - // Audio fields - /** @deprecated Use {@link #getAverageBitrate()} instead. */ - @Deprecated - public int avgBitrate = AVERAGE_BITRATE_UNKNOWN; - private int sampleRate = SAMPLE_RATE_UNKNOWN; - private int audioChannels = AUDIO_CHANNELS_NOT_APPLICABLE_OR_UNKNOWN; - - // Video fields - /** @deprecated Use {@link #getResolutionString()} instead. */ - @Deprecated - public String resolutionString; - - /** @deprecated Use {@link #getFps()} and {@link #setFps(int)} instead. */ - @Deprecated - public int fps = FPS_NOT_APPLICABLE_OR_UNKNOWN; - - // Fields for Dash - private int bitrate; - private int width; - private int height; - private int initStart; - private int initEnd; - private int indexStart; - private int indexEnd; - private String quality; - private String codec; - private int targetDurationSec = TARGET_DURATION_SEC_UNKNOWN; - private long approxDurationMs = APPROX_DURATION_MS_UNKNOWN; - private long contentLength = CONTENT_LENGTH_UNKNOWN; - - public int getBitrate() { - return bitrate; - } - - public void setBitrate(final int bitrate) { - this.bitrate = bitrate; - } - - public int getWidth() { - return width; - } - - public void setWidth(final int width) { - this.width = width; - } - - public int getHeight() { - return height; - } - - public void setHeight(final int height) { - this.height = height; - } - - /** - * Get the frame rate. - * - *

- * It is set to the {@code fps} value returned in the corresponding itag in the YouTube player - * response. - *

- * - *

- * It defaults to the standard value associated with this itag. - *

- * - *

- * Note that this value is only known for video itags, so {@link - * #FPS_NOT_APPLICABLE_OR_UNKNOWN} is returned for non video itags. - *

- * - * @return the frame rate or {@link #FPS_NOT_APPLICABLE_OR_UNKNOWN} - */ - public int getFps() { - return fps; - } - - /** - * Set the frame rate. - * - *

- * It is only known for video itags, so {@link #FPS_NOT_APPLICABLE_OR_UNKNOWN} is set/used for - * non video itags or if the sample rate value is less than or equal to 0. - *

- * - * @param fps the frame rate - */ - public void setFps(final int fps) { - this.fps = fps > 0 ? fps : FPS_NOT_APPLICABLE_OR_UNKNOWN; - } - - public int getInitStart() { - return initStart; - } - - public void setInitStart(final int initStart) { - this.initStart = initStart; - } - - public int getInitEnd() { - return initEnd; - } - - public void setInitEnd(final int initEnd) { - this.initEnd = initEnd; - } - - public int getIndexStart() { - return indexStart; - } - - public void setIndexStart(final int indexStart) { - this.indexStart = indexStart; - } - - public int getIndexEnd() { - return indexEnd; - } - - public void setIndexEnd(final int indexEnd) { - this.indexEnd = indexEnd; - } - - public String getQuality() { - return quality; - } - - public void setQuality(final String quality) { - this.quality = quality; - } - - /** - * Get the resolution string associated with this {@code ItagItem}. - * - *

- * It is only known for video itags. - *

- * - * @return the resolution string associated with this {@code ItagItem} or - * {@code null}. - */ - @Nullable - public String getResolutionString() { - return resolutionString; - } - - public String getCodec() { - return codec; - } - - public void setCodec(final String codec) { - this.codec = codec; - } - - /** - * Get the average bitrate. - * - *

- * It is only known for audio itags, so {@link #AVERAGE_BITRATE_UNKNOWN} is always returned for - * other itag types. - *

- * - *

- * Bitrate of video itags and precise bitrate of audio itags can be known using - * {@link #getBitrate()}. - *

- * - * @return the average bitrate or {@link #AVERAGE_BITRATE_UNKNOWN} - * @see #getBitrate() - */ - public int getAverageBitrate() { - return avgBitrate; - } - - /** - * Get the sample rate. - * - *

- * It is only known for audio itags, so {@link #SAMPLE_RATE_UNKNOWN} is returned for non audio - * itags, or if the sample rate is unknown. - *

- * - * @return the sample rate or {@link #SAMPLE_RATE_UNKNOWN} - */ - public int getSampleRate() { - return sampleRate; - } - - /** - * Set the sample rate. - * - *

- * It is only known for audio itags, so {@link #SAMPLE_RATE_UNKNOWN} is set/used for non audio - * itags, or if the sample rate value is less than or equal to 0. - *

- * - * @param sampleRate the sample rate of an audio itag - */ - public void setSampleRate(final int sampleRate) { - this.sampleRate = sampleRate > 0 ? sampleRate : SAMPLE_RATE_UNKNOWN; - } - - /** - * Get the number of audio channels. - * - *

- * It is only known for audio itags, so {@link #AUDIO_CHANNELS_NOT_APPLICABLE_OR_UNKNOWN} is - * returned for non audio itags, or if it is unknown. - *

- * - * @return the number of audio channels or {@link #AUDIO_CHANNELS_NOT_APPLICABLE_OR_UNKNOWN} - */ - public int getAudioChannels() { - return audioChannels; - } - - /** - * Set the number of audio channels. - * - *

- * It is only known for audio itags, so {@link #AUDIO_CHANNELS_NOT_APPLICABLE_OR_UNKNOWN} is - * set/used for non audio itags, or if the {@code audioChannels} value is less than or equal to - * 0. - *

- * - * @param audioChannels the number of audio channels of an audio itag - */ - public void setAudioChannels(final int audioChannels) { - this.audioChannels = audioChannels > 0 - ? audioChannels - : AUDIO_CHANNELS_NOT_APPLICABLE_OR_UNKNOWN; - } - - /** - * Get the {@code targetDurationSec} value. - * - *

- * This value is the average time in seconds of the duration of sequences of livestreams and - * ended livestreams. It is only returned by YouTube for these stream types, and makes no sense - * for videos, so {@link #TARGET_DURATION_SEC_UNKNOWN} is returned for those. - *

- * - * @return the {@code targetDurationSec} value or {@link #TARGET_DURATION_SEC_UNKNOWN} - */ - public int getTargetDurationSec() { - return targetDurationSec; - } - - /** - * Set the {@code targetDurationSec} value. - * - *

- * This value is the average time in seconds of the duration of sequences of livestreams and - * ended livestreams. - *

- * - *

- * It is only returned for these stream types by YouTube and makes no sense for videos, so - * {@link #TARGET_DURATION_SEC_UNKNOWN} will be set/used for video streams or if this value is - * less than or equal to 0. - *

- * - * @param targetDurationSec the target duration of a segment of streams which are using the - * live delivery method type - */ - public void setTargetDurationSec(final int targetDurationSec) { - this.targetDurationSec = targetDurationSec > 0 - ? targetDurationSec - : TARGET_DURATION_SEC_UNKNOWN; - } - - /** - * Get the {@code approxDurationMs} value. - * - *

- * It is only known for DASH progressive streams, so {@link #APPROX_DURATION_MS_UNKNOWN} is - * returned for other stream types or if this value is less than or equal to 0. - *

- * - * @return the {@code approxDurationMs} value or {@link #APPROX_DURATION_MS_UNKNOWN} - */ - public long getApproxDurationMs() { - return approxDurationMs; - } - - /** - * Set the {@code approxDurationMs} value. - * - *

- * It is only known for DASH progressive streams, so {@link #APPROX_DURATION_MS_UNKNOWN} is - * set/used for other stream types or if this value is less than or equal to 0. - *

- * - * @param approxDurationMs the approximate duration of a DASH progressive stream, in - * milliseconds - */ - public void setApproxDurationMs(final long approxDurationMs) { - this.approxDurationMs = approxDurationMs > 0 - ? approxDurationMs - : APPROX_DURATION_MS_UNKNOWN; - } - - /** - * Get the {@code contentLength} value. - * - *

- * It is only known for DASH progressive streams, so {@link #CONTENT_LENGTH_UNKNOWN} is - * returned for other stream types or if this value is less than or equal to 0. - *

- * - * @return the {@code contentLength} value or {@link #CONTENT_LENGTH_UNKNOWN} - */ - public long getContentLength() { - return contentLength; - } - - /** - * Set the content length of stream. - * - *

- * It is only known for DASH progressive streams, so {@link #CONTENT_LENGTH_UNKNOWN} is - * set/used for other stream types or if this value is less than or equal to 0. - *

- * - * @param contentLength the content length of a DASH progressive stream - */ - public void setContentLength(final long contentLength) { - this.contentLength = contentLength > 0 ? contentLength : CONTENT_LENGTH_UNKNOWN; - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 273d17abd7..d9fd8cf283 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -680,18 +680,10 @@ private static void extractClientVersionAndKeyFromHtmlSearchResultsPage() } catch (final Parser.RegexException ignored) { } - if (isNullOrEmpty(key)) { - throw new ParsingException( - // CHECKSTYLE:OFF - "Could not extract YouTube WEB InnerTube API key from HTML search results page"); - // CHECKSTYLE:ON - } - - if (clientVersion == null) { - throw new ParsingException( - // CHECKSTYLE:OFF - "Could not extract YouTube WEB InnerTube client version from HTML search results page"); - // CHECKSTYLE:ON + if (isNullOrEmpty(key) || clientVersion == null) { + throw new ParsingException("Could not extract YouTube WEB InnerTube " + + (isNullOrEmpty(key) ? "API key" : "client version") + + " from HTML search results page"); } keyAndVersionExtracted = true; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/AbstractYoutubeDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/AbstractYoutubeDashManifestCreator.java new file mode 100644 index 0000000000..1333c5db40 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/AbstractYoutubeDashManifestCreator.java @@ -0,0 +1,679 @@ +package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5SimplyEmbeddedPlayerStreamingUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.ADAPTATION_SET; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.AUDIO_CHANNEL_CONFIGURATION; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.MPD; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.PERIOD; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.REPRESENTATION; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.ROLE; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.SEGMENT_TEMPLATE; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.SEGMENT_TIMELINE; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.youtube.itag.format.AudioItagFormat; +import org.schabi.newpipe.extractor.services.youtube.itag.format.VideoItagFormat; +import org.schabi.newpipe.extractor.services.youtube.itag.info.ItagInfo; +import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreationException; +import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreator; +import org.schabi.newpipe.extractor.streamdata.stream.quality.VideoQualityData; +import org.w3c.dom.Attr; +import org.w3c.dom.DOMException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import java.io.IOException; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; + +import javax.annotation.Nonnull; +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; + +/** + * Abstract class for YouTube DASH manifest creation. + * + *

+ * This class includes common methods of manifest creators and useful constants. + *

+ * + *

+ * Generation of DASH documents and their conversion as a string is done using external classes + * from {@link org.w3c.dom} and {@link javax.xml} packages. + *

+ */ +public abstract class AbstractYoutubeDashManifestCreator implements DashManifestCreator { + + /** + * The redirect count limit that this class uses, which is the same limit as OkHttp. + */ + public static final int MAXIMUM_REDIRECT_COUNT = 20; + + /** + * URL parameter of the first sequence for live, post-live-DVR and OTF streams. + */ + public static final String SQ_0 = "&sq=0"; + + /** + * URL parameter of the first stream request made by official clients. + */ + public static final String RN_0 = "&rn=0"; + + /** + * URL parameter specific to web clients. When this param is added, if a redirection occurs, + * the server will not redirect clients to the redirect URL. Instead, it will provide this URL + * as the response body. + */ + public static final String ALR_YES = "&alr=yes"; + + protected final ItagInfo itagInfo; + protected final long durationSecondsFallback; + + protected Document document; + + protected AbstractYoutubeDashManifestCreator( + @Nonnull final ItagInfo itagInfo, + final long durationSecondsFallback) { + this.itagInfo = Objects.requireNonNull(itagInfo); + this.durationSecondsFallback = durationSecondsFallback; + } + + @Nonnull + @Override + public String downloadUrl() { + return itagInfo.getStreamUrl(); + } + + @Override + public long getExpectedContentLength(final Downloader downloader) { + return downloader.getContentLength(itagInfo.getStreamUrl()); + } + + protected boolean isLiveDelivery() { + return false; + } + + // region generate manifest elements + + /** + * Generate a {@link Document} with common manifest creator elements added to it. + * + *

+ * Those are: + *

    + *
  • {@code MPD} (using {@link #generateDocumentAndMpdElement(long)});
    • + *
  • {@code Period} (using {@link #generatePeriodElement()});
    • + *
  • {@code AdaptationSet} (using {@link #generateAdaptationSetElement()});
    • + *
  • {@code Role} (using {@link #generateRoleElement()});
    • + *
  • {@code Representation} (using {@link #generateRepresentationElement()});
    • + *
  • and, for audio streams, {@code AudioChannelConfiguration} (using + * {@link #generateAudioChannelConfigurationElement()}).
    • + *
+ *

+ * + * @param streamDurationMs the duration of the stream, in milliseconds + * @throws DashManifestCreationException May throw a CreationException + */ + protected void generateDocumentAndCommonElements(final long streamDurationMs) { + generateDocumentAndMpdElement(streamDurationMs); + + generatePeriodElement(); + generateAdaptationSetElement(); + generateRoleElement(); + generateRepresentationElement(); + if (itagInfo.getItagFormat() instanceof AudioItagFormat) { + generateAudioChannelConfigurationElement(); + } + } + + /** + * Create a {@link Document} instance and generate the {@code } element of the manifest. + * + *

+ * The generated {@code } element looks like the manifest returned into the player + * response of videos: + *

+ * + *

+ * {@code } + * (where {@code $duration$} represents the duration in seconds (a number with 3 digits after + * the decimal point)). + *

+ * + * @param durationMs the duration of the stream, in milliseconds + * @throws DashManifestCreationException May throw a CreationException + */ + protected void generateDocumentAndMpdElement(final long durationMs) { + try { + newDocument(); + + final Element mpdElement = createElement(MPD); + document.appendChild(mpdElement); + + appendNewAttrWithValue( + mpdElement, "xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"); + + appendNewAttrWithValue(mpdElement, "xmlns", "urn:mpeg:DASH:schema:MPD:2011"); + + appendNewAttrWithValue( + mpdElement, + "xsi:schemaLocation", + "urn:mpeg:DASH:schema:MPD:2011 DASH-MPD.xsd"); + + appendNewAttrWithValue(mpdElement, "minBufferTime", "PT1.500S"); + + appendNewAttrWithValue( + mpdElement, "profiles", "urn:mpeg:dash:profile:full:2011"); + + appendNewAttrWithValue(mpdElement, "type", "static"); + + final String durationSeconds = + String.format(Locale.ENGLISH, "%.3f", durationMs / 1000.0); + appendNewAttrWithValue( + mpdElement, "mediaPresentationDuration", "PT" + durationSeconds + "S"); + } catch (final Exception e) { + throw new DashManifestCreationException( + "Could not generate the DASH manifest or append the MPD document to it", e); + } + } + + /** + * Generate the {@code } element, appended as a child of the {@code } element. + * + *

+ * The {@code } element needs to be generated before this element with + * {@link #generateDocumentAndMpdElement(long)}. + *

+ * + * @throws DashManifestCreationException May throw a CreationException + */ + protected void generatePeriodElement() { + try { + getFirstElementByName(MPD).appendChild(createElement(PERIOD)); + } catch (final DOMException e) { + throw DashManifestCreationException.couldNotAddElement(PERIOD, e); + } + } + + /** + * Generate the {@code } element, appended as a child of the {@code } + * element. + * + *

+ * The {@code } element needs to be generated before this element with + * {@link #generatePeriodElement()}. + *

+ * + * @throws DashManifestCreationException May throw a CreationException + */ + protected void generateAdaptationSetElement() { + try { + final Element adaptationSetElement = createElement(ADAPTATION_SET); + + appendNewAttrWithValue(adaptationSetElement, "id", "0"); + + appendNewAttrWithValue( + adaptationSetElement, + "mimeType", + itagInfo.getItagFormat().mediaFormat().mimeType()); + + appendNewAttrWithValue(adaptationSetElement, "subsegmentAlignment", "true"); + + getFirstElementByName(PERIOD).appendChild(adaptationSetElement); + } catch (final DOMException e) { + throw DashManifestCreationException.couldNotAddElement(ADAPTATION_SET, e); + } + } + + /** + * Generate the {@code } element, appended as a child of the {@code } + * element. + * + *

+ * This element, with its attributes and values, is: + *

+ * + *

+ * {@code } + *

+ * + *

+ * The {@code } element needs to be generated before this element with + * {@link #generateAdaptationSetElement(Document)}). + *

+ * + * @throws DashManifestCreationException May throw a CreationException + */ + protected void generateRoleElement() { + try { + final Element roleElement = createElement(ROLE); + + appendNewAttrWithValue(roleElement, "schemeIdUri", "urn:mpeg:DASH:role:2011"); + + appendNewAttrWithValue(roleElement, "value", "main"); + + getFirstElementByName(ADAPTATION_SET).appendChild(roleElement); + } catch (final DOMException e) { + throw DashManifestCreationException.couldNotAddElement(ROLE, e); + } + } + + /** + * Generate the {@code } element, appended as a child of the + * {@code } element. + * + *

+ * The {@code } element needs to be generated before this element with + * {@link #generateAdaptationSetElement()}). + *

+ * @throws DashManifestCreationException May throw a CreationException + */ + protected void generateRepresentationElement() { + try { + final Element representationElement = createElement(REPRESENTATION); + + appendNewAttrWithValue( + representationElement, "id", itagInfo.getItagFormat().id()); + + final String codec = itagInfo.getCodec(); + if (isNullOrEmpty(codec)) { + throw DashManifestCreationException.couldNotAddElement(ADAPTATION_SET, + "invalid codec=" + codec); + } + + appendNewAttrWithValue( + representationElement, "codecs", codec); + + appendNewAttrWithValue( + representationElement, "startWithSAP", "1"); + + appendNewAttrWithValue( + representationElement, "maxPlayoutRate", "1"); + + final Integer bitrate = itagInfo.getBitRate(); + if (bitrate == null || bitrate <= 0) { + throw DashManifestCreationException.couldNotAddElement( + REPRESENTATION, + "invalid bitrate=" + bitrate); + } + + appendNewAttrWithValue( + representationElement, "bandwidth", bitrate); + + if (itagInfo.getItagFormat() instanceof VideoItagFormat) { + + final VideoQualityData videoQualityData = itagInfo.getCombinedVideoQualityData(); + + if (videoQualityData.height() <= 0 && videoQualityData.width() <= 0) { + throw DashManifestCreationException.couldNotAddElement( + REPRESENTATION, + "both width and height are <= 0"); + } + + if (videoQualityData.width() > 0) { + appendNewAttrWithValue( + representationElement, "width", videoQualityData.width()); + } + + appendNewAttrWithValue( + representationElement, "height", videoQualityData.height()); + + if (videoQualityData.fps() > 0) { + appendNewAttrWithValue( + representationElement, "frameRate", videoQualityData.fps()); + } + } + + if (itagInfo.getItagFormat() instanceof AudioItagFormat + && itagInfo.getAudioSampleRate() != null + && itagInfo.getAudioSampleRate() > 0) { + + appendNewAttrWithValue( + representationElement, + "audioSamplingRate", + itagInfo.getAudioSampleRate()); + } + + getFirstElementByName(ADAPTATION_SET).appendChild(representationElement); + } catch (final DOMException e) { + throw DashManifestCreationException.couldNotAddElement(REPRESENTATION, e); + } + } + + /** + * Generate the {@code } element, appended as a child of the + * {@code } element. + * + *

+ * This method is only used when generating DASH manifests of audio streams. + *

+ * + *

+ * It will produce the following element: + *
+ * {@code + * (where {@code audioChannelsValue} is get from the {@link ItagInfo} passed as the second + * parameter of this method) + *

+ * + *

+ * The {@code } element needs to be generated before this element with + * {@link #generateRepresentationElement()}). + *

+ * + * @throws DashManifestCreationException May throw a CreationException + */ + protected void generateAudioChannelConfigurationElement() { + try { + final Element audioChannelConfigElement = createElement(AUDIO_CHANNEL_CONFIGURATION); + + appendNewAttrWithValue( + audioChannelConfigElement, + "schemeIdUri", + "urn:mpeg:dash:23003:3:audio_channel_configuration:2011"); + + Integer audioChannels = itagInfo.getAudioChannels(); + if (audioChannels != null && audioChannels <= 0) { + // Encountered invalid number of audio channels + // Most audio streams have 2 audio channels, so use this value as fallback + // see also https://github.com/TeamNewPipe/NewPipeExtractor/pull/859 + audioChannels = 2; + } + + appendNewAttrWithValue( + audioChannelConfigElement, "value", audioChannels); + + getFirstElementByName(REPRESENTATION).appendChild(audioChannelConfigElement); + } catch (final DOMException e) { + throw DashManifestCreationException.couldNotAddElement(AUDIO_CHANNEL_CONFIGURATION, e); + } + } + + /** + * Generate the {@code } element, appended as a child of the + * {@code } element. + * + *

+ * This method is only used when generating DASH manifests from OTF and post-live-DVR streams. + *

+ * + *

+ * It will produce a {@code } element with the following attributes: + *

    + *
  • {@code startNumber}, which takes the value {@code 0} for post-live-DVR streams and + * {@code 1} for OTF streams;
    • + *
  • {@code timescale}, which is always {@code 1000};
    • + *
  • {@code media}, which is the base URL of the stream on which is appended + * {@code &sq=$Number$};
    • + *
  • {@code initialization} (only for OTF streams), which is the base URL of the stream + * on which is appended {@link #SQ_0}.
    • + *
+ *

+ * + *

+ * The {@code } element needs to be generated before this element with + * {@link #generateRepresentationElement()}). + *

+ * + * @param baseUrl the base URL of the OTF/post-live-DVR stream + * @throws DashManifestCreationException May throw a CreationException + */ + protected void generateSegmentTemplateElement(@Nonnull final String baseUrl) { + try { + final Element segmentTemplateElement = createElement(SEGMENT_TEMPLATE); + + // The first sequence of post DVR streams is the beginning of the video stream and not + // an initialization segment + appendNewAttrWithValue( + segmentTemplateElement, "startNumber", isLiveDelivery() ? "0" : "1"); + + appendNewAttrWithValue( + segmentTemplateElement, "timescale", "1000"); + + // Post-live-DVR/ended livestreams streams don't require an initialization sequence + if (!isLiveDelivery()) { + appendNewAttrWithValue( + segmentTemplateElement, "initialization", baseUrl + SQ_0); + } + + appendNewAttrWithValue( + segmentTemplateElement, "media", baseUrl + "&sq=$Number$"); + + getFirstElementByName(REPRESENTATION).appendChild(segmentTemplateElement); + } catch (final DOMException e) { + throw DashManifestCreationException.couldNotAddElement(SEGMENT_TEMPLATE, e); + } + } + + /** + * Generate the {@code } element, appended as a child of the + * {@code } element. + * + *

+ * The {@code } element needs to be generated before this element with + * {@link #generateSegmentTemplateElement(String)}. + *

+ * + * @throws DashManifestCreationException May throw a CreationException + */ + protected void generateSegmentTimelineElement() + throws DashManifestCreationException { + try { + final Element segmentTemplateElement = getFirstElementByName(SEGMENT_TEMPLATE); + final Element segmentTimelineElement = createElement(SEGMENT_TIMELINE); + + segmentTemplateElement.appendChild(segmentTimelineElement); + } catch (final DOMException e) { + throw DashManifestCreationException.couldNotAddElement(SEGMENT_TIMELINE, e); + } + } + // endregion + + // region initResponse + + @SuppressWarnings("checkstyle:FinalParameters") + @Nonnull + protected Response getInitializationResponse(@Nonnull final String baseStreamingUrl) { + final boolean isHtml5StreamingUrl = isWebStreamingUrl(baseStreamingUrl) + || isTvHtml5SimplyEmbeddedPlayerStreamingUrl(baseStreamingUrl); + final boolean isAndroidStreamingUrl = isAndroidStreamingUrl(baseStreamingUrl); + final boolean isIosStreamingUrl = isIosStreamingUrl(baseStreamingUrl); + + String streamingUrl = baseStreamingUrl; + if (isHtml5StreamingUrl) { + streamingUrl += ALR_YES; + } + streamingUrl = appendBaseStreamingUrlParams(streamingUrl); + + final Downloader downloader = NewPipe.getDownloader(); + if (isHtml5StreamingUrl) { + return getStreamingWebUrlWithoutRedirects(downloader, streamingUrl); + } else if (isAndroidStreamingUrl || isIosStreamingUrl) { + try { + final Map> headers = new HashMap<>(); + headers.put("User-Agent", Collections.singletonList( + isAndroidStreamingUrl + ? getAndroidUserAgent(null) + : getIosUserAgent(null))); + final byte[] emptyBody = "".getBytes(StandardCharsets.UTF_8); + return downloader.post(streamingUrl, headers, emptyBody); + } catch (final IOException | ExtractionException e) { + throw new DashManifestCreationException("Could not get the " + + (isIosStreamingUrl ? "IOS" : "ANDROID") + " streaming URL response", e); + } + } + + try { + return downloader.get(streamingUrl); + } catch (final IOException | ExtractionException e) { + throw new DashManifestCreationException("Could not get the streaming URL response", e); + } + } + + @Nonnull + protected Response getStreamingWebUrlWithoutRedirects( + @Nonnull final Downloader downloader, + @Nonnull final String streamingUrl) { + try { + final Map> headers = new HashMap<>(); + addClientInfoHeaders(headers); + + String currentStreamingUrl = streamingUrl; + + for (int r = 0; r < MAXIMUM_REDIRECT_COUNT; r++) { + final Response response = downloader.get(currentStreamingUrl, headers); + + final int responseCode = response.responseCode(); + if (responseCode != 200) { + throw new DashManifestCreationException( + "Could not get the initialization URL: HTTP response code " + + responseCode); + } + + // A valid HTTP 1.0+ response should include a Content-Type header, so we can + // require that the response from video servers has this header. + final String responseMimeType = + Objects.requireNonNull( + response.getHeader("Content-Type"), + "Could not get the Content-Type header from the response headers"); + + // The response body is not the redirection URL + if (!responseMimeType.equals("text/plain")) { + return response; + } + + currentStreamingUrl = response.responseBody(); + } + + throw new DashManifestCreationException("Too many redirects"); + + } catch (final IOException | ExtractionException e) { + throw new DashManifestCreationException( + "Could not get the streaming URL response of a HTML5 client", e); + } + } + + @Nonnull + protected String appendBaseStreamingUrlParams(@Nonnull final String baseStreamingUrl) { + return baseStreamingUrl + SQ_0 + RN_0; + } + + // endregion + + // region document util + + /** + * Generate a new {@link DocumentBuilder} secured from XXE attacks, on platforms which + * support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and + * {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link DocumentBuilderFactory} instances. + */ + protected void newDocument() throws ParserConfigurationException { + final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance(); + try { + documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); + documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); + } catch (final Exception ignored) { + // Ignore exceptions as setting these attributes to secure XML generation is not + // supported by all platforms (like the Android implementation) + } + + final DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder(); + document = documentBuilder.newDocument(); + } + + protected Attr appendNewAttrWithValue( + final Element baseElement, + final String name, + final Object value + ) { + return appendNewAttrWithValue(baseElement, name, String.valueOf(value)); + } + + protected Attr appendNewAttrWithValue( + final Element baseElement, + final String name, + final String value + ) { + final Attr attr = document.createAttribute(name); + attr.setValue(value); + baseElement.setAttributeNode(attr); + + return attr; + } + + protected Element getFirstElementByName(final String name) { + return (Element) document.getElementsByTagName(name).item(0); + } + + protected Element createElement(final String name) { + return document.createElement(name); + } + + @SuppressWarnings("squid:S2755") // warning is still shown despite applied solution + protected String documentToXml() throws TransformerException { + /* + * Generate a new {@link TransformerFactory} secured from XXE attacks, on platforms which + * support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and + * {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link TransformerFactory} instances. + */ + final TransformerFactory transformerFactory = TransformerFactory.newInstance(); + try { + transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); + transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); + } catch (final Exception ignored) { + // Ignore exceptions as setting these attributes to secure XML generation is not + // supported by all platforms (like the Android implementation) + } + + final Transformer transformer = transformerFactory.newTransformer(); + transformer.setOutputProperty(OutputKeys.VERSION, "1.0"); + transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); + transformer.setOutputProperty(OutputKeys.STANDALONE, "no"); + + final StringWriter result = new StringWriter(); + transformer.transform(new DOMSource(document), new StreamResult(result)); + + return result.toString(); + } + + protected String documentToXmlSafe() { + try { + return documentToXml(); + } catch (final TransformerException e) { + throw new DashManifestCreationException("Failed to convert XML-document to string", e); + } + } + + // endregion +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/YoutubeOtfDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/YoutubeOtfDashManifestCreator.java new file mode 100644 index 0000000000..df24ece3b6 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/YoutubeOtfDashManifestCreator.java @@ -0,0 +1,158 @@ +package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator; + +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.SEGMENT_TIMELINE; +import static org.schabi.newpipe.extractor.utils.Utils.isBlank; +import static org.schabi.newpipe.extractor.utils.Utils.removeNonDigitCharacters; + +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.services.youtube.itag.info.ItagInfo; +import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreationException; +import org.w3c.dom.Element; + +import java.util.Arrays; +import java.util.stream.Stream; + +import javax.annotation.Nonnull; + +public class YoutubeOtfDashManifestCreator extends AbstractYoutubeDashManifestCreator { + + public YoutubeOtfDashManifestCreator(@Nonnull final ItagInfo itagInfo, + final long durationSecondsFallback) { + super(itagInfo, durationSecondsFallback); + } + + @Nonnull + @Override + public String generateManifest() { + // Try to avoid redirects when streaming the content by saving the last URL we get + // from video servers. + final Response response = getInitializationResponse(itagInfo.getStreamUrl()); + final String otfBaseStreamingUrl = response.latestUrl() + .replace(SQ_0, "") + .replace(RN_0, "") + .replace(ALR_YES, ""); + + final int responseCode = response.responseCode(); + if (responseCode != 200) { + throw new DashManifestCreationException("Could not get the initialization URL: " + + "response code " + responseCode); + } + + final String[] segmentDurations; + + try { + final String[] segmentsAndDurationsResponseSplit = response.responseBody() + // Get the lines with the durations and the following + .split("Segment-Durations-Ms: ")[1] + // Remove the other lines + .split("\n")[0] + // Get all durations and repetitions which are separated by a comma + .split(","); + final int lastIndex = segmentsAndDurationsResponseSplit.length - 1; + segmentDurations = isBlank(segmentsAndDurationsResponseSplit[lastIndex]) + ? Arrays.copyOf(segmentsAndDurationsResponseSplit, lastIndex) + : segmentsAndDurationsResponseSplit; + } catch (final Exception e) { + throw new DashManifestCreationException("Could not get segment durations", e); + } + + long streamDurationMs; + try { + streamDurationMs = getStreamDuration(segmentDurations); + } catch (final DashManifestCreationException e) { + streamDurationMs = durationSecondsFallback * 1000; + } + + generateDocumentAndCommonElements(streamDurationMs); + generateSegmentTemplateElement(otfBaseStreamingUrl); + generateSegmentTimelineElement(); + generateSegmentElementsForOtfStreams(segmentDurations); + + return documentToXmlSafe(); + } + + /** + * Generate segment elements for OTF streams. + * + *

+ * By parsing by the first media sequence, we know how many durations and repetitions there are + * so we just have to loop into segment durations to generate the following elements for each + * duration repeated X times: + *

+ * + *

+ * {@code } + *

+ * + *

+ * If there is no repetition of the duration between two segments, the {@code r} attribute is + * not added to the {@code S} element, as it is not needed. + *

+ * + *

+ * These elements will be appended as children of the {@code } element, which + * needs to be generated before these elements with + * {@link #generateSegmentTimelineElement()}. + *

+ * + * @param segmentDurations the sequences "length" or "length(r=repeat_count" extracted with the + * regular expressions + * @throws DashManifestCreationException May throw a CreationException + */ + protected void generateSegmentElementsForOtfStreams(@Nonnull final String[] segmentDurations) { + try { + final Element segmentTimelineElement = getFirstElementByName(SEGMENT_TIMELINE); + streamAndSplitSegmentDurations(segmentDurations) + .map(segmentLengthRepeat -> { + final Element sElement = createElement("S"); + // There are repetitions of a segment duration in other segments + if (segmentLengthRepeat.length > 1) { + appendNewAttrWithValue(sElement, "r", Integer.parseInt( + removeNonDigitCharacters(segmentLengthRepeat[1]))); + } + + appendNewAttrWithValue( + sElement, "d", Integer.parseInt(segmentLengthRepeat[0])); + return sElement; + }) + .forEach(segmentTimelineElement::appendChild); + } catch (final Exception e) { + throw DashManifestCreationException.couldNotAddElement("segment (S)", e); + } + } + + /** + * Get the duration of an OTF stream. + * + *

+ * The duration of OTF streams is not returned into the player response and needs to be + * calculated by adding the duration of each segment. + *

+ * + * @param segmentDurations the segment duration object extracted from the initialization + * sequence of the stream + * @return the duration of the OTF stream, in milliseconds + * @throws DashManifestCreationException May throw a CreationException + */ + protected long getStreamDuration(@Nonnull final String[] segmentDurations) { + try { + return streamAndSplitSegmentDurations(segmentDurations) + .mapToLong(segmentLengthRepeat -> { + final long segmentLength = Integer.parseInt(segmentLengthRepeat[0]); + final long segmentRepeatCount = segmentLengthRepeat.length > 1 + ? Long.parseLong(removeNonDigitCharacters(segmentLengthRepeat[1])) + : 0; + return segmentLength + segmentRepeatCount * segmentLength; + }) + .sum(); + } catch (final NumberFormatException e) { + throw new DashManifestCreationException( + "Could not get stream length from sequences list", e); + } + } + + protected Stream streamAndSplitSegmentDurations(@Nonnull final String[] durations) { + return Stream.of(durations) + .map(segDuration -> segDuration.split("\\(r=")); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/YoutubePostLiveStreamDvrDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/YoutubePostLiveStreamDvrDashManifestCreator.java new file mode 100644 index 0000000000..1b7c769cf2 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/YoutubePostLiveStreamDvrDashManifestCreator.java @@ -0,0 +1,122 @@ +package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator; + +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.SEGMENT_TIMELINE; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.services.youtube.itag.info.ItagInfo; +import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreationException; +import org.w3c.dom.Attr; +import org.w3c.dom.DOMException; +import org.w3c.dom.Element; + +import java.util.List; +import java.util.Map; + +import javax.annotation.Nonnull; + +public class YoutubePostLiveStreamDvrDashManifestCreator + extends AbstractYoutubeDashManifestCreator { + + public YoutubePostLiveStreamDvrDashManifestCreator(@Nonnull final ItagInfo itagInfo, + final long durationSecondsFallback) { + super(itagInfo, durationSecondsFallback); + } + + @Override + protected boolean isLiveDelivery() { + return true; + } + + @Nonnull + @Override + public String generateManifest() { + final Integer targetDurationSec = itagInfo.getTargetDurationSec(); + if (targetDurationSec == null || targetDurationSec <= 0) { + throw new DashManifestCreationException( + "Invalid value for 'targetDurationSec'=" + targetDurationSec); + } + + // Try to avoid redirects when streaming the content by saving the latest URL we get + // from video servers. + final Response response = getInitializationResponse(itagInfo.getStreamUrl()); + final String realPostLiveStreamDvrStreamingUrl = response.latestUrl() + .replace(SQ_0, "") + .replace(RN_0, "") + .replace(ALR_YES, ""); + + final int responseCode = response.responseCode(); + if (responseCode != 200) { + throw new DashManifestCreationException( + "Could not get the initialization sequence: response code " + responseCode); + } + + final String streamDurationMsString; + final String segmentCount; + try { + final Map> responseHeaders = response.responseHeaders(); + streamDurationMsString = responseHeaders.get("X-Head-Time-Millis").get(0); + segmentCount = responseHeaders.get("X-Head-Seqnum").get(0); + } catch (final IndexOutOfBoundsException e) { + throw new DashManifestCreationException( + "Could not get the value of the X-Head-Time-Millis or the X-Head-Seqnum header", + e); + } + + if (isNullOrEmpty(segmentCount)) { + throw new DashManifestCreationException("Could not get the number of segments"); + } + + long streamDurationMs; + try { + streamDurationMs = Long.parseLong(streamDurationMsString); + } catch (final NumberFormatException e) { + streamDurationMs = durationSecondsFallback * 1000; + } + + generateDocumentAndCommonElements(streamDurationMs); + + generateSegmentTemplateElement(realPostLiveStreamDvrStreamingUrl); + generateSegmentTimelineElement(); + generateSegmentElementForPostLiveDvrStreams(targetDurationSec, segmentCount); + + return documentToXmlSafe(); + } + + /** + * Generate the segment ({@code }) element. + * + *

+ * We don't know the exact duration of segments for post-live-DVR streams but an + * average instead (which is the {@code targetDurationSec} value), so we can use the following + * structure to generate the segment timeline for DASH manifests of ended livestreams: + *
+ * {@code } + *

+ * + * @param targetDurationSeconds the {@code targetDurationSec} value from YouTube player + * response's stream + * @param segmentCount the number of segments + * @throws DashManifestCreationException May throw a CreationException + */ + private void generateSegmentElementForPostLiveDvrStreams( + final int targetDurationSeconds, + @Nonnull final String segmentCount + ) { + try { + final Element sElement = document.createElement("S"); + + appendNewAttrWithValue(sElement, "d", targetDurationSeconds * 1000); + + final Attr rAttribute = document.createAttribute("r"); + rAttribute.setValue(segmentCount); + sElement.setAttributeNode(rAttribute); + + appendNewAttrWithValue(sElement, "r", segmentCount); + + getFirstElementByName(SEGMENT_TIMELINE).appendChild(sElement); + } catch (final DOMException e) { + throw DashManifestCreationException.couldNotAddElement("segment (S)", e); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/YoutubeProgressiveDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/YoutubeProgressiveDashManifestCreator.java new file mode 100644 index 0000000000..5e4a5d074f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/YoutubeProgressiveDashManifestCreator.java @@ -0,0 +1,162 @@ +package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator; + +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.BASE_URL; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.INITIALIZATION; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.MPD; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.REPRESENTATION; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.SEGMENT_BASE; + +import org.schabi.newpipe.extractor.services.youtube.itag.info.ItagInfo; +import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreationException; +import org.w3c.dom.DOMException; +import org.w3c.dom.Element; + +import javax.annotation.Nonnull; + +public class YoutubeProgressiveDashManifestCreator extends AbstractYoutubeDashManifestCreator { + + public YoutubeProgressiveDashManifestCreator(@Nonnull final ItagInfo itagInfo, + final long durationSecondsFallback) { + super(itagInfo, durationSecondsFallback); + } + + @Nonnull + @Override + public String generateManifest() { + final long streamDurationMs; + if (itagInfo.getApproxDurationMs() != null) { + streamDurationMs = itagInfo.getApproxDurationMs(); + } else if (durationSecondsFallback > 0) { + streamDurationMs = durationSecondsFallback * 1000; + } else { + throw DashManifestCreationException.couldNotAddElement(MPD, + "unable to determine duration and fallback is invalid"); + } + + generateDocumentAndCommonElements(streamDurationMs); + generateBaseUrlElement(itagInfo.getStreamUrl()); + generateSegmentBaseElement(); + generateInitializationElement(); + + return documentToXmlSafe(); + } + + /** + * Generate the {@code } element, appended as a child of the + * {@code } element. + * + *

+ * The {@code } element needs to be generated before this element with + * {@link #generateRepresentationElement()}). + *

+ * + * @param baseUrl the base URL of the stream, which must not be null and will be set as the + * content of the {@code } element + * @throws DashManifestCreationException May throw a CreationException + */ + protected void generateBaseUrlElement(@Nonnull final String baseUrl) + throws DashManifestCreationException { + try { + final Element baseURLElement = createElement(BASE_URL); + + baseURLElement.setTextContent(baseUrl); + + getFirstElementByName(REPRESENTATION).appendChild(baseURLElement); + } catch (final DOMException e) { + throw DashManifestCreationException.couldNotAddElement(BASE_URL, e); + } + } + + /** + * Generate the {@code } element, appended as a child of the + * {@code } element. + * + *

+ * It generates the following element: + *
+ * {@code } + *
+ * (where {@code indexStart} and {@code indexEnd} are gotten from the {@link ItagInfo} passed + * as the second parameter) + *

+ * + *

+ * The {@code } element needs to be generated before this element with + * {@link #generateRepresentationElement()}), + * and the {@code BaseURL} element with {@link #generateBaseUrlElement(String)} + * should be generated too. + *

+ * + * @throws DashManifestCreationException May throw a CreationException + */ + protected void generateSegmentBaseElement() { + try { + final Element segmentBaseElement = createElement(SEGMENT_BASE); + + if (itagInfo.getIndexRange() == null + || itagInfo.getIndexRange().start() < 0 + || itagInfo.getIndexRange().end() < 0) { + throw DashManifestCreationException.couldNotAddElement(SEGMENT_BASE, + "invalid index-range: " + itagInfo.getIndexRange()); + } + + appendNewAttrWithValue( + segmentBaseElement, + "indexRange", + itagInfo.getIndexRange().start() + "-" + itagInfo.getIndexRange().end()); + + getFirstElementByName(REPRESENTATION).appendChild(segmentBaseElement); + } catch (final DOMException e) { + throw DashManifestCreationException.couldNotAddElement(SEGMENT_BASE, e); + } + } + + /** + * Generate the {@code } element, appended as a child of the + * {@code } element. + * + *

+ * It generates the following element: + *
+ * {@code } + *
+ * (where {@code indexStart} and {@code indexEnd} are gotten from the {@link ItagInfo} passed + * as the second parameter) + *

+ * + *

+ * The {@code } element needs to be generated before this element with + * {@link #generateSegmentBaseElement()}). + *

+ * + * @throws DashManifestCreationException May throw a CreationException + */ + protected void generateInitializationElement() { + try { + final Element initializationElement = createElement(INITIALIZATION); + + if (itagInfo.getInitRange() == null + || itagInfo.getInitRange().start() < 0 + || itagInfo.getInitRange().end() < 0) { + throw DashManifestCreationException.couldNotAddElement(SEGMENT_BASE, + "invalid (init)-range: " + itagInfo.getInitRange()); + } + + appendNewAttrWithValue( + initializationElement, + "range", + itagInfo.getInitRange().start() + "-" + itagInfo.getInitRange().end()); + + getFirstElementByName(SEGMENT_BASE).appendChild(initializationElement); + } catch (final DOMException e) { + throw DashManifestCreationException.couldNotAddElement(INITIALIZATION, e); + } + } + + + @Nonnull + @Override + protected String appendBaseStreamingUrlParams(@Nonnull final String baseStreamingUrl) { + return baseStreamingUrl + RN_0; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/CreationException.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/CreationException.java deleted file mode 100644 index 46f32664b1..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/CreationException.java +++ /dev/null @@ -1,63 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators; - -import javax.annotation.Nonnull; - -/** - * Exception that is thrown when a YouTube DASH manifest creator encounters a problem - * while creating a manifest. - */ -public final class CreationException extends RuntimeException { - - /** - * Create a new {@link CreationException} with a detail message. - * - * @param message the detail message to add in the exception - */ - public CreationException(final String message) { - super(message); - } - - /** - * Create a new {@link CreationException} with a detail message and a cause. - * @param message the detail message to add in the exception - * @param cause the exception cause of this {@link CreationException} - */ - public CreationException(final String message, final Exception cause) { - super(message, cause); - } - - // Methods to create exceptions easily without having to use big exception messages and to - // reduce duplication - - /** - * Create a new {@link CreationException} with a cause and the following detail message format: - *
- * {@code "Could not add " + element + " element", cause}, where {@code element} is an element - * of a DASH manifest. - * - * @param element the element which was not added to the DASH document - * @param cause the exception which prevented addition of the element to the DASH document - * @return a new {@link CreationException} - */ - @Nonnull - public static CreationException couldNotAddElement(final String element, - final Exception cause) { - return new CreationException("Could not add " + element + " element", cause); - } - - /** - * Create a new {@link CreationException} with a cause and the following detail message format: - *
- * {@code "Could not add " + element + " element: " + reason}, where {@code element} is an - * element of a DASH manifest and {@code reason} the reason why this element cannot be added to - * the DASH document. - * - * @param element the element which was not added to the DASH document - * @param reason the reason message of why the element has been not added to the DASH document - * @return a new {@link CreationException} - */ - @Nonnull - public static CreationException couldNotAddElement(final String element, final String reason) { - return new CreationException("Could not add " + element + " element: " + reason); - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java deleted file mode 100644 index 045e5dda49..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java +++ /dev/null @@ -1,756 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators; - -import org.schabi.newpipe.extractor.MediaFormat; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.services.youtube.DeliveryType; -import org.schabi.newpipe.extractor.services.youtube.ItagItem; -import org.schabi.newpipe.extractor.utils.ManifestCreatorCache; -import org.w3c.dom.Attr; -import org.w3c.dom.DOMException; -import org.w3c.dom.Document; -import org.w3c.dom.Element; - -import javax.annotation.Nonnull; -import javax.xml.XMLConstants; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.OutputKeys; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; - -import java.io.IOException; -import java.io.StringWriter; -import java.nio.charset.StandardCharsets; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; - -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5SimplyEmbeddedPlayerStreamingUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - -/** - * Utilities and constants for YouTube DASH manifest creators. - * - *

- * This class includes common methods of manifest creators and useful constants. - *

- * - *

- * Generation of DASH documents and their conversion as a string is done using external classes - * from {@link org.w3c.dom} and {@link javax.xml} packages. - *

- */ -public final class YoutubeDashManifestCreatorsUtils { - - private YoutubeDashManifestCreatorsUtils() { - } - - /** - * The redirect count limit that this class uses, which is the same limit as OkHttp. - */ - public static final int MAXIMUM_REDIRECT_COUNT = 20; - - /** - * URL parameter of the first sequence for live, post-live-DVR and OTF streams. - */ - public static final String SQ_0 = "&sq=0"; - - /** - * URL parameter of the first stream request made by official clients. - */ - public static final String RN_0 = "&rn=0"; - - /** - * URL parameter specific to web clients. When this param is added, if a redirection occurs, - * the server will not redirect clients to the redirect URL. Instead, it will provide this URL - * as the response body. - */ - public static final String ALR_YES = "&alr=yes"; - - // XML elements of DASH MPD manifests - // see https://www.brendanlong.com/the-structure-of-an-mpeg-dash-mpd.html - public static final String MPD = "MPD"; - public static final String PERIOD = "Period"; - public static final String ADAPTATION_SET = "AdaptationSet"; - public static final String ROLE = "Role"; - public static final String REPRESENTATION = "Representation"; - public static final String AUDIO_CHANNEL_CONFIGURATION = "AudioChannelConfiguration"; - public static final String SEGMENT_TEMPLATE = "SegmentTemplate"; - public static final String SEGMENT_TIMELINE = "SegmentTimeline"; - public static final String BASE_URL = "BaseURL"; - public static final String SEGMENT_BASE = "SegmentBase"; - public static final String INITIALIZATION = "Initialization"; - - /** - * Create an attribute with {@link Document#createAttribute(String)}, assign to it the provided - * name and value, then add it to the provided element using {@link - * Element#setAttributeNode(Attr)}. - * - * @param element element to which to add the created node - * @param doc document to use to create the attribute - * @param name name of the attribute - * @param value value of the attribute, will be set using {@link Attr#setValue(String)} - */ - public static void setAttribute(final Element element, - final Document doc, - final String name, - final String value) { - final Attr attr = doc.createAttribute(name); - attr.setValue(value); - element.setAttributeNode(attr); - } - - /** - * Generate a {@link Document} with common manifest creator elements added to it. - * - *

- * Those are: - *

    - *
  • {@code MPD} (using {@link #generateDocumentAndMpdElement(long)});
    • - *
  • {@code Period} (using {@link #generatePeriodElement(Document)});
    • - *
  • {@code AdaptationSet} (using {@link #generateAdaptationSetElement(Document, - * ItagItem)});
    • - *
  • {@code Role} (using {@link #generateRoleElement(Document)});
    • - *
  • {@code Representation} (using {@link #generateRepresentationElement(Document, - * ItagItem)});
    • - *
  • and, for audio streams, {@code AudioChannelConfiguration} (using - * {@link #generateAudioChannelConfigurationElement(Document, ItagItem)}).
    • - *
- *

- * - * @param itagItem the {@link ItagItem} associated to the stream, which must not be null - * @param streamDuration the duration of the stream, in milliseconds - * @return a {@link Document} with the common elements added in it - */ - @Nonnull - public static Document generateDocumentAndDoCommonElementsGeneration( - @Nonnull final ItagItem itagItem, - final long streamDuration) throws CreationException { - final Document doc = generateDocumentAndMpdElement(streamDuration); - - generatePeriodElement(doc); - generateAdaptationSetElement(doc, itagItem); - generateRoleElement(doc); - generateRepresentationElement(doc, itagItem); - if (itagItem.itagType == ItagItem.ItagType.AUDIO) { - generateAudioChannelConfigurationElement(doc, itagItem); - } - - return doc; - } - - /** - * Create a {@link Document} instance and generate the {@code } element of the manifest. - * - *

- * The generated {@code } element looks like the manifest returned into the player - * response of videos: - *

- * - *

- * {@code } - * (where {@code $duration$} represents the duration in seconds (a number with 3 digits after - * the decimal point)). - *

- * - * @param duration the duration of the stream, in milliseconds - * @return a {@link Document} instance which contains a {@code } element - */ - @Nonnull - public static Document generateDocumentAndMpdElement(final long duration) - throws CreationException { - try { - final Document doc = newDocument(); - - final Element mpdElement = doc.createElement(MPD); - doc.appendChild(mpdElement); - - setAttribute(mpdElement, doc, "xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"); - setAttribute(mpdElement, doc, "xmlns", "urn:mpeg:DASH:schema:MPD:2011"); - setAttribute(mpdElement, doc, "xsi:schemaLocation", - "urn:mpeg:DASH:schema:MPD:2011 DASH-MPD.xsd"); - setAttribute(mpdElement, doc, "minBufferTime", "PT1.500S"); - setAttribute(mpdElement, doc, "profiles", "urn:mpeg:dash:profile:full:2011"); - setAttribute(mpdElement, doc, "type", "static"); - setAttribute(mpdElement, doc, "mediaPresentationDuration", - String.format(Locale.ENGLISH, "PT%.3fS", duration / 1000.0)); - - return doc; - } catch (final Exception e) { - throw new CreationException( - "Could not generate the DASH manifest or append the MPD doc to it", e); - } - } - - /** - * Generate the {@code } element, appended as a child of the {@code } element. - * - *

- * The {@code } element needs to be generated before this element with - * {@link #generateDocumentAndMpdElement(long)}. - *

- * - * @param doc the {@link Document} on which the the {@code } element will be appended - */ - public static void generatePeriodElement(@Nonnull final Document doc) - throws CreationException { - try { - final Element mpdElement = (Element) doc.getElementsByTagName(MPD).item(0); - final Element periodElement = doc.createElement(PERIOD); - mpdElement.appendChild(periodElement); - } catch (final DOMException e) { - throw CreationException.couldNotAddElement(PERIOD, e); - } - } - - /** - * Generate the {@code } element, appended as a child of the {@code } - * element. - * - *

- * The {@code } element needs to be generated before this element with - * {@link #generatePeriodElement(Document)}. - *

- * - * @param doc the {@link Document} on which the {@code } element will be appended - * @param itagItem the {@link ItagItem} corresponding to the stream, which must not be null - */ - public static void generateAdaptationSetElement(@Nonnull final Document doc, - @Nonnull final ItagItem itagItem) - throws CreationException { - try { - final Element periodElement = (Element) doc.getElementsByTagName(PERIOD) - .item(0); - final Element adaptationSetElement = doc.createElement(ADAPTATION_SET); - - setAttribute(adaptationSetElement, doc, "id", "0"); - - final MediaFormat mediaFormat = itagItem.getMediaFormat(); - if (mediaFormat == null || isNullOrEmpty(mediaFormat.getMimeType())) { - throw CreationException.couldNotAddElement(ADAPTATION_SET, - "the MediaFormat or its mime type is null or empty"); - } - - setAttribute(adaptationSetElement, doc, "mimeType", mediaFormat.getMimeType()); - setAttribute(adaptationSetElement, doc, "subsegmentAlignment", "true"); - - periodElement.appendChild(adaptationSetElement); - } catch (final DOMException e) { - throw CreationException.couldNotAddElement(ADAPTATION_SET, e); - } - } - - /** - * Generate the {@code } element, appended as a child of the {@code } - * element. - * - *

- * This element, with its attributes and values, is: - *

- * - *

- * {@code } - *

- * - *

- * The {@code } element needs to be generated before this element with - * {@link #generateAdaptationSetElement(Document, ItagItem)}). - *

- * - * @param doc the {@link Document} on which the the {@code } element will be appended - */ - public static void generateRoleElement(@Nonnull final Document doc) - throws CreationException { - try { - final Element adaptationSetElement = (Element) doc.getElementsByTagName( - ADAPTATION_SET).item(0); - final Element roleElement = doc.createElement(ROLE); - - setAttribute(roleElement, doc, "schemeIdUri", "urn:mpeg:DASH:role:2011"); - setAttribute(roleElement, doc, "value", "main"); - - adaptationSetElement.appendChild(roleElement); - } catch (final DOMException e) { - throw CreationException.couldNotAddElement(ROLE, e); - } - } - - /** - * Generate the {@code } element, appended as a child of the - * {@code } element. - * - *

- * The {@code } element needs to be generated before this element with - * {@link #generateAdaptationSetElement(Document, ItagItem)}). - *

- * - * @param doc the {@link Document} on which the the {@code } element will be - * appended - * @param itagItem the {@link ItagItem} to use, which must not be null - */ - public static void generateRepresentationElement(@Nonnull final Document doc, - @Nonnull final ItagItem itagItem) - throws CreationException { - try { - final Element adaptationSetElement = (Element) doc.getElementsByTagName( - ADAPTATION_SET).item(0); - final Element representationElement = doc.createElement(REPRESENTATION); - - final int id = itagItem.id; - if (id <= 0) { - throw CreationException.couldNotAddElement(REPRESENTATION, - "the id of the ItagItem is <= 0"); - } - setAttribute(representationElement, doc, "id", String.valueOf(id)); - - final String codec = itagItem.getCodec(); - if (isNullOrEmpty(codec)) { - throw CreationException.couldNotAddElement(ADAPTATION_SET, - "the codec value of the ItagItem is null or empty"); - } - setAttribute(representationElement, doc, "codecs", codec); - setAttribute(representationElement, doc, "startWithSAP", "1"); - setAttribute(representationElement, doc, "maxPlayoutRate", "1"); - - final int bitrate = itagItem.getBitrate(); - if (bitrate <= 0) { - throw CreationException.couldNotAddElement(REPRESENTATION, - "the bitrate of the ItagItem is <= 0"); - } - setAttribute(representationElement, doc, "bandwidth", String.valueOf(bitrate)); - - if (itagItem.itagType == ItagItem.ItagType.VIDEO - || itagItem.itagType == ItagItem.ItagType.VIDEO_ONLY) { - final int height = itagItem.getHeight(); - final int width = itagItem.getWidth(); - if (height <= 0 && width <= 0) { - throw CreationException.couldNotAddElement(REPRESENTATION, - "both width and height of the ItagItem are <= 0"); - } - - if (width > 0) { - setAttribute(representationElement, doc, "width", String.valueOf(width)); - } - setAttribute(representationElement, doc, "height", - String.valueOf(itagItem.getHeight())); - - final int fps = itagItem.getFps(); - if (fps > 0) { - setAttribute(representationElement, doc, "frameRate", String.valueOf(fps)); - } - } - - if (itagItem.itagType == ItagItem.ItagType.AUDIO && itagItem.getSampleRate() > 0) { - final Attr audioSamplingRateAttribute = doc.createAttribute( - "audioSamplingRate"); - audioSamplingRateAttribute.setValue(String.valueOf(itagItem.getSampleRate())); - } - - adaptationSetElement.appendChild(representationElement); - } catch (final DOMException e) { - throw CreationException.couldNotAddElement(REPRESENTATION, e); - } - } - - /** - * Generate the {@code } element, appended as a child of the - * {@code } element. - * - *

- * This method is only used when generating DASH manifests of audio streams. - *

- * - *

- * It will produce the following element: - *
- * {@code - * (where {@code audioChannelsValue} is get from the {@link ItagItem} passed as the second - * parameter of this method) - *

- * - *

- * The {@code } element needs to be generated before this element with - * {@link #generateRepresentationElement(Document, ItagItem)}). - *

- * - * @param doc the {@link Document} on which the {@code } element will - * be appended - * @param itagItem the {@link ItagItem} to use, which must not be null - */ - public static void generateAudioChannelConfigurationElement( - @Nonnull final Document doc, - @Nonnull final ItagItem itagItem) throws CreationException { - try { - final Element representationElement = (Element) doc.getElementsByTagName( - REPRESENTATION).item(0); - final Element audioChannelConfigurationElement = doc.createElement( - AUDIO_CHANNEL_CONFIGURATION); - - setAttribute(audioChannelConfigurationElement, doc, "schemeIdUri", - "urn:mpeg:dash:23003:3:audio_channel_configuration:2011"); - - if (itagItem.getAudioChannels() <= 0) { - throw new CreationException("the number of audioChannels in the ItagItem is <= 0: " - + itagItem.getAudioChannels()); - } - setAttribute(audioChannelConfigurationElement, doc, "value", - String.valueOf(itagItem.getAudioChannels())); - - representationElement.appendChild(audioChannelConfigurationElement); - } catch (final DOMException e) { - throw CreationException.couldNotAddElement(AUDIO_CHANNEL_CONFIGURATION, e); - } - } - - /** - * Convert a DASH manifest {@link Document doc} to a string and cache it. - * - * @param originalBaseStreamingUrl the original base URL of the stream - * @param doc the doc to be converted - * @param manifestCreatorCache the {@link ManifestCreatorCache} on which store the string - * generated - * @return the DASH manifest {@link Document doc} converted to a string - */ - public static String buildAndCacheResult( - @Nonnull final String originalBaseStreamingUrl, - @Nonnull final Document doc, - @Nonnull final ManifestCreatorCache manifestCreatorCache) - throws CreationException { - - try { - final String documentXml = documentToXml(doc); - manifestCreatorCache.put(originalBaseStreamingUrl, documentXml); - return documentXml; - } catch (final Exception e) { - throw new CreationException( - "Could not convert the DASH manifest generated to a string", e); - } - } - - /** - * Generate the {@code } element, appended as a child of the - * {@code } element. - * - *

- * This method is only used when generating DASH manifests from OTF and post-live-DVR streams. - *

- * - *

- * It will produce a {@code } element with the following attributes: - *

    - *
  • {@code startNumber}, which takes the value {@code 0} for post-live-DVR streams and - * {@code 1} for OTF streams;
    • - *
  • {@code timescale}, which is always {@code 1000};
    • - *
  • {@code media}, which is the base URL of the stream on which is appended - * {@code &sq=$Number$};
    • - *
  • {@code initialization} (only for OTF streams), which is the base URL of the stream - * on which is appended {@link #SQ_0}.
    • - *
- *

- * - *

- * The {@code } element needs to be generated before this element with - * {@link #generateRepresentationElement(Document, ItagItem)}). - *

- * - * @param doc the {@link Document} on which the {@code } element will - * be appended - * @param baseUrl the base URL of the OTF/post-live-DVR stream - * @param deliveryType the stream {@link DeliveryType delivery type}, which must be either - * {@link DeliveryType#OTF OTF} or {@link DeliveryType#LIVE LIVE} - */ - public static void generateSegmentTemplateElement(@Nonnull final Document doc, - @Nonnull final String baseUrl, - final DeliveryType deliveryType) - throws CreationException { - if (deliveryType != DeliveryType.OTF && deliveryType != DeliveryType.LIVE) { - throw CreationException.couldNotAddElement(SEGMENT_TEMPLATE, "invalid delivery type: " - + deliveryType); - } - - try { - final Element representationElement = (Element) doc.getElementsByTagName( - REPRESENTATION).item(0); - final Element segmentTemplateElement = doc.createElement(SEGMENT_TEMPLATE); - - // The first sequence of post DVR streams is the beginning of the video stream and not - // an initialization segment - setAttribute(segmentTemplateElement, doc, "startNumber", - deliveryType == DeliveryType.LIVE ? "0" : "1"); - setAttribute(segmentTemplateElement, doc, "timescale", "1000"); - - // Post-live-DVR/ended livestreams streams don't require an initialization sequence - if (deliveryType != DeliveryType.LIVE) { - setAttribute(segmentTemplateElement, doc, "initialization", baseUrl + SQ_0); - } - - setAttribute(segmentTemplateElement, doc, "media", baseUrl + "&sq=$Number$"); - - representationElement.appendChild(segmentTemplateElement); - } catch (final DOMException e) { - throw CreationException.couldNotAddElement(SEGMENT_TEMPLATE, e); - } - } - - /** - * Generate the {@code } element, appended as a child of the - * {@code } element. - * - *

- * The {@code } element needs to be generated before this element with - * {@link #generateSegmentTemplateElement(Document, String, DeliveryType)}. - *

- * - * @param doc the {@link Document} on which the the {@code } element will be - * appended - */ - public static void generateSegmentTimelineElement(@Nonnull final Document doc) - throws CreationException { - try { - final Element segmentTemplateElement = (Element) doc.getElementsByTagName( - SEGMENT_TEMPLATE).item(0); - final Element segmentTimelineElement = doc.createElement(SEGMENT_TIMELINE); - - segmentTemplateElement.appendChild(segmentTimelineElement); - } catch (final DOMException e) { - throw CreationException.couldNotAddElement(SEGMENT_TIMELINE, e); - } - } - - /** - * Get the "initialization" {@link Response response} of a stream. - * - *

This method fetches, for OTF streams and for post-live-DVR streams: - *

    - *
  • the base URL of the stream, to which are appended {@link #SQ_0} and - * {@link #RN_0} parameters, with a {@code GET} request for streaming URLs from HTML5 - * clients and a {@code POST} request for the ones from the {@code ANDROID} and the - * {@code IOS} clients;
    • - *
  • for streaming URLs from HTML5 clients, the {@link #ALR_YES} param is also added. - *
    • - *
- *

- * - * @param baseStreamingUrl the base URL of the stream, which must not be null - * @param itagItem the {@link ItagItem} of stream, which must not be null - * @param deliveryType the {@link DeliveryType} of the stream - * @return the "initialization" response, without redirections on the network on which the - * request(s) is/are made - */ - @SuppressWarnings("checkstyle:FinalParameters") - @Nonnull - public static Response getInitializationResponse(@Nonnull String baseStreamingUrl, - @Nonnull final ItagItem itagItem, - final DeliveryType deliveryType) - throws CreationException { - final boolean isHtml5StreamingUrl = isWebStreamingUrl(baseStreamingUrl) - || isTvHtml5SimplyEmbeddedPlayerStreamingUrl(baseStreamingUrl); - final boolean isAndroidStreamingUrl = isAndroidStreamingUrl(baseStreamingUrl); - final boolean isIosStreamingUrl = isIosStreamingUrl(baseStreamingUrl); - if (isHtml5StreamingUrl) { - baseStreamingUrl += ALR_YES; - } - baseStreamingUrl = appendRnSqParamsIfNeeded(baseStreamingUrl, deliveryType); - - final Downloader downloader = NewPipe.getDownloader(); - if (isHtml5StreamingUrl) { - final String mimeTypeExpected = itagItem.getMediaFormat().getMimeType(); - if (!isNullOrEmpty(mimeTypeExpected)) { - return getStreamingWebUrlWithoutRedirects(downloader, baseStreamingUrl, - mimeTypeExpected); - } - } else if (isAndroidStreamingUrl || isIosStreamingUrl) { - try { - final Map> headers = Collections.singletonMap("User-Agent", - Collections.singletonList(isAndroidStreamingUrl - ? getAndroidUserAgent(null) : getIosUserAgent(null))); - final byte[] emptyBody = "".getBytes(StandardCharsets.UTF_8); - return downloader.post(baseStreamingUrl, headers, emptyBody); - } catch (final IOException | ExtractionException e) { - throw new CreationException("Could not get the " - + (isIosStreamingUrl ? "ANDROID" : "IOS") + " streaming URL response", e); - } - } - - try { - return downloader.get(baseStreamingUrl); - } catch (final IOException | ExtractionException e) { - throw new CreationException("Could not get the streaming URL response", e); - } - } - - /** - * Generate a new {@link DocumentBuilder} secured from XXE attacks, on platforms which - * support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and - * {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link DocumentBuilderFactory} instances. - * - * @return an instance of {@link Document} secured against XXE attacks on supported platforms, - * that should then be convertible to an XML string without security problems - */ - private static Document newDocument() throws ParserConfigurationException { - final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance(); - try { - documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); - documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); - } catch (final Exception ignored) { - // Ignore exceptions as setting these attributes to secure XML generation is not - // supported by all platforms (like the Android implementation) - } - - return documentBuilderFactory.newDocumentBuilder().newDocument(); - } - - /** - * Generate a new {@link TransformerFactory} secured from XXE attacks, on platforms which - * support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and - * {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link TransformerFactory} instances. - * - * @param doc the doc to convert, which must have been created using {@link #newDocument()} to - * properly prevent XXE attacks - * @return the doc converted to an XML string, making sure there can't be XXE attacks - */ - // Sonar warning is suppressed because it is still shown even if we apply its solution - @SuppressWarnings("squid:S2755") - private static String documentToXml(@Nonnull final Document doc) - throws TransformerException { - - final TransformerFactory transformerFactory = TransformerFactory.newInstance(); - try { - transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); - transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); - } catch (final Exception ignored) { - // Ignore exceptions as setting these attributes to secure XML generation is not - // supported by all platforms (like the Android implementation) - } - - final Transformer transformer = transformerFactory.newTransformer(); - transformer.setOutputProperty(OutputKeys.VERSION, "1.0"); - transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); - transformer.setOutputProperty(OutputKeys.STANDALONE, "no"); - - final StringWriter result = new StringWriter(); - transformer.transform(new DOMSource(doc), new StreamResult(result)); - - return result.toString(); - } - - /** - * Append {@link #SQ_0} for post-live-DVR and OTF streams and {@link #RN_0} to all streams. - * - * @param baseStreamingUrl the base streaming URL to which the parameter(s) are being appended - * @param deliveryType the {@link DeliveryType} of the stream - * @return the base streaming URL to which the param(s) are appended, depending on the - * {@link DeliveryType} of the stream - */ - @Nonnull - private static String appendRnSqParamsIfNeeded(@Nonnull final String baseStreamingUrl, - @Nonnull final DeliveryType deliveryType) { - return baseStreamingUrl + (deliveryType == DeliveryType.PROGRESSIVE ? "" : SQ_0) + RN_0; - } - - /** - * Get a URL on which no redirection between playback hosts should be present on the network - * and/or IP used to fetch the streaming URL, for HTML5 clients. - * - *

This method will follow redirects which works in the following way: - *

    - *
  1. the {@link #ALR_YES} param is appended to all streaming URLs
    2. - *
  2. if no redirection occurs, the video server will return the streaming data;
    3. - *
  3. if a redirection occurs, the server will respond with HTTP status code 200 and a - * {@code text/plain} mime type. The redirection URL is the response body;
    4. - *
  4. the redirection URL is requested and the steps above from step 2 are repeated, - * until too many redirects are reached of course (the maximum number of redirects is - * {@link #MAXIMUM_REDIRECT_COUNT the same as OkHttp}).
    5. - *
- *

- * - *

- * For non-HTML5 clients, redirections are managed in the standard way in - * {@link #getInitializationResponse(String, ItagItem, DeliveryType)}. - *

- * - * @param downloader the {@link Downloader} instance to be used - * @param streamingUrl the streaming URL which we are trying to get a streaming URL - * without any redirection on the network and/or IP used - * @param responseMimeTypeExpected the response mime type expected from Google video servers - * @return the {@link Response} of the stream, which should have no redirections - */ - @SuppressWarnings("checkstyle:FinalParameters") - @Nonnull - private static Response getStreamingWebUrlWithoutRedirects( - @Nonnull final Downloader downloader, - @Nonnull String streamingUrl, - @Nonnull final String responseMimeTypeExpected) - throws CreationException { - try { - final Map> headers = new HashMap<>(); - addClientInfoHeaders(headers); - - String responseMimeType = ""; - - int redirectsCount = 0; - while (!responseMimeType.equals(responseMimeTypeExpected) - && redirectsCount < MAXIMUM_REDIRECT_COUNT) { - final Response response = downloader.get(streamingUrl, headers); - - final int responseCode = response.responseCode(); - if (responseCode != 200) { - throw new CreationException( - "Could not get the initialization URL: HTTP response code " - + responseCode); - } - - // A valid HTTP 1.0+ response should include a Content-Type header, so we can - // require that the response from video servers has this header. - responseMimeType = Objects.requireNonNull(response.getHeader("Content-Type"), - "Could not get the Content-Type header from the response headers"); - - // The response body is the redirection URL - if (responseMimeType.equals("text/plain")) { - streamingUrl = response.responseBody(); - redirectsCount++; - } else { - return response; - } - } - - if (redirectsCount >= MAXIMUM_REDIRECT_COUNT) { - throw new CreationException( - "Too many redirects when trying to get the the streaming URL response of a " - + "HTML5 client"); - } - - // This should never be reached, but is required because we don't want to return null - // here - throw new CreationException( - "Could not get the streaming URL response of a HTML5 client: unreachable code " - + "reached!"); - } catch (final IOException | ExtractionException e) { - throw new CreationException( - "Could not get the streaming URL response of a HTML5 client", e); - } - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeOtfDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeOtfDashManifestCreator.java deleted file mode 100644 index 46e84df1db..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeOtfDashManifestCreator.java +++ /dev/null @@ -1,265 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators; - -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ALR_YES; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.RN_0; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TIMELINE; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SQ_0; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTemplateElement; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTimelineElement; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.getInitializationResponse; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute; -import static org.schabi.newpipe.extractor.utils.Utils.isBlank; - -import org.schabi.newpipe.extractor.downloader.Response; -import org.schabi.newpipe.extractor.services.youtube.DeliveryType; -import org.schabi.newpipe.extractor.services.youtube.ItagItem; -import org.schabi.newpipe.extractor.utils.ManifestCreatorCache; -import org.schabi.newpipe.extractor.utils.Utils; -import org.w3c.dom.DOMException; -import org.w3c.dom.Document; -import org.w3c.dom.Element; - -import java.util.Arrays; -import java.util.Objects; - -import javax.annotation.Nonnull; - -/** - * Class which generates DASH manifests of YouTube {@link DeliveryType#OTF OTF streams}. - */ -public final class YoutubeOtfDashManifestCreator { - - /** - * Cache of DASH manifests generated for OTF streams. - */ - private static final ManifestCreatorCache OTF_STREAMS_CACHE - = new ManifestCreatorCache<>(); - - private YoutubeOtfDashManifestCreator() { - } - - /** - * Create DASH manifests from a YouTube OTF stream. - * - *

- * OTF streams are YouTube-DASH specific streams which work with sequences and without the need - * to get a manifest (even if one is provided, it is not used by official clients). - *

- * - *

- * They can be found only on videos; mostly those with a small amount of views, or ended - * livestreams which have just been re-encoded as normal videos. - *

- * - *

This method needs: - *

    - *
  • the base URL of the stream (which, if you try to access to it, returns HTTP - * status code 404 after redirects, and if the URL is valid);
    • - *
  • an {@link ItagItem}, which needs to contain the following information: - *
      - *
    • its type (see {@link ItagItem.ItagType}), to identify if the content is - * an audio or a video stream;
      • - *
    • its bitrate;
      • - *
    • its mime type;
      • - *
    • its codec(s);
      • - *
    • for an audio stream: its audio channels;
      • - *
    • for a video stream: its width and height.
      • - *
    - *
    • - *
  • the duration of the video, which will be used if the duration could not be - * parsed from the first sequence of the stream.
    • - *
- *

- * - *

In order to generate the DASH manifest, this method will: - *

    - *
  • request the first sequence of the stream (the base URL on which the first - * sequence parameter is appended (see {@link YoutubeDashManifestCreatorsUtils#SQ_0})) - * with a {@code POST} or {@code GET} request (depending of the client on which the - * streaming URL comes from is a mobile one ({@code POST}) or not ({@code GET}));
    • - *
  • follow its redirection(s), if any;
    • - *
  • save the last URL, remove the first sequence parameter;
    • - *
  • use the information provided in the {@link ItagItem} to generate all - * elements of the DASH manifest.
    • - *
- *

- * - *

- * If the duration cannot be extracted, the {@code durationSecondsFallback} value will be used - * as the stream duration. - *

- * - * @param otfBaseStreamingUrl the base URL of the OTF stream, which must not be null - * @param itagItem the {@link ItagItem} corresponding to the stream, which - * must not be null - * @param durationSecondsFallback the duration of the video, which will be used if the duration - * could not be extracted from the first sequence - * @return the manifest generated into a string - */ - @Nonnull - public static String fromOtfStreamingUrl( - @Nonnull final String otfBaseStreamingUrl, - @Nonnull final ItagItem itagItem, - final long durationSecondsFallback) throws CreationException { - if (OTF_STREAMS_CACHE.containsKey(otfBaseStreamingUrl)) { - return Objects.requireNonNull(OTF_STREAMS_CACHE.get(otfBaseStreamingUrl)).getSecond(); - } - - String realOtfBaseStreamingUrl = otfBaseStreamingUrl; - // Try to avoid redirects when streaming the content by saving the last URL we get - // from video servers. - final Response response = getInitializationResponse(realOtfBaseStreamingUrl, - itagItem, DeliveryType.OTF); - realOtfBaseStreamingUrl = response.latestUrl().replace(SQ_0, "") - .replace(RN_0, "").replace(ALR_YES, ""); - - final int responseCode = response.responseCode(); - if (responseCode != 200) { - throw new CreationException("Could not get the initialization URL: response code " - + responseCode); - } - - final String[] segmentDuration; - - try { - final String[] segmentsAndDurationsResponseSplit = response.responseBody() - // Get the lines with the durations and the following - .split("Segment-Durations-Ms: ")[1] - // Remove the other lines - .split("\n")[0] - // Get all durations and repetitions which are separated by a comma - .split(","); - final int lastIndex = segmentsAndDurationsResponseSplit.length - 1; - if (isBlank(segmentsAndDurationsResponseSplit[lastIndex])) { - segmentDuration = Arrays.copyOf(segmentsAndDurationsResponseSplit, lastIndex); - } else { - segmentDuration = segmentsAndDurationsResponseSplit; - } - } catch (final Exception e) { - throw new CreationException("Could not get segment durations", e); - } - - long streamDuration; - try { - streamDuration = getStreamDuration(segmentDuration); - } catch (final CreationException e) { - streamDuration = durationSecondsFallback * 1000; - } - - final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem, - streamDuration); - - generateSegmentTemplateElement(doc, realOtfBaseStreamingUrl, DeliveryType.OTF); - generateSegmentTimelineElement(doc); - generateSegmentElementsForOtfStreams(segmentDuration, doc); - - return buildAndCacheResult(otfBaseStreamingUrl, doc, OTF_STREAMS_CACHE); - } - - /** - * @return the cache of DASH manifests generated for OTF streams - */ - @Nonnull - public static ManifestCreatorCache getCache() { - return OTF_STREAMS_CACHE; - } - - /** - * Generate segment elements for OTF streams. - * - *

- * By parsing by the first media sequence, we know how many durations and repetitions there are - * so we just have to loop into segment durations to generate the following elements for each - * duration repeated X times: - *

- * - *

- * {@code } - *

- * - *

- * If there is no repetition of the duration between two segments, the {@code r} attribute is - * not added to the {@code S} element, as it is not needed. - *

- * - *

- * These elements will be appended as children of the {@code } element, which - * needs to be generated before these elements with - * {@link YoutubeDashManifestCreatorsUtils#generateSegmentTimelineElement(Document)}. - *

- * - * @param segmentDurations the sequences "length" or "length(r=repeat_count" extracted with the - * regular expressions - * @param doc the {@link Document} on which the {@code } elements will be - * appended - */ - private static void generateSegmentElementsForOtfStreams( - @Nonnull final String[] segmentDurations, - @Nonnull final Document doc) throws CreationException { - try { - final Element segmentTimelineElement = (Element) doc.getElementsByTagName( - SEGMENT_TIMELINE).item(0); - - for (final String segmentDuration : segmentDurations) { - final Element sElement = doc.createElement("S"); - - final String[] segmentLengthRepeat = segmentDuration.split("\\(r="); - // make sure segmentLengthRepeat[0], which is the length, is convertible to int - Integer.parseInt(segmentLengthRepeat[0]); - - // There are repetitions of a segment duration in other segments - if (segmentLengthRepeat.length > 1) { - final int segmentRepeatCount = Integer.parseInt( - Utils.removeNonDigitCharacters(segmentLengthRepeat[1])); - setAttribute(sElement, doc, "r", String.valueOf(segmentRepeatCount)); - } - setAttribute(sElement, doc, "d", segmentLengthRepeat[0]); - - segmentTimelineElement.appendChild(sElement); - } - - } catch (final DOMException | IllegalStateException | IndexOutOfBoundsException - | NumberFormatException e) { - throw CreationException.couldNotAddElement("segment (S)", e); - } - } - - /** - * Get the duration of an OTF stream. - * - *

- * The duration of OTF streams is not returned into the player response and needs to be - * calculated by adding the duration of each segment. - *

- * - * @param segmentDuration the segment duration object extracted from the initialization - * sequence of the stream - * @return the duration of the OTF stream, in milliseconds - */ - private static long getStreamDuration(@Nonnull final String[] segmentDuration) - throws CreationException { - try { - long streamLengthMs = 0; - - for (final String segDuration : segmentDuration) { - final String[] segmentLengthRepeat = segDuration.split("\\(r="); - long segmentRepeatCount = 0; - - // There are repetitions of a segment duration in other segments - if (segmentLengthRepeat.length > 1) { - segmentRepeatCount = Long.parseLong(Utils.removeNonDigitCharacters( - segmentLengthRepeat[1])); - } - - final long segmentLength = Integer.parseInt(segmentLengthRepeat[0]); - streamLengthMs += segmentLength + segmentRepeatCount * segmentLength; - } - - return streamLengthMs; - } catch (final NumberFormatException e) { - throw new CreationException("Could not get stream length from sequences list", e); - } - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java deleted file mode 100644 index 3a5a7dd23d..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java +++ /dev/null @@ -1,217 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators; - -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ALR_YES; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.RN_0; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TIMELINE; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SQ_0; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTemplateElement; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTimelineElement; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.getInitializationResponse; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - -import org.schabi.newpipe.extractor.downloader.Response; -import org.schabi.newpipe.extractor.services.youtube.DeliveryType; -import org.schabi.newpipe.extractor.services.youtube.ItagItem; -import org.schabi.newpipe.extractor.utils.ManifestCreatorCache; -import org.w3c.dom.DOMException; -import org.w3c.dom.Document; -import org.w3c.dom.Element; - -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import javax.annotation.Nonnull; - -/** - * Class which generates DASH manifests of YouTube post-live DVR streams (which use the - * {@link DeliveryType#LIVE LIVE delivery type}). - */ -public final class YoutubePostLiveStreamDvrDashManifestCreator { - - /** - * Cache of DASH manifests generated for post-live-DVR streams. - */ - private static final ManifestCreatorCache POST_LIVE_DVR_STREAMS_CACHE - = new ManifestCreatorCache<>(); - - private YoutubePostLiveStreamDvrDashManifestCreator() { - } - - /** - * Create DASH manifests from a YouTube post-live-DVR stream/ended livestream. - * - *

- * Post-live-DVR streams/ended livestreams are one of the YouTube DASH specific streams which - * works with sequences and without the need to get a manifest (even if one is provided but not - * used by main clients (and is not complete for big ended livestreams because it doesn't - * return the full stream)). - *

- * - *

- * They can be found only on livestreams which have ended very recently (a few hours, most of - * the time) - *

- * - *

This method needs: - *

    - *
  • the base URL of the stream (which, if you try to access to it, returns HTTP - * status code 404 after redirects, and if the URL is valid);
    • - *
  • an {@link ItagItem}, which needs to contain the following information: - *
      - *
    • its type (see {@link ItagItem.ItagType}), to identify if the content is - * an audio or a video stream;
      • - *
    • its bitrate;
      • - *
    • its mime type;
      • - *
    • its codec(s);
      • - *
    • for an audio stream: its audio channels;
      • - *
    • for a video stream: its width and height.
      • - *
    - *
    • - *
  • the duration of the video, which will be used if the duration could not be - * parsed from the first sequence of the stream.
    • - *
- *

- * - *

In order to generate the DASH manifest, this method will: - *

    - *
  • request the first sequence of the stream (the base URL on which the first - * sequence parameter is appended (see {@link YoutubeDashManifestCreatorsUtils#SQ_0})) - * with a {@code POST} or {@code GET} request (depending of the client on which the - * streaming URL comes from is a mobile one ({@code POST}) or not ({@code GET}));
    • - *
  • follow its redirection(s), if any;
    • - *
  • save the last URL, remove the first sequence parameters;
    • - *
  • use the information provided in the {@link ItagItem} to generate all elements - * of the DASH manifest.
    • - *
- *

- * - *

- * If the duration cannot be extracted, the {@code durationSecondsFallback} value will be used - * as the stream duration. - *

- * - * @param postLiveStreamDvrStreamingUrl the base URL of the post-live-DVR stream/ended - * livestream, which must not be null - * @param itagItem the {@link ItagItem} corresponding to the stream, which - * must not be null - * @param targetDurationSec the target duration of each sequence, in seconds (this - * value is returned with the {@code targetDurationSec} - * field for each stream in YouTube's player response) - * @param durationSecondsFallback the duration of the ended livestream, which will be - * used if the duration could not be extracted from the - * first sequence - * @return the manifest generated into a string - */ - @Nonnull - public static String fromPostLiveStreamDvrStreamingUrl( - @Nonnull final String postLiveStreamDvrStreamingUrl, - @Nonnull final ItagItem itagItem, - final int targetDurationSec, - final long durationSecondsFallback) throws CreationException { - if (POST_LIVE_DVR_STREAMS_CACHE.containsKey(postLiveStreamDvrStreamingUrl)) { - return Objects.requireNonNull( - POST_LIVE_DVR_STREAMS_CACHE.get(postLiveStreamDvrStreamingUrl)).getSecond(); - } - - String realPostLiveStreamDvrStreamingUrl = postLiveStreamDvrStreamingUrl; - final String streamDurationString; - final String segmentCount; - - if (targetDurationSec <= 0) { - throw new CreationException("targetDurationSec value is <= 0: " + targetDurationSec); - } - - try { - // Try to avoid redirects when streaming the content by saving the latest URL we get - // from video servers. - final Response response = getInitializationResponse(realPostLiveStreamDvrStreamingUrl, - itagItem, DeliveryType.LIVE); - realPostLiveStreamDvrStreamingUrl = response.latestUrl().replace(SQ_0, "") - .replace(RN_0, "").replace(ALR_YES, ""); - - final int responseCode = response.responseCode(); - if (responseCode != 200) { - throw new CreationException( - "Could not get the initialization sequence: response code " + responseCode); - } - - final Map> responseHeaders = response.responseHeaders(); - streamDurationString = responseHeaders.get("X-Head-Time-Millis").get(0); - segmentCount = responseHeaders.get("X-Head-Seqnum").get(0); - } catch (final IndexOutOfBoundsException e) { - throw new CreationException( - "Could not get the value of the X-Head-Time-Millis or the X-Head-Seqnum header", - e); - } - - if (isNullOrEmpty(segmentCount)) { - throw new CreationException("Could not get the number of segments"); - } - - long streamDuration; - try { - streamDuration = Long.parseLong(streamDurationString); - } catch (final NumberFormatException e) { - streamDuration = durationSecondsFallback; - } - - final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem, - streamDuration); - - generateSegmentTemplateElement(doc, realPostLiveStreamDvrStreamingUrl, - DeliveryType.LIVE); - generateSegmentTimelineElement(doc); - generateSegmentElementForPostLiveDvrStreams(doc, targetDurationSec, segmentCount); - - return buildAndCacheResult(postLiveStreamDvrStreamingUrl, doc, - POST_LIVE_DVR_STREAMS_CACHE); - } - - /** - * @return the cache of DASH manifests generated for post-live-DVR streams - */ - @Nonnull - public static ManifestCreatorCache getCache() { - return POST_LIVE_DVR_STREAMS_CACHE; - } - - /** - * Generate the segment ({@code }) element. - * - *

- * We don't know the exact duration of segments for post-live-DVR streams but an - * average instead (which is the {@code targetDurationSec} value), so we can use the following - * structure to generate the segment timeline for DASH manifests of ended livestreams: - *
- * {@code } - *

- * - * @param doc the {@link Document} on which the {@code } element will - * be appended - * @param targetDurationSeconds the {@code targetDurationSec} value from YouTube player - * response's stream - * @param segmentCount the number of segments, extracted by {@link - * #fromPostLiveStreamDvrStreamingUrl(String, ItagItem, int, long)} - */ - private static void generateSegmentElementForPostLiveDvrStreams( - @Nonnull final Document doc, - final int targetDurationSeconds, - @Nonnull final String segmentCount) throws CreationException { - try { - final Element segmentTimelineElement = (Element) doc.getElementsByTagName( - SEGMENT_TIMELINE).item(0); - final Element sElement = doc.createElement("S"); - - setAttribute(sElement, doc, "d", String.valueOf(targetDurationSeconds * 1000)); - setAttribute(sElement, doc, "r", segmentCount); - - segmentTimelineElement.appendChild(sElement); - } catch (final DOMException e) { - throw CreationException.couldNotAddElement("segment (S)", e); - } - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeProgressiveDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeProgressiveDashManifestCreator.java deleted file mode 100644 index 0f69895bba..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeProgressiveDashManifestCreator.java +++ /dev/null @@ -1,235 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators; - -import org.schabi.newpipe.extractor.services.youtube.DeliveryType; -import org.schabi.newpipe.extractor.services.youtube.ItagItem; -import org.schabi.newpipe.extractor.utils.ManifestCreatorCache; -import org.w3c.dom.DOMException; -import org.w3c.dom.Document; -import org.w3c.dom.Element; - -import javax.annotation.Nonnull; -import java.util.Objects; - -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.BASE_URL; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.INITIALIZATION; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.MPD; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.REPRESENTATION; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_BASE; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute; - -/** - * Class which generates DASH manifests of {@link DeliveryType#PROGRESSIVE YouTube progressive} - * streams. - */ -public final class YoutubeProgressiveDashManifestCreator { - - /** - * Cache of DASH manifests generated for progressive streams. - */ - private static final ManifestCreatorCache PROGRESSIVE_STREAMS_CACHE - = new ManifestCreatorCache<>(); - - private YoutubeProgressiveDashManifestCreator() { - } - - /** - * Create DASH manifests from a YouTube progressive stream. - * - *

- * Progressive streams are YouTube DASH streams which work with range requests and without the - * need to get a manifest. - *

- * - *

- * They can be found on all videos, and for all streams for most of videos which come from a - * YouTube partner, and on videos with a large number of views. - *

- * - *

This method needs: - *

    - *
  • the base URL of the stream (which, if you try to access to it, returns the whole - * stream, after redirects, and if the URL is valid);
    • - *
  • an {@link ItagItem}, which needs to contain the following information: - *
      - *
    • its type (see {@link ItagItem.ItagType}), to identify if the content is - * an audio or a video stream;
      • - *
    • its bitrate;
      • - *
    • its mime type;
      • - *
    • its codec(s);
      • - *
    • for an audio stream: its audio channels;
      • - *
    • for a video stream: its width and height.
      • - *
    - *
    • - *
  • the duration of the video (parameter {@code durationSecondsFallback}), which - * will be used as the stream duration if the duration could not be parsed from the - * {@link ItagItem}.
    • - *
- *

- * - * @param progressiveStreamingBaseUrl the base URL of the progressive stream, which must not be - * null - * @param itagItem the {@link ItagItem} corresponding to the stream, which - * must not be null - * @param durationSecondsFallback the duration of the progressive stream which will be used - * if the duration could not be extracted from the - * {@link ItagItem} - * @return the manifest generated into a string - */ - @Nonnull - public static String fromProgressiveStreamingUrl( - @Nonnull final String progressiveStreamingBaseUrl, - @Nonnull final ItagItem itagItem, - final long durationSecondsFallback) throws CreationException { - if (PROGRESSIVE_STREAMS_CACHE.containsKey(progressiveStreamingBaseUrl)) { - return Objects.requireNonNull( - PROGRESSIVE_STREAMS_CACHE.get(progressiveStreamingBaseUrl)).getSecond(); - } - - final long itagItemDuration = itagItem.getApproxDurationMs(); - final long streamDuration; - if (itagItemDuration != -1) { - streamDuration = itagItemDuration; - } else { - if (durationSecondsFallback > 0) { - streamDuration = durationSecondsFallback * 1000; - } else { - throw CreationException.couldNotAddElement(MPD, "the duration of the stream " - + "could not be determined and durationSecondsFallback is <= 0"); - } - } - - final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem, - streamDuration); - - generateBaseUrlElement(doc, progressiveStreamingBaseUrl); - generateSegmentBaseElement(doc, itagItem); - generateInitializationElement(doc, itagItem); - - return buildAndCacheResult(progressiveStreamingBaseUrl, doc, - PROGRESSIVE_STREAMS_CACHE); - } - - /** - * @return the cache of DASH manifests generated for progressive streams - */ - @Nonnull - public static ManifestCreatorCache getCache() { - return PROGRESSIVE_STREAMS_CACHE; - } - - /** - * Generate the {@code } element, appended as a child of the - * {@code } element. - * - *

- * The {@code } element needs to be generated before this element with - * {@link YoutubeDashManifestCreatorsUtils#generateRepresentationElement(Document, ItagItem)}). - *

- * - * @param doc the {@link Document} on which the {@code } element will be appended - * @param baseUrl the base URL of the stream, which must not be null and will be set as the - * content of the {@code } element - */ - private static void generateBaseUrlElement(@Nonnull final Document doc, - @Nonnull final String baseUrl) - throws CreationException { - try { - final Element representationElement = (Element) doc.getElementsByTagName( - REPRESENTATION).item(0); - final Element baseURLElement = doc.createElement(BASE_URL); - baseURLElement.setTextContent(baseUrl); - representationElement.appendChild(baseURLElement); - } catch (final DOMException e) { - throw CreationException.couldNotAddElement(BASE_URL, e); - } - } - - /** - * Generate the {@code } element, appended as a child of the - * {@code } element. - * - *

- * It generates the following element: - *
- * {@code } - *
- * (where {@code indexStart} and {@code indexEnd} are gotten from the {@link ItagItem} passed - * as the second parameter) - *

- * - *

- * The {@code } element needs to be generated before this element with - * {@link YoutubeDashManifestCreatorsUtils#generateRepresentationElement(Document, ItagItem)}), - * and the {@code BaseURL} element with {@link #generateBaseUrlElement(Document, String)} - * should be generated too. - *

- * - * @param doc the {@link Document} on which the {@code } element will be appended - * @param itagItem the {@link ItagItem} to use, which must not be null - */ - private static void generateSegmentBaseElement(@Nonnull final Document doc, - @Nonnull final ItagItem itagItem) - throws CreationException { - try { - final Element representationElement = (Element) doc.getElementsByTagName( - REPRESENTATION).item(0); - final Element segmentBaseElement = doc.createElement(SEGMENT_BASE); - - final String range = itagItem.getIndexStart() + "-" + itagItem.getIndexEnd(); - if (itagItem.getIndexStart() < 0 || itagItem.getIndexEnd() < 0) { - throw CreationException.couldNotAddElement(SEGMENT_BASE, - "ItagItem's indexStart or " + "indexEnd are < 0: " + range); - } - setAttribute(segmentBaseElement, doc, "indexRange", range); - - representationElement.appendChild(segmentBaseElement); - } catch (final DOMException e) { - throw CreationException.couldNotAddElement(SEGMENT_BASE, e); - } - } - - /** - * Generate the {@code } element, appended as a child of the - * {@code } element. - * - *

- * It generates the following element: - *
- * {@code } - *
- * (where {@code indexStart} and {@code indexEnd} are gotten from the {@link ItagItem} passed - * as the second parameter) - *

- * - *

- * The {@code } element needs to be generated before this element with - * {@link #generateSegmentBaseElement(Document, ItagItem)}). - *

- * - * @param doc the {@link Document} on which the {@code } element will be - * appended - * @param itagItem the {@link ItagItem} to use, which must not be null - */ - private static void generateInitializationElement(@Nonnull final Document doc, - @Nonnull final ItagItem itagItem) - throws CreationException { - try { - final Element segmentBaseElement = (Element) doc.getElementsByTagName( - SEGMENT_BASE).item(0); - final Element initializationElement = doc.createElement(INITIALIZATION); - - final String range = itagItem.getInitStart() + "-" + itagItem.getInitEnd(); - if (itagItem.getInitStart() < 0 || itagItem.getInitEnd() < 0) { - throw CreationException.couldNotAddElement(INITIALIZATION, - "ItagItem's initStart and/or " + "initEnd are/is < 0: " + range); - } - setAttribute(initializationElement, doc, "range", range); - - segmentBaseElement.appendChild(initializationElement); - } catch (final DOMException e) { - throw CreationException.couldNotAddElement(INITIALIZATION, e); - } - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/ItagInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/ItagInfo.java deleted file mode 100644 index c1ac4f5f6d..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/ItagInfo.java +++ /dev/null @@ -1,80 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; - -import org.schabi.newpipe.extractor.services.youtube.ItagItem; - -import javax.annotation.Nonnull; -import java.io.Serializable; - -/** - * Class to build easier {@link org.schabi.newpipe.extractor.stream.Stream}s for - * {@link YoutubeStreamExtractor}. - * - *

- * It stores, per stream: - *

    - *
  • its content (the URL/the base URL of streams);
    • - *
  • whether its content is the URL the content itself or the base URL;
    • - *
  • its associated {@link ItagItem}.
    • - *
- *

- */ -final class ItagInfo implements Serializable { - @Nonnull - private final String content; - @Nonnull - private final ItagItem itagItem; - private boolean isUrl; - - /** - * Creates a new {@code ItagInfo} instance. - * - * @param content the content of the stream, which must be not null - * @param itagItem the {@link ItagItem} associated with the stream, which must be not null - */ - ItagInfo(@Nonnull final String content, - @Nonnull final ItagItem itagItem) { - this.content = content; - this.itagItem = itagItem; - } - - /** - * Sets whether the stream is a URL. - * - * @param isUrl whether the content is a URL - */ - void setIsUrl(final boolean isUrl) { - this.isUrl = isUrl; - } - - /** - * Gets the content stored in this {@code ItagInfo} instance, which is either the URL to the - * content itself or the base URL. - * - * @return the content stored in this {@code ItagInfo} instance - */ - @Nonnull - String getContent() { - return content; - } - - /** - * Gets the {@link ItagItem} associated with this {@code ItagInfo} instance. - * - * @return the {@link ItagItem} associated with this {@code ItagInfo} instance, which is not - * null - */ - @Nonnull - ItagItem getItagItem() { - return itagItem; - } - - /** - * Gets whether the content stored is the URL to the content itself or the base URL of it. - * - * @return whether the content stored is the URL to the content itself or the base URL of it - * @see #getContent() for more details - */ - boolean getIsUrl() { - return isUrl; - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java index a79586e8f4..ba8f4f8d07 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java @@ -4,12 +4,12 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; -import javax.annotation.Nullable; import java.time.OffsetDateTime; import java.time.format.DateTimeParseException; +import javax.annotation.Nullable; + public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor { private final Element entryElement; @@ -17,13 +17,6 @@ public YoutubeFeedInfoItemExtractor(final Element entryElement) { this.entryElement = entryElement; } - @Override - public StreamType getStreamType() { - // It is not possible to determine the stream type using the feed endpoint. - // All entries are considered a video stream. - return StreamType.VIDEO_STREAM; - } - @Override public boolean isAd() { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index a5cb04b456..878424dd63 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -20,8 +20,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import static org.schabi.newpipe.extractor.services.youtube.ItagItem.APPROX_DURATION_MS_UNKNOWN; -import static org.schabi.newpipe.extractor.services.youtube.ItagItem.CONTENT_LENGTH_UNKNOWN; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONTENT_CHECK_OK; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK; @@ -37,6 +35,7 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareIosMobileJsonBuilder; +import static org.schabi.newpipe.extractor.utils.JsonUtils.getNullableInteger; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; @@ -46,7 +45,6 @@ import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; -import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.StreamingService; @@ -65,21 +63,41 @@ import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; -import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptExtractor; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter; +import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeOtfDashManifestCreator; +import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubePostLiveStreamDvrDashManifestCreator; +import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeProgressiveDashManifestCreator; +import org.schabi.newpipe.extractor.services.youtube.itag.delivery.HLSItagFormatDeliveryData; +import org.schabi.newpipe.extractor.services.youtube.itag.delivery.ItagFormatDeliveryData; +import org.schabi.newpipe.extractor.services.youtube.itag.delivery.ProgressiveHTTPItagFormatDeliveryData; +import org.schabi.newpipe.extractor.services.youtube.itag.format.BaseAudioItagFormat; +import org.schabi.newpipe.extractor.services.youtube.itag.format.ItagFormat; +import org.schabi.newpipe.extractor.services.youtube.itag.format.VideoItagFormat; +import org.schabi.newpipe.extractor.services.youtube.itag.format.registry.ItagFormatRegistry; +import org.schabi.newpipe.extractor.services.youtube.itag.info.ItagInfo; +import org.schabi.newpipe.extractor.services.youtube.itag.info.builder.ItagInfoRangeHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; -import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.Frameset; -import org.schabi.newpipe.extractor.stream.Stream; +import org.schabi.newpipe.extractor.stream.Privacy; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamSegment; -import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.stream.SubtitlesStream; -import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.streamdata.delivery.DeliveryData; +import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreator; +import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleDASHManifestDeliveryDataImpl; +import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleHLSDeliveryDataImpl; +import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleProgressiveHTTPDeliveryDataImpl; +import org.schabi.newpipe.extractor.streamdata.format.registry.SubtitleFormatRegistry; +import org.schabi.newpipe.extractor.streamdata.stream.AudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.SubtitleStream; +import org.schabi.newpipe.extractor.streamdata.stream.VideoAudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.VideoStream; +import org.schabi.newpipe.extractor.streamdata.stream.simpleimpl.SimpleAudioStreamImpl; +import org.schabi.newpipe.extractor.streamdata.stream.simpleimpl.SimpleSubtitleStreamImpl; +import org.schabi.newpipe.extractor.streamdata.stream.simpleimpl.SimpleVideoAudioStreamImpl; +import org.schabi.newpipe.extractor.streamdata.stream.simpleimpl.SimpleVideoStreamImpl; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Pair; import org.schabi.newpipe.extractor.utils.Parser; @@ -97,7 +115,9 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.function.BiFunction; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -139,7 +159,9 @@ public static class DeobfuscateException extends ParsingException { private JsonObject videoSecondaryInfoRenderer; private JsonObject playerMicroFormatRenderer; private int ageLimit = -1; - private StreamType streamType; + + private Boolean isLive; + private Boolean isPostLive; // We need to store the contentPlaybackNonces because we need to append them to videoplayback // URLs (with the cpn parameter). @@ -198,7 +220,7 @@ public String getTextualUploadDate() throws ParsingException { } else if (!liveDetails.getString("startTimestamp", "").isEmpty()) { // a running live stream return liveDetails.getString("startTimestamp"); - } else if (getStreamType() == StreamType.LIVE_STREAM) { + } else if (isLive()) { // this should never be reached, but a live stream without upload date is valid return null; } @@ -352,23 +374,22 @@ public long getLength() throws ParsingException { } } - private int getDurationFromFirstAdaptiveFormat(@Nonnull final List streamingDatas) + private int getDurationFromFirstAdaptiveFormat(@Nonnull final List streamingData) throws ParsingException { - for (final JsonObject streamingData : streamingDatas) { - final JsonArray adaptiveFormats = streamingData.getArray(ADAPTIVE_FORMATS); - if (adaptiveFormats.isEmpty()) { - continue; - } - - final String durationMs = adaptiveFormats.getObject(0) - .getString("approxDurationMs"); - try { - return Math.round(Long.parseLong(durationMs) / 1000f); - } catch (final NumberFormatException ignored) { - } - } - - throw new ParsingException("Could not get duration"); + return streamingData.stream() + .map(s -> s.getArray(ADAPTIVE_FORMATS)) + .filter(af -> !af.isEmpty()) + .map(af -> af.getObject(0).getString("approxDurationMs")) + .map(durationMs -> { + try { + return Math.round(Long.parseLong(durationMs) / 1000f); + } catch (final NumberFormatException ignored) { + return null; + } + }) + .filter(Objects::nonNull) + .findFirst() + .orElseThrow(() -> new ParsingException("Could not get duration")); } /** @@ -549,31 +570,31 @@ public String getDashMpdUrl() throws ParsingException { // There is no DASH manifest available in the iOS clients and the DASH manifest of the // Android client doesn't contain all available streams (mainly the WEBM ones) return getManifestUrl( - "dash", + "dashManifestUrl", Arrays.asList(html5StreamingData, androidStreamingData)); } @Nonnull @Override - public String getHlsUrl() throws ParsingException { + public String getHlsMasterPlaylistUrl() throws ParsingException { assertPageFetched(); // Return HLS manifest of the iOS client first because on livestreams, the HLS manifest // returned has separated audio and video streams // Also, on videos, non-iOS clients don't have an HLS manifest URL in their player response return getManifestUrl( - "hls", + "hlsManifestUrl", Arrays.asList(iosStreamingData, html5StreamingData, androidStreamingData)); } @Nonnull - private static String getManifestUrl(@Nonnull final String manifestType, - @Nonnull final List streamingDataObjects) { - final String manifestKey = manifestType + "ManifestUrl"; - + private static String getManifestUrl( + @Nonnull final String manifestKey, + @Nonnull final List streamingDataObjects + ) { return streamingDataObjects.stream() .filter(Objects::nonNull) - .map(streamingDataObject -> streamingDataObject.getString(manifestKey)) + .map(streamingData -> streamingData.getString(manifestKey)) .filter(Objects::nonNull) .findFirst() .orElse(""); @@ -581,23 +602,39 @@ private static String getManifestUrl(@Nonnull final String manifestType, @Override public List getAudioStreams() throws ExtractionException { - assertPageFetched(); - return getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO, - getAudioStreamBuilderHelper(), "audio"); + return buildStrems( + ADAPTIVE_FORMATS, + ItagFormatRegistry.AUDIO_FORMATS, + (itagInfo, deliveryData) -> new SimpleAudioStreamImpl( + itagInfo.getItagFormat().mediaFormat(), + deliveryData, + itagInfo.getCombinedAverageBitrate()), + "audio"); } @Override - public List getVideoStreams() throws ExtractionException { - assertPageFetched(); - return getItags(FORMATS, ItagItem.ItagType.VIDEO, - getVideoStreamBuilderHelper(false), "video"); + public List getVideoStreams() throws ExtractionException { + return buildStrems( + FORMATS, + ItagFormatRegistry.VIDEO_AUDIO_FORMATS, + (itagInfo, deliveryData) -> new SimpleVideoAudioStreamImpl( + itagInfo.getItagFormat().mediaFormat(), + deliveryData, + itagInfo.getCombinedVideoQualityData(), + itagInfo.getCombinedAverageBitrate()), + "video"); } @Override public List getVideoOnlyStreams() throws ExtractionException { - assertPageFetched(); - return getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY, - getVideoStreamBuilderHelper(true), "video-only"); + return buildStrems( + ADAPTIVE_FORMATS, + ItagFormatRegistry.VIDEO_FORMATS, + (itagInfo, deliveryData) -> new SimpleVideoStreamImpl( + itagInfo.getItagFormat().mediaFormat(), + deliveryData, + itagInfo.getCombinedVideoQualityData()), + "video-only"); } /** @@ -621,63 +658,43 @@ private String tryDecryptUrl(final String streamingUrl, final String videoId) { @Override @Nonnull - public List getSubtitlesDefault() throws ParsingException { - return getSubtitles(MediaFormat.TTML); - } - - @Override - @Nonnull - public List getSubtitles(final MediaFormat format) throws ParsingException { + public List getSubtitles() throws ParsingException { assertPageFetched(); - // We cannot store the subtitles list because the media format may change - final List subtitlesToReturn = new ArrayList<>(); - final JsonObject renderer = playerResponse.getObject("captions") - .getObject("playerCaptionsTracklistRenderer"); - final JsonArray captionsArray = renderer.getArray("captionTracks"); - // TODO: use this to apply auto translation to different language from a source language - // final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages"); - - for (int i = 0; i < captionsArray.size(); i++) { - final String languageCode = captionsArray.getObject(i).getString("languageCode"); - final String baseUrl = captionsArray.getObject(i).getString("baseUrl"); - final String vssId = captionsArray.getObject(i).getString("vssId"); - - if (languageCode != null && baseUrl != null && vssId != null) { - final boolean isAutoGenerated = vssId.startsWith("a."); - final String cleanUrl = baseUrl - // Remove preexisting format if exists - .replaceAll("&fmt=[^&]*", "") - // Remove translation language - .replaceAll("&tlang=[^&]*", ""); - - subtitlesToReturn.add(new SubtitlesStream.Builder() - .setContent(cleanUrl + "&fmt=" + format.getSuffix(), true) - .setMediaFormat(format) - .setLanguageCode(languageCode) - .setAutoGenerated(isAutoGenerated) - .build()); - } - } - - return subtitlesToReturn; + return playerResponse + .getObject("captions") + .getObject("playerCaptionsTracklistRenderer") + .getArray("captionTracks") + .stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(jsObj -> jsObj.getString("languageCode") != null + && jsObj.getString("baseUrl") != null + && jsObj.getString("vssId") != null) + .map(jsObj -> new SimpleSubtitleStreamImpl( + SubtitleFormatRegistry.TTML, + new SimpleProgressiveHTTPDeliveryDataImpl(jsObj.getString("baseUrl") + // Remove preexisting format if exists + .replaceAll("&fmt=[^&]*", "") + // Remove translation language + .replaceAll("&tlang=[^&]*", "")), + jsObj.getString("vssId").startsWith("a."), + jsObj.getString("languageCode") + )) + .collect(Collectors.toList()); } @Override - public StreamType getStreamType() { + public boolean isLive() { assertPageFetched(); - return streamType; + return isLive; } - private void setStreamType() { - if (playerResponse.getObject("playabilityStatus").has("liveStreamability")) { - streamType = StreamType.LIVE_STREAM; - } else if (playerResponse.getObject("videoDetails").getBoolean("isPostLiveDvr", false)) { - streamType = StreamType.POST_LIVE_STREAM; - } else { - streamType = StreamType.VIDEO_STREAM; - } + public boolean isPostLive() { + assertPageFetched(); + + return isPostLive; } @Nullable @@ -793,8 +810,6 @@ public void onFetchPage(@Nonnull final Downloader downloader) final boolean isAgeRestricted = playabilityStatus.getString("reason", "") .contains("age"); - setStreamType(); - if (!playerResponse.has(STREAMING_DATA)) { try { fetchTvHtml5EmbedJsonPlayer(contentCountry, localization, videoId); @@ -802,9 +817,12 @@ public void onFetchPage(@Nonnull final Downloader downloader) } } + isLive = playerResponse.getObject("playabilityStatus").has("liveStreamability"); + isPostLive = !isLive + && playerResponse.getObject("videoDetails").getBoolean("isPostLiveDvr", false); + // Refresh the stream type because the stream type may be not properly known for // age-restricted videos - setStreamType(); if (html5StreamingData == null && playerResponse.has(STREAMING_DATA)) { html5StreamingData = playerResponse.getObject(STREAMING_DATA); @@ -829,10 +847,8 @@ public void onFetchPage(@Nonnull final Downloader downloader) .getBytes(StandardCharsets.UTF_8); nextResponse = getJsonPostResponse(NEXT, body, localization); - // streamType can only have LIVE_STREAM, POST_LIVE_STREAM and VIDEO_STREAM values (see - // setStreamType()), so this block will be run only for POST_LIVE_STREAM and VIDEO_STREAM - // values if fetching of the ANDROID client is not forced - if ((!isAgeRestricted && streamType != StreamType.LIVE_STREAM) + // this will only be run for post-live and normal streams + if ((!isAgeRestricted && !isLive) || isAndroidClientFetchForced) { try { fetchAndroidMobileJsonPlayer(contentCountry, localization, videoId); @@ -842,7 +858,7 @@ public void onFetchPage(@Nonnull final Downloader downloader) } } - if ((!isAgeRestricted && streamType == StreamType.LIVE_STREAM) + if ((!isAgeRestricted && isLive) || isIosClientFetchForced) { try { fetchIosMobileJsonPlayer(contentCountry, localization, videoId); @@ -1194,195 +1210,147 @@ private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException { } @Nonnull - private List getItags( + private , + I extends ItagFormat> List buildStrems( final String streamingDataKey, - final ItagItem.ItagType itagTypeWanted, - final java.util.function.Function streamBuilderHelper, - final String streamTypeExceptionMessage) throws ParsingException { + final I[] itagFormats, + final BiFunction, DeliveryData, T> streamBuilder, + final String streamTypeExceptionMessage + ) throws ParsingException { + assertPageFetched(); try { - final String videoId = getId(); - final List streamList = new ArrayList<>(); + if (html5StreamingData == null + && androidStreamingData == null + && iosStreamingData == null) { + return Collections.emptyList(); + } - java.util.stream.Stream.of( + final List> streamingDataAndCpnLoopList = Arrays.asList( // Use the androidStreamingData object first because there is no n param and no // signatureCiphers in streaming URLs of the Android client new Pair<>(androidStreamingData, androidCpn), new Pair<>(html5StreamingData, html5Cpn), // Use the iosStreamingData object in the last position because most of the - // available streams can be extracted with the Android and web clients and also - // because the iOS client is only enabled by default on livestreams + // available streams can be extracted with the Android and web clients and + // also because the iOS client is only enabled by default on livestreams new Pair<>(iosStreamingData, iosCpn) - ) - .flatMap(pair -> getStreamsFromStreamingDataKey(videoId, pair.getFirst(), - streamingDataKey, itagTypeWanted, pair.getSecond())) - .map(streamBuilderHelper) - .forEachOrdered(stream -> { - if (!Stream.containSimilarStream(stream, streamList)) { - streamList.add(stream); - } - }); + ); + + return streamingDataAndCpnLoopList.stream() + .flatMap(p -> buildItagInfoForItags( + p.getFirst(), + streamingDataKey, + itagFormats, + p.getSecond())) + .map(i -> streamBuilder.apply(i, buildDeliveryData(i))) + .collect(Collectors.toList()); - return streamList; } catch (final Exception e) { throw new ParsingException( "Could not get " + streamTypeExceptionMessage + " streams", e); } } - /** - * Get the stream builder helper which will be used to build {@link AudioStream}s in - * {@link #getItags(String, ItagItem.ItagType, java.util.function.Function, String)} - * - *

- * The {@code StreamBuilderHelper} will set the following attributes in the - * {@link AudioStream}s built: + /* + * Note: We build the manifests for YT ourself because the provided ones (according to AudricV) *

    - *
  • the {@link ItagItem}'s id of the stream as its id;
    • - *
  • {@link ItagInfo#getContent()} and {@link ItagInfo#getIsUrl()} as its content and - * and as the value of {@code isUrl};
    • - *
  • the media format returned by the {@link ItagItem} as its media format;
    • - *
  • its average bitrate with the value returned by {@link - * ItagItem#getAverageBitrate()};
    • - *
  • the {@link ItagItem};
    • - *
  • the {@link DeliveryMethod#DASH DASH delivery method}, for OTF streams, live streams - * and ended streams.
    • + *
  • aren't working https://github.com/google/ExoPlayer/issues/2422#issuecomment-283080031 + *
    • + *
  • causes memory problems; TransactionTooLargeException: data parcel size 3174340
    • + *
  • are not always returned, only for videos with OTF streams, or on (ended or not) + * livestreams
    • + *
  • Instead of downloading a 10MB manifest when you can generate one which is 1 or 2MB + * large
    • + *
  • Also, this manifest isn't used at all by modern YouTube clients.
    • *
- *

- * - *

- * Note that the {@link ItagItem} comes from an {@link ItagInfo} instance. - *

- * - * @return a stream builder helper to build {@link AudioStream}s */ @Nonnull - private java.util.function.Function getAudioStreamBuilderHelper() { - return (itagInfo) -> { - final ItagItem itagItem = itagInfo.getItagItem(); - final AudioStream.Builder builder = new AudioStream.Builder() - .setId(String.valueOf(itagItem.id)) - .setContent(itagInfo.getContent(), itagInfo.getIsUrl()) - .setMediaFormat(itagItem.getMediaFormat()) - .setAverageBitrate(itagItem.getAverageBitrate()) - .setItagItem(itagItem); - - if (streamType == StreamType.LIVE_STREAM - || streamType == StreamType.POST_LIVE_STREAM - || !itagInfo.getIsUrl()) { - // For YouTube videos on OTF streams and for all streams of post-live streams - // and live streams, only the DASH delivery method can be used. - builder.setDeliveryMethod(DeliveryMethod.DASH); - } + private > DeliveryData buildDeliveryData(final ItagInfo itagInfo) { + final ItagFormatDeliveryData iDeliveryData = itagInfo.getItagFormat().deliveryData(); + if (iDeliveryData instanceof ProgressiveHTTPItagFormatDeliveryData) { + return new SimpleProgressiveHTTPDeliveryDataImpl(itagInfo.getStreamUrl()); + } else if (iDeliveryData instanceof HLSItagFormatDeliveryData) { + return new SimpleHLSDeliveryDataImpl(itagInfo.getStreamUrl()); + } - return builder.build(); - }; - } + // DASH + // Duration in seconds used as fallback inside the dashManifestCreators + long durationInSec; + try { + durationInSec = getLength(); + } catch (final ParsingException e) { + durationInSec = -1; + } - /** - * Get the stream builder helper which will be used to build {@link VideoStream}s in - * {@link #getItags(String, ItagItem.ItagType, java.util.function.Function, String)} - * - *

- * The {@code StreamBuilderHelper} will set the following attributes in the - * {@link VideoStream}s built: - *

    - *
  • the {@link ItagItem}'s id of the stream as its id;
    • - *
  • {@link ItagInfo#getContent()} and {@link ItagInfo#getIsUrl()} as its content and - * and as the value of {@code isUrl};
    • - *
  • the media format returned by the {@link ItagItem} as its media format;
    • - *
  • whether it is video-only with the {@code areStreamsVideoOnly} parameter
    • - *
  • the {@link ItagItem};
    • - *
  • the resolution, by trying to use, in this order: - *
      - *
    1. the height returned by the {@link ItagItem} + {@code p} + the frame rate if - * it is more than 30;
      2. - *
    2. the default resolution string from the {@link ItagItem};
      3. - *
    3. an empty string.
      4. - *
    - *
    • - *
  • the {@link DeliveryMethod#DASH DASH delivery method}, for OTF streams, live streams - * and ended streams.
    • - *
- * - *

- * Note that the {@link ItagItem} comes from an {@link ItagInfo} instance. - *

- * - * @param areStreamsVideoOnly whether the stream builder helper will set the video - * streams as video-only streams - * @return a stream builder helper to build {@link VideoStream}s - */ - @Nonnull - private java.util.function.Function getVideoStreamBuilderHelper( - final boolean areStreamsVideoOnly) { - return (itagInfo) -> { - final ItagItem itagItem = itagInfo.getItagItem(); - final VideoStream.Builder builder = new VideoStream.Builder() - .setId(String.valueOf(itagItem.id)) - .setContent(itagInfo.getContent(), itagInfo.getIsUrl()) - .setMediaFormat(itagItem.getMediaFormat()) - .setIsVideoOnly(areStreamsVideoOnly) - .setItagItem(itagItem); - - final String resolutionString = itagItem.getResolutionString(); - builder.setResolution(resolutionString != null ? resolutionString - : ""); - - if (streamType != StreamType.VIDEO_STREAM || !itagInfo.getIsUrl()) { - // For YouTube videos on OTF streams and for all streams of post-live streams - // and live streams, only the DASH delivery method can be used. - builder.setDeliveryMethod(DeliveryMethod.DASH); - } + return new SimpleDASHManifestDeliveryDataImpl( + getDashManifestCreatorConstructor(itagInfo).apply(itagInfo, durationInSec)); + } - return builder.build(); - }; + private BiFunction, Long, DashManifestCreator> getDashManifestCreatorConstructor( + final ItagInfo itagInfo + ) { + if ("FORMAT_STREAM_TYPE_OTF".equalsIgnoreCase(itagInfo.getType())) { + return YoutubeOtfDashManifestCreator::new; + } else if (isPostLive()) { + return YoutubePostLiveStreamDvrDashManifestCreator::new; + } + return YoutubeProgressiveDashManifestCreator::new; } @Nonnull - private java.util.stream.Stream getStreamsFromStreamingDataKey( - final String videoId, + private > Stream> buildItagInfoForItags( final JsonObject streamingData, final String streamingDataKey, - @Nonnull final ItagItem.ItagType itagTypeWanted, + final I[] itagFormats, @Nonnull final String contentPlaybackNonce) { if (streamingData == null || !streamingData.has(streamingDataKey)) { - return java.util.stream.Stream.empty(); + return Stream.of(); } - return streamingData.getArray(streamingDataKey).stream() + final String videoId; + try { + videoId = getId(); + } catch (final ParsingException ignored) { + return Stream.of(); + } + return streamingData.getArray(streamingDataKey) + .stream() .filter(JsonObject.class::isInstance) .map(JsonObject.class::cast) + .filter(formatData -> + ItagFormatRegistry.isSupported( + itagFormats, + formatData.getInt("itag"))) .map(formatData -> { try { - final ItagItem itagItem = ItagItem.getItag(formatData.getInt("itag")); - if (itagItem.itagType == itagTypeWanted) { - return buildAndAddItagInfoToList(videoId, formatData, itagItem, - itagItem.itagType, contentPlaybackNonce); - } - } catch (final IOException | ExtractionException ignored) { - // if the itag is not supported and getItag fails, we end up here + final I itagFormat = ItagFormatRegistry.getById( + itagFormats, + formatData.getInt("itag")); + + return buildItagInfo(videoId, formatData, itagFormat, contentPlaybackNonce); + } catch (final Exception ignored) { + return null; } - return null; }) .filter(Objects::nonNull); } - private ItagInfo buildAndAddItagInfoToList( + private > ItagInfo buildItagInfo( @Nonnull final String videoId, @Nonnull final JsonObject formatData, - @Nonnull final ItagItem itagItem, - @Nonnull final ItagItem.ItagType itagType, - @Nonnull final String contentPlaybackNonce) throws IOException, ExtractionException { + @Nonnull final I itagFormat, + @Nonnull final String contentPlaybackNonce + ) throws ParsingException { + + // Build url String streamUrl; if (formatData.has("url")) { streamUrl = formatData.getString("url"); } else { // This url has an obfuscated signature - final String cipherString = formatData.has(CIPHER) + final Map cipher = Parser.compatParseMap(formatData.has(CIPHER) ? formatData.getString(CIPHER) - : formatData.getString(SIGNATURE_CIPHER); - final Map cipher = Parser.compatParseMap( - cipherString); + : formatData.getString(SIGNATURE_CIPHER)); streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + deobfuscateSignature(cipher.get("s")); } @@ -1393,59 +1361,55 @@ private ItagInfo buildAndAddItagInfoToList( // Decrypt the n parameter if it is present streamUrl = tryDecryptUrl(streamUrl, videoId); - final JsonObject initRange = formatData.getObject("initRange"); - final JsonObject indexRange = formatData.getObject("indexRange"); + final ItagInfo itagInfo = new ItagInfo<>(itagFormat, streamUrl); + + if (itagFormat instanceof BaseAudioItagFormat) { + final Integer averageBitrate = getNullableInteger(formatData, "averageBitrate"); + if (averageBitrate != null) { + itagInfo.setAverageBitrate((int) Math.round(averageBitrate / 1000d)); + } + // YouTube returns the audio sample rate as a string + try { + itagInfo.setAudioSampleRate( + Integer.parseInt(formatData.getString("audioSampleRate"))); + } catch (final Exception ignore) { + // Ignore errors - leave default value + } + itagInfo.setAudioChannels(getNullableInteger(formatData, "audioChannels")); + } + if (itagFormat instanceof VideoItagFormat) { + itagInfo.setHeight(getNullableInteger(formatData, "height")); + itagInfo.setWidth(getNullableInteger(formatData, "width")); + itagInfo.setFps(getNullableInteger(formatData, "fps")); + } + + itagInfo.setBitRate(getNullableInteger(formatData, "bitrate")); + itagInfo.setQuality(formatData.getString("quality")); + final String mimeType = formatData.getString("mimeType", ""); - final String codec = mimeType.contains("codecs") - ? mimeType.split("\"")[1] : ""; - - itagItem.setBitrate(formatData.getInt("bitrate")); - itagItem.setWidth(formatData.getInt("width")); - itagItem.setHeight(formatData.getInt("height")); - itagItem.setInitStart(Integer.parseInt(initRange.getString("start", "-1"))); - itagItem.setInitEnd(Integer.parseInt(initRange.getString("end", "-1"))); - itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start", "-1"))); - itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end", "-1"))); - itagItem.setQuality(formatData.getString("quality")); - itagItem.setCodec(codec); - - if (streamType == StreamType.LIVE_STREAM || streamType == StreamType.POST_LIVE_STREAM) { - itagItem.setTargetDurationSec(formatData.getInt("targetDurationSec")); - } - - if (itagType == ItagItem.ItagType.VIDEO || itagType == ItagItem.ItagType.VIDEO_ONLY) { - itagItem.setFps(formatData.getInt("fps")); - } else if (itagType == ItagItem.ItagType.AUDIO) { - // YouTube return the audio sample rate as a string - itagItem.setSampleRate(Integer.parseInt(formatData.getString("audioSampleRate"))); - itagItem.setAudioChannels(formatData.getInt("audioChannels", - // Most audio streams have two audio channels, so use this value if the real - // count cannot be extracted - // Doing this prevents an exception when generating the - // AudioChannelConfiguration element of DASH manifests of audio streams in - // YoutubeDashManifestCreatorUtils - 2)); - } - - // YouTube return the content length and the approximate duration as strings - itagItem.setContentLength(Long.parseLong(formatData.getString("contentLength", - String.valueOf(CONTENT_LENGTH_UNKNOWN)))); - itagItem.setApproxDurationMs(Long.parseLong(formatData.getString("approxDurationMs", - String.valueOf(APPROX_DURATION_MS_UNKNOWN)))); - - final ItagInfo itagInfo = new ItagInfo(streamUrl, itagItem); - - if (streamType == StreamType.VIDEO_STREAM) { - itagInfo.setIsUrl(!formatData.getString("type", "") - .equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")); - } else { - // We are currently not able to generate DASH manifests for running - // livestreams, so because of the requirements of StreamInfo - // objects, return these streams as DASH URL streams (even if they - // are not playable). - // Ended livestreams are returned as non URL streams - itagInfo.setIsUrl(streamType != StreamType.POST_LIVE_STREAM); + itagInfo.setCodec(mimeType.contains("codecs") + ? mimeType.split("\"")[1] + : null); + + itagInfo.setInitRange(ItagInfoRangeHelper.buildFrom(formatData.getObject("initRange"))); + itagInfo.setIndexRange(ItagInfoRangeHelper.buildFrom(formatData.getObject("indexRange"))); + + // YouTube returns the content length and the approximate duration as strings + try { + itagInfo.setContentLength( + Long.parseLong(formatData.getString("contentLength"))); + } catch (final Exception ignored) { + // Ignore errors - leave default value } + try { + itagInfo.setApproxDurationMs( + Long.parseLong(formatData.getString("approxDurationMs"))); + } catch (final Exception ignored) { + // Ignore errors - leave default value + } + + itagInfo.setType(formatData.getString("type")); + itagInfo.setTargetDurationSec(getNullableInteger(formatData, "targetDurationSec")); return itagInfo; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index d731729519..239b254a63 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -14,7 +14,6 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; @@ -46,7 +45,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { private final JsonObject videoInfo; private final TimeAgoParser timeAgoParser; - private StreamType cachedStreamType; + private Boolean cachedIsLive; /** * Creates an extractor of StreamInfoItems from a YouTube page. @@ -61,19 +60,23 @@ public YoutubeStreamInfoItemExtractor(final JsonObject videoInfoItem, } @Override - public StreamType getStreamType() { - if (cachedStreamType != null) { - return cachedStreamType; + public boolean isLive() { + if (cachedIsLive != null) { + return cachedIsLive; } + cachedIsLive = determineIfIsLive(); + return cachedIsLive; + } + + private boolean determineIfIsLive() { final JsonArray badges = videoInfo.getArray("badges"); for (final Object badge : badges) { final JsonObject badgeRenderer = ((JsonObject) badge).getObject("metadataBadgeRenderer"); - if (badgeRenderer.getString("style", "").equals("BADGE_STYLE_TYPE_LIVE_NOW") - || badgeRenderer.getString("label", "").equals("LIVE NOW")) { - cachedStreamType = StreamType.LIVE_STREAM; - return cachedStreamType; + if ("BADGE_STYLE_TYPE_LIVE_NOW".equals(badgeRenderer.getString("style")) + || "LIVE NOW".equals(badgeRenderer.getString("label"))) { + return true; } } @@ -82,13 +85,11 @@ public StreamType getStreamType() { .getObject("thumbnailOverlayTimeStatusRenderer") .getString("style", ""); if (style.equalsIgnoreCase("LIVE")) { - cachedStreamType = StreamType.LIVE_STREAM; - return cachedStreamType; + return true; } } - cachedStreamType = StreamType.VIDEO_STREAM; - return cachedStreamType; + return false; } @Override @@ -118,7 +119,7 @@ public String getName() throws ParsingException { @Override public long getDuration() throws ParsingException { - if (getStreamType() == StreamType.LIVE_STREAM || isPremiere()) { + if (isLive() || isPremiere()) { return -1; } @@ -212,7 +213,7 @@ public boolean isUploaderVerified() throws ParsingException { @Nullable @Override public String getTextualUploadDate() throws ParsingException { - if (getStreamType().equals(StreamType.LIVE_STREAM)) { + if (isLive()) { return null; } @@ -232,7 +233,7 @@ public String getTextualUploadDate() throws ParsingException { @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { - if (getStreamType().equals(StreamType.LIVE_STREAM)) { + if (isLive()) { return null; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/DASHItagFormatDeliveryData.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/DASHItagFormatDeliveryData.java new file mode 100644 index 0000000000..26df3ff3c7 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/DASHItagFormatDeliveryData.java @@ -0,0 +1,5 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.delivery; + +public interface DASHItagFormatDeliveryData extends ItagFormatDeliveryData { + // Just a marker interface for now +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/HLSItagFormatDeliveryData.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/HLSItagFormatDeliveryData.java new file mode 100644 index 0000000000..aed19acaa2 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/HLSItagFormatDeliveryData.java @@ -0,0 +1,5 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.delivery; + +public interface HLSItagFormatDeliveryData extends ItagFormatDeliveryData { + // Just a marker interface for now +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/ItagFormatDeliveryData.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/ItagFormatDeliveryData.java new file mode 100644 index 0000000000..c66bb41736 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/ItagFormatDeliveryData.java @@ -0,0 +1,5 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.delivery; + +public interface ItagFormatDeliveryData { + // Just a marker interface +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/ProgressiveHTTPItagFormatDeliveryData.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/ProgressiveHTTPItagFormatDeliveryData.java new file mode 100644 index 0000000000..825f702f5c --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/ProgressiveHTTPItagFormatDeliveryData.java @@ -0,0 +1,5 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.delivery; + +public interface ProgressiveHTTPItagFormatDeliveryData extends ItagFormatDeliveryData { + // Just a marker interface for now +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/simpleimpl/SimpleDASHItagFormatDeliveryData.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/simpleimpl/SimpleDASHItagFormatDeliveryData.java new file mode 100644 index 0000000000..6f41ac7ad8 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/simpleimpl/SimpleDASHItagFormatDeliveryData.java @@ -0,0 +1,7 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.delivery.simpleimpl; + +import org.schabi.newpipe.extractor.services.youtube.itag.delivery.DASHItagFormatDeliveryData; + +public class SimpleDASHItagFormatDeliveryData implements DASHItagFormatDeliveryData { + // Just a marker for now +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/simpleimpl/SimpleHLSItagFormatDeliveryData.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/simpleimpl/SimpleHLSItagFormatDeliveryData.java new file mode 100644 index 0000000000..8fedc97336 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/simpleimpl/SimpleHLSItagFormatDeliveryData.java @@ -0,0 +1,7 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.delivery.simpleimpl; + +import org.schabi.newpipe.extractor.services.youtube.itag.delivery.HLSItagFormatDeliveryData; + +public class SimpleHLSItagFormatDeliveryData implements HLSItagFormatDeliveryData { + // Just a marker for now +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/simpleimpl/SimpleItagDeliveryDataBuilder.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/simpleimpl/SimpleItagDeliveryDataBuilder.java new file mode 100644 index 0000000000..70f90a8e3d --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/simpleimpl/SimpleItagDeliveryDataBuilder.java @@ -0,0 +1,23 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.delivery.simpleimpl; + +import org.schabi.newpipe.extractor.services.youtube.itag.delivery.DASHItagFormatDeliveryData; +import org.schabi.newpipe.extractor.services.youtube.itag.delivery.HLSItagFormatDeliveryData; +import org.schabi.newpipe.extractor.services.youtube.itag.delivery.ProgressiveHTTPItagFormatDeliveryData; + +public final class SimpleItagDeliveryDataBuilder { + private SimpleItagDeliveryDataBuilder() { + // No impl + } + + public static ProgressiveHTTPItagFormatDeliveryData progressiveHTTP() { + return new SimpleProgressiveHTTPItagFormatDeliveryData(); + } + + public static HLSItagFormatDeliveryData hls() { + return new SimpleHLSItagFormatDeliveryData(); + } + + public static DASHItagFormatDeliveryData dash() { + return new SimpleDASHItagFormatDeliveryData(); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/simpleimpl/SimpleProgressiveHTTPItagFormatDeliveryData.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/simpleimpl/SimpleProgressiveHTTPItagFormatDeliveryData.java new file mode 100644 index 0000000000..db805f874a --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/delivery/simpleimpl/SimpleProgressiveHTTPItagFormatDeliveryData.java @@ -0,0 +1,8 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.delivery.simpleimpl; + +import org.schabi.newpipe.extractor.services.youtube.itag.delivery.ProgressiveHTTPItagFormatDeliveryData; + +public class SimpleProgressiveHTTPItagFormatDeliveryData + implements ProgressiveHTTPItagFormatDeliveryData { + // Just a marker for now +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/AudioItagFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/AudioItagFormat.java new file mode 100644 index 0000000000..e4268a24f5 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/AudioItagFormat.java @@ -0,0 +1,7 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.format; + +import org.schabi.newpipe.extractor.streamdata.format.AudioMediaFormat; + +public interface AudioItagFormat extends ItagFormat, BaseAudioItagFormat { + // Nothing additional to implement +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/BaseAudioItagFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/BaseAudioItagFormat.java new file mode 100644 index 0000000000..d1faef556f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/BaseAudioItagFormat.java @@ -0,0 +1,8 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.format; + +public interface BaseAudioItagFormat { + /** + * Average audio bitrate in KBit/s + */ + int averageBitrate(); +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/ItagFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/ItagFormat.java new file mode 100644 index 0000000000..dc9136ee02 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/ItagFormat.java @@ -0,0 +1,21 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.format; + +import org.schabi.newpipe.extractor.services.youtube.itag.delivery.ItagFormatDeliveryData; +import org.schabi.newpipe.extractor.streamdata.format.MediaFormat; + +import javax.annotation.Nonnull; + +public interface ItagFormat { + int id(); + + /** + * The (container) media format, e.g. mp3 for audio streams or webm for video(+audio) streams. + * + * @return The (container) media format + */ + @Nonnull + M mediaFormat(); + + @Nonnull + ItagFormatDeliveryData deliveryData(); +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/VideoAudioItagFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/VideoAudioItagFormat.java new file mode 100644 index 0000000000..396cd51d4a --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/VideoAudioItagFormat.java @@ -0,0 +1,5 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.format; + +public interface VideoAudioItagFormat extends VideoItagFormat, BaseAudioItagFormat { + // Nothing additional to implement +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/VideoItagFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/VideoItagFormat.java new file mode 100644 index 0000000000..366b351700 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/VideoItagFormat.java @@ -0,0 +1,11 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.format; + +import org.schabi.newpipe.extractor.streamdata.format.VideoAudioMediaFormat; +import org.schabi.newpipe.extractor.streamdata.stream.quality.VideoQualityData; + +import javax.annotation.Nonnull; + +public interface VideoItagFormat extends ItagFormat { + @Nonnull + VideoQualityData videoQualityData(); +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/registry/ItagFormatRegistry.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/registry/ItagFormatRegistry.java new file mode 100644 index 0000000000..030230ee0a --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/registry/ItagFormatRegistry.java @@ -0,0 +1,145 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.format.registry; + +import static org.schabi.newpipe.extractor.services.youtube.itag.delivery.simpleimpl.SimpleItagDeliveryDataBuilder.dash; +import static org.schabi.newpipe.extractor.services.youtube.itag.delivery.simpleimpl.SimpleItagDeliveryDataBuilder.hls; +import static org.schabi.newpipe.extractor.streamdata.format.registry.VideoAudioFormatRegistry.MPEG_4; +import static org.schabi.newpipe.extractor.streamdata.format.registry.VideoAudioFormatRegistry.V3GPP; +import static org.schabi.newpipe.extractor.streamdata.format.registry.VideoAudioFormatRegistry.WEBM; +import static org.schabi.newpipe.extractor.streamdata.stream.quality.VideoQualityData.fromHeight; +import static org.schabi.newpipe.extractor.streamdata.stream.quality.VideoQualityData.fromHeightFps; +import static org.schabi.newpipe.extractor.streamdata.stream.quality.VideoQualityData.fromHeightWidth; + +import org.schabi.newpipe.extractor.services.youtube.itag.format.AudioItagFormat; +import org.schabi.newpipe.extractor.services.youtube.itag.format.ItagFormat; +import org.schabi.newpipe.extractor.services.youtube.itag.format.VideoAudioItagFormat; +import org.schabi.newpipe.extractor.services.youtube.itag.format.VideoItagFormat; +import org.schabi.newpipe.extractor.services.youtube.itag.format.simpleimpl.SimpleAudioItagFormat; +import org.schabi.newpipe.extractor.services.youtube.itag.format.simpleimpl.SimpleVideoAudioItagFormat; +import org.schabi.newpipe.extractor.services.youtube.itag.format.simpleimpl.SimpleVideoItagFormat; +import org.schabi.newpipe.extractor.streamdata.format.registry.AudioFormatRegistry; + +import java.util.stream.Stream; + +// CHECKSTYLE:OFF - Link is too long +/** + * A registry that contains all supported YouTube itags. + *

+ * For additional information you may also check: + *

    + *
  • https://github.com/ytdl-org/youtube-dl/blob/9aa8e5340f3d5ece372b983f8e399277ca1f1fe4/youtube_dl/extractor/youtube.py#L1195
    • + *
  • https://gist.github.com/AgentOak/34d47c65b1d28829bb17c24c04a0096f
    • + *
+ */ +// CHECKSTYLE:ON +public final class ItagFormatRegistry { + + public static final VideoAudioItagFormat[] VIDEO_AUDIO_FORMATS = new VideoAudioItagFormat[]{ + // v-- Video-codec: mp4v; Audio-codec: aac --v + new SimpleVideoAudioItagFormat(17, V3GPP, fromHeightWidth(144, 176), 24), + // v-- Video-codec: h264; Audio-codec: aac --v + new SimpleVideoAudioItagFormat(18, MPEG_4, fromHeightWidth(360, 640), 96), + new SimpleVideoAudioItagFormat(22, MPEG_4, fromHeightWidth(720, 1280), 192), + + // Note: According to yt-dl Itag 34 and 35 are flv-files + new SimpleVideoAudioItagFormat(34, MPEG_4, fromHeightWidth(360, 640), 128), + new SimpleVideoAudioItagFormat(35, MPEG_4, fromHeightWidth(480, 854), 128), + + // Itag 36 is no longer used because the height is unstable and it's not returned by YT + // CHECKSTYLE:OFF - Link is too long + // see also: https://github.com/ytdl-org/youtube-dl/blob/9aa8e5340f3d5ece372b983f8e399277ca1f1fe4/youtube_dl/extractor/youtube.py#L1204 + // CHECKSTYLE:ON + new SimpleVideoAudioItagFormat(37, MPEG_4, fromHeightWidth(1080, 1920), 192), + new SimpleVideoAudioItagFormat(38, MPEG_4, fromHeightWidth(3072, 4092), 192), + + // v-- Video-codec: vp8; Audio-codec: vorbis --v + new SimpleVideoAudioItagFormat(43, WEBM, fromHeightWidth(360, 640), 128), + new SimpleVideoAudioItagFormat(44, WEBM, fromHeightWidth(480, 854), 128), + new SimpleVideoAudioItagFormat(45, WEBM, fromHeightWidth(720, 1280), 192), + new SimpleVideoAudioItagFormat(46, WEBM, fromHeightWidth(1080, 1920), 192), + + // HLS (used for live streaming) + // v-- Video-codec: h264; Audio-codec: acc --v + new SimpleVideoAudioItagFormat(91, MPEG_4, fromHeight(144), 48, hls()), + new SimpleVideoAudioItagFormat(92, MPEG_4, fromHeight(240), 48, hls()), + new SimpleVideoAudioItagFormat(93, MPEG_4, fromHeight(360), 128, hls()), + new SimpleVideoAudioItagFormat(94, MPEG_4, fromHeight(480), 128, hls()), + new SimpleVideoAudioItagFormat(95, MPEG_4, fromHeight(720), 256, hls()), + new SimpleVideoAudioItagFormat(96, MPEG_4, fromHeight(1080), 256, hls()), + new SimpleVideoAudioItagFormat(132, MPEG_4, fromHeight(240), 48, hls()), + new SimpleVideoAudioItagFormat(151, MPEG_4, fromHeight(72), 24, hls()) + }; + + public static final AudioItagFormat[] AUDIO_FORMATS = new AudioItagFormat[] { + // DASH MP4 audio + // v-- Audio-codec: aac --v + new SimpleAudioItagFormat(139, AudioFormatRegistry.M4A, 48, dash()), + new SimpleAudioItagFormat(140, AudioFormatRegistry.M4A, 128, dash()), + new SimpleAudioItagFormat(141, AudioFormatRegistry.M4A, 256, dash()), + + // DASH WEBM audio + // v-- Audio-codec: vorbis --v + new SimpleAudioItagFormat(171, AudioFormatRegistry.WEBMA, 128, dash()), + new SimpleAudioItagFormat(172, AudioFormatRegistry.WEBMA, 256, dash()), + + // DASH WEBM audio with opus inside + // v-- Audio-codec: opus --v + new SimpleAudioItagFormat(249, AudioFormatRegistry.WEBMA_OPUS, 50, dash()), + new SimpleAudioItagFormat(250, AudioFormatRegistry.WEBMA_OPUS, 70, dash()), + new SimpleAudioItagFormat(251, AudioFormatRegistry.WEBMA_OPUS, 160, dash()) + }; + + public static final VideoItagFormat[] VIDEO_FORMATS = new VideoItagFormat[] { + // DASH MP4 video + // v-- Video-codec: h264 --v + new SimpleVideoItagFormat(133, MPEG_4, fromHeight(240), dash()), + new SimpleVideoItagFormat(134, MPEG_4, fromHeight(360), dash()), + new SimpleVideoItagFormat(135, MPEG_4, fromHeight(480), dash()), + new SimpleVideoItagFormat(136, MPEG_4, fromHeight(720), dash()), + new SimpleVideoItagFormat(137, MPEG_4, fromHeight(1080), dash()), + // Itag 138 has an unknown height and is ignored + new SimpleVideoItagFormat(160, MPEG_4, fromHeight(144), dash()), + new SimpleVideoItagFormat(212, MPEG_4, fromHeight(480), dash()), + new SimpleVideoItagFormat(298, MPEG_4, fromHeightFps(720, 60), dash()), + new SimpleVideoItagFormat(299, MPEG_4, fromHeightFps(1080, 60), dash()), + new SimpleVideoItagFormat(266, MPEG_4, fromHeight(2160), dash()), + + // DASH WEBM video + // v-- Video-codec: vp9 --v + new SimpleVideoItagFormat(278, WEBM, fromHeight(144), dash()), + new SimpleVideoItagFormat(242, WEBM, fromHeight(240), dash()), + new SimpleVideoItagFormat(243, WEBM, fromHeight(360), dash()), + // Itag 244, 245 and 246 are identical? + new SimpleVideoItagFormat(244, WEBM, fromHeight(480), dash()), + new SimpleVideoItagFormat(245, WEBM, fromHeight(480), dash()), + new SimpleVideoItagFormat(246, WEBM, fromHeight(480), dash()), + new SimpleVideoItagFormat(247, WEBM, fromHeight(720), dash()), + new SimpleVideoItagFormat(248, WEBM, fromHeight(1080), dash()), + new SimpleVideoItagFormat(271, WEBM, fromHeight(1440), dash()), + // Itag 272 is either 3840x2160 (RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug) + new SimpleVideoItagFormat(272, WEBM, fromHeight(2160), dash()), + + new SimpleVideoItagFormat(302, WEBM, fromHeightFps(720, 60), dash()), + new SimpleVideoItagFormat(303, WEBM, fromHeightFps(1080, 60), dash()), + new SimpleVideoItagFormat(308, WEBM, fromHeightFps(1440, 60), dash()), + new SimpleVideoItagFormat(312, WEBM, fromHeight(2160), dash()), + new SimpleVideoItagFormat(315, WEBM, fromHeightFps(2160, 60), dash()), + }; + + private ItagFormatRegistry() { + // No impl + } + + public static boolean isSupported(final ItagFormat[] itagFormats, final int id) { + return Stream.of(itagFormats) + .flatMap(Stream::of) + .anyMatch(itagFormat -> itagFormat.id() == id); + } + + public static > I getById(final I[] itagFormats, final int id) { + return Stream.of(itagFormats) + .flatMap(Stream::of) + .filter(itagFormat -> itagFormat.id() == id) + .findFirst() + .orElse(null); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/simpleimpl/AbstractSimpleItagFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/simpleimpl/AbstractSimpleItagFormat.java new file mode 100644 index 0000000000..0fad80bca4 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/simpleimpl/AbstractSimpleItagFormat.java @@ -0,0 +1,46 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.format.simpleimpl; + +import org.schabi.newpipe.extractor.services.youtube.itag.delivery.ItagFormatDeliveryData; +import org.schabi.newpipe.extractor.services.youtube.itag.delivery.simpleimpl.SimpleItagDeliveryDataBuilder; +import org.schabi.newpipe.extractor.services.youtube.itag.format.ItagFormat; +import org.schabi.newpipe.extractor.streamdata.format.MediaFormat; + +import java.util.Objects; + +import javax.annotation.Nonnull; + +public abstract class AbstractSimpleItagFormat implements ItagFormat { + + private final int id; + private final M mediaFormat; + private final ItagFormatDeliveryData deliveryData; + + protected AbstractSimpleItagFormat( + final int id, + final M mediaFormat, + final ItagFormatDeliveryData deliveryData) { + this.id = id; + this.mediaFormat = Objects.requireNonNull(mediaFormat); + this.deliveryData = Objects.requireNonNull(deliveryData); + } + + protected AbstractSimpleItagFormat(final int id, final M mediaFormat) { + this(id, mediaFormat, SimpleItagDeliveryDataBuilder.progressiveHTTP()); + } + + @Override + public int id() { + return id; + } + + @Nonnull + @Override + public M mediaFormat() { + return mediaFormat; + } + + @Override + public ItagFormatDeliveryData deliveryData() { + return deliveryData; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/simpleimpl/SimpleAudioItagFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/simpleimpl/SimpleAudioItagFormat.java new file mode 100644 index 0000000000..565b8de70f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/simpleimpl/SimpleAudioItagFormat.java @@ -0,0 +1,30 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.format.simpleimpl; + +import org.schabi.newpipe.extractor.services.youtube.itag.delivery.ItagFormatDeliveryData; +import org.schabi.newpipe.extractor.services.youtube.itag.format.AudioItagFormat; +import org.schabi.newpipe.extractor.streamdata.format.AudioMediaFormat; + +public class SimpleAudioItagFormat extends AbstractSimpleItagFormat + implements AudioItagFormat { + private final int averageBitrate; + + public SimpleAudioItagFormat(final int id, + final AudioMediaFormat mediaFormat, + final int averageBitrate, + final ItagFormatDeliveryData deliveryData) { + super(id, mediaFormat, deliveryData); + this.averageBitrate = averageBitrate; + } + + public SimpleAudioItagFormat(final int id, + final AudioMediaFormat mediaFormat, + final int averageBitrate) { + super(id, mediaFormat); + this.averageBitrate = averageBitrate; + } + + @Override + public int averageBitrate() { + return averageBitrate; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/simpleimpl/SimpleVideoAudioItagFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/simpleimpl/SimpleVideoAudioItagFormat.java new file mode 100644 index 0000000000..09dfd7ce16 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/simpleimpl/SimpleVideoAudioItagFormat.java @@ -0,0 +1,35 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.format.simpleimpl; + +import org.schabi.newpipe.extractor.services.youtube.itag.delivery.ItagFormatDeliveryData; +import org.schabi.newpipe.extractor.services.youtube.itag.format.VideoAudioItagFormat; +import org.schabi.newpipe.extractor.streamdata.format.VideoAudioMediaFormat; +import org.schabi.newpipe.extractor.streamdata.stream.quality.VideoQualityData; + +import javax.annotation.Nonnull; + +public class SimpleVideoAudioItagFormat extends SimpleVideoItagFormat + implements VideoAudioItagFormat { + private final int averageBitrate; + + public SimpleVideoAudioItagFormat(final int id, + @Nonnull final VideoAudioMediaFormat mediaFormat, + @Nonnull final VideoQualityData videoQualityData, + final int averageBitrate, + @Nonnull final ItagFormatDeliveryData deliveryData) { + super(id, mediaFormat, videoQualityData, deliveryData); + this.averageBitrate = averageBitrate; + } + + public SimpleVideoAudioItagFormat(final int id, + @Nonnull final VideoAudioMediaFormat mediaFormat, + @Nonnull final VideoQualityData videoQualityData, + final int averageBitrate) { + super(id, mediaFormat, videoQualityData); + this.averageBitrate = averageBitrate; + } + + @Override + public int averageBitrate() { + return averageBitrate; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/simpleimpl/SimpleVideoItagFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/simpleimpl/SimpleVideoItagFormat.java new file mode 100644 index 0000000000..82f12075fa --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/simpleimpl/SimpleVideoItagFormat.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.format.simpleimpl; + +import org.schabi.newpipe.extractor.services.youtube.itag.delivery.ItagFormatDeliveryData; +import org.schabi.newpipe.extractor.services.youtube.itag.format.VideoItagFormat; +import org.schabi.newpipe.extractor.streamdata.format.VideoAudioMediaFormat; +import org.schabi.newpipe.extractor.streamdata.stream.quality.VideoQualityData; + +import java.util.Objects; + +import javax.annotation.Nonnull; + +public class SimpleVideoItagFormat extends AbstractSimpleItagFormat + implements VideoItagFormat { + @Nonnull + private final VideoQualityData videoQualityData; + + public SimpleVideoItagFormat(final int id, + @Nonnull final VideoAudioMediaFormat mediaFormat, + @Nonnull final VideoQualityData videoQualityData, + @Nonnull final ItagFormatDeliveryData deliveryData) { + super(id, mediaFormat, deliveryData); + this.videoQualityData = Objects.requireNonNull(videoQualityData); + } + + public SimpleVideoItagFormat(final int id, + @Nonnull final VideoAudioMediaFormat mediaFormat, + @Nonnull final VideoQualityData videoQualityData) { + super(id, mediaFormat); + this.videoQualityData = Objects.requireNonNull(videoQualityData); + } + + @Nonnull + @Override + public VideoQualityData videoQualityData() { + return videoQualityData; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/info/ItagInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/info/ItagInfo.java new file mode 100644 index 0000000000..13c704d4cb --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/info/ItagInfo.java @@ -0,0 +1,280 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.info; + +import org.schabi.newpipe.extractor.services.youtube.itag.format.BaseAudioItagFormat; +import org.schabi.newpipe.extractor.services.youtube.itag.format.ItagFormat; +import org.schabi.newpipe.extractor.services.youtube.itag.format.VideoItagFormat; +import org.schabi.newpipe.extractor.streamdata.stream.quality.VideoQualityData; + +import java.util.Optional; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public class ItagInfo> { + + @Nonnull + private final I itagFormat; + + @Nonnull + private final String streamUrl; + + // region Audio + + /** + * The average bitrate in KBit/s + */ + @Nullable + private Integer averageBitrate; + + @Nullable + private Integer audioSampleRate; + + @Nullable + private Integer audioChannels; + + // endregion + + // region Video + + @Nullable + private Integer width; + + @Nullable + private Integer height; + + @Nullable + private Integer fps; + + // endregion + + @Nullable + private Integer bitRate; + + @Nullable + private String quality; + + @Nullable + private String codec; + + @Nullable + private ItagInfoRange initRange; + + @Nullable + private ItagInfoRange indexRange; + + @Nullable + private Long contentLength; + + @Nullable + private Long approxDurationMs; + + @Nullable + private String type; + + // region live or post-live + + @Nullable + private Integer targetDurationSec; + + // endregion + + + public ItagInfo( + @Nonnull final I itagFormat, + @Nonnull final String streamUrl) { + this.itagFormat = itagFormat; + this.streamUrl = streamUrl; + } + + // region Getters + Setters + + @Nonnull + public I getItagFormat() { + return itagFormat; + } + + @Nonnull + public String getStreamUrl() { + return streamUrl; + } + + @Nullable + public Integer getAverageBitrate() { + return averageBitrate; + } + + public void setAverageBitrate(@Nullable final Integer averageBitrate) { + this.averageBitrate = averageBitrate; + } + + @Nullable + public Integer getAudioSampleRate() { + return audioSampleRate; + } + + public void setAudioSampleRate(@Nullable final Integer audioSampleRate) { + this.audioSampleRate = audioSampleRate; + } + + @Nullable + public Integer getAudioChannels() { + return audioChannels; + } + + public void setAudioChannels(@Nullable final Integer audioChannels) { + this.audioChannels = audioChannels; + } + + @Nullable + public Integer getWidth() { + return width; + } + + public void setWidth(@Nullable final Integer width) { + this.width = width; + } + + @Nullable + public Integer getHeight() { + return height; + } + + public void setHeight(@Nullable final Integer height) { + this.height = height; + } + + @Nullable + public Integer getFps() { + return fps; + } + + public void setFps(@Nullable final Integer fps) { + this.fps = fps; + } + + @Nullable + public Integer getBitRate() { + return bitRate; + } + + public void setBitRate(@Nullable final Integer bitRate) { + this.bitRate = bitRate; + } + + @Nullable + public String getQuality() { + return quality; + } + + public void setQuality(@Nullable final String quality) { + this.quality = quality; + } + + @Nullable + public String getCodec() { + return codec; + } + + public void setCodec(@Nullable final String codec) { + this.codec = codec; + } + + @Nullable + public ItagInfoRange getInitRange() { + return initRange; + } + + public void setInitRange(@Nullable final ItagInfoRange initRange) { + this.initRange = initRange; + } + + @Nullable + public ItagInfoRange getIndexRange() { + return indexRange; + } + + public void setIndexRange(@Nullable final ItagInfoRange indexRange) { + this.indexRange = indexRange; + } + + @Nullable + public Long getContentLength() { + return contentLength; + } + + public void setContentLength(@Nullable final Long contentLength) { + this.contentLength = contentLength; + } + + @Nullable + public Long getApproxDurationMs() { + return approxDurationMs; + } + + public void setApproxDurationMs(@Nullable final Long approxDurationMs) { + this.approxDurationMs = approxDurationMs; + } + + @Nullable + public String getType() { + return type; + } + + public void setType(@Nullable final String type) { + this.type = type; + } + + @Nullable + public Integer getTargetDurationSec() { + return targetDurationSec; + } + + public void setTargetDurationSec(@Nullable final Integer targetDurationSec) { + this.targetDurationSec = targetDurationSec; + } + + // endregion + + /** + * Returns the combined averageBitrate from the current information and {@link #itagFormat}. + * @return averageBitRate in KBit/s or -1 + */ + public int getCombinedAverageBitrate() { + if (averageBitrate != null) { + return averageBitrate; + } + + if (itagFormat instanceof BaseAudioItagFormat) { + return ((BaseAudioItagFormat) itagFormat).averageBitrate(); + } + + return -1; + } + + /** + * Returns the combined video-quality data from the current information and {@link #itagFormat}. + * @return video-quality data + */ + @Nonnull + public VideoQualityData getCombinedVideoQualityData() { + final Optional optVideoItagFormatQualityData = + itagFormat instanceof VideoItagFormat + ? Optional.of(((VideoItagFormat) itagFormat).videoQualityData()) + : Optional.empty(); + + return new VideoQualityData( + Optional.ofNullable(height) + .orElse(optVideoItagFormatQualityData + .map(VideoQualityData::height) + .orElse(VideoQualityData.UNKNOWN)), + Optional.ofNullable(width) + .orElse(optVideoItagFormatQualityData + .map(VideoQualityData::width) + .orElse(VideoQualityData.UNKNOWN)), + Optional.ofNullable(fps) + .orElse(optVideoItagFormatQualityData + .map(VideoQualityData::fps) + .orElse(VideoQualityData.UNKNOWN)) + ); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/info/ItagInfoRange.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/info/ItagInfoRange.java new file mode 100644 index 0000000000..6b5916985b --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/info/ItagInfoRange.java @@ -0,0 +1,27 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.info; + +public class ItagInfoRange { + private final int start; + private final int end; + + public ItagInfoRange(final int start, final int end) { + this.start = start; + this.end = end; + } + + public int start() { + return start; + } + + public int end() { + return end; + } + + @Override + public String toString() { + return "ItagInfoRange{" + + "start=" + start + + ", end=" + end + + '}'; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/info/builder/ItagInfoRangeHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/info/builder/ItagInfoRangeHelper.java new file mode 100644 index 0000000000..aad2a8325f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/info/builder/ItagInfoRangeHelper.java @@ -0,0 +1,24 @@ +package org.schabi.newpipe.extractor.services.youtube.itag.info.builder; + +import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.services.youtube.itag.info.ItagInfoRange; + +import javax.annotation.Nonnull; + +public final class ItagInfoRangeHelper { + private ItagInfoRangeHelper() { + // No impl + } + + public static ItagInfoRange buildFrom(@Nonnull final JsonObject jsonRangeObj) { + try { + return new ItagInfoRange( + Integer.parseInt(jsonRangeObj.getString("start", "-1")), + Integer.parseInt(jsonRangeObj.getString("end", "-1")) + ); + } catch (final NumberFormatException ignored) { + return null; + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java deleted file mode 100644 index 59cf9a3231..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java +++ /dev/null @@ -1,383 +0,0 @@ -package org.schabi.newpipe.extractor.stream; - -/* - * Created by Christian Schabesberger on 04.03.16. - * - * Copyright (C) Christian Schabesberger 2016 - * AudioStream.java is part of NewPipe Extractor. - * - * NewPipe Extractor is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe Extractor is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe Extractor. If not, see . - */ - -import org.schabi.newpipe.extractor.MediaFormat; -import org.schabi.newpipe.extractor.services.youtube.ItagItem; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -public final class AudioStream extends Stream { - public static final int UNKNOWN_BITRATE = -1; - - private final int averageBitrate; - - // Fields for DASH - private int itag = ITAG_NOT_AVAILABLE_OR_NOT_APPLICABLE; - private int bitrate; - private int initStart; - private int initEnd; - private int indexStart; - private int indexEnd; - private String quality; - private String codec; - @Nullable - private ItagItem itagItem; - - /** - * Class to build {@link AudioStream} objects. - */ - @SuppressWarnings("checkstyle:hiddenField") - public static final class Builder { - private String id; - private String content; - private boolean isUrl; - private DeliveryMethod deliveryMethod = DeliveryMethod.PROGRESSIVE_HTTP; - @Nullable - private MediaFormat mediaFormat; - @Nullable - private String manifestUrl; - private int averageBitrate = UNKNOWN_BITRATE; - @Nullable - private ItagItem itagItem; - - /** - * Create a new {@link Builder} instance with its default values. - */ - public Builder() { - } - - /** - * Set the identifier of the {@link AudioStream}. - * - *

- * It must not be null and should be non empty. - *

- * - *

- * If you are not able to get an identifier, use the static constant {@link - * Stream#ID_UNKNOWN ID_UNKNOWN} of the {@link Stream} class. - *

- * - * @param id the identifier of the {@link AudioStream}, which must not be null - * @return this {@link Builder} instance - */ - public Builder setId(@Nonnull final String id) { - this.id = id; - return this; - } - - /** - * Set the content of the {@link AudioStream}. - * - *

- * It must not be null, and should be non empty. - *

- * - * @param content the content of the {@link AudioStream} - * @param isUrl whether the content is a URL - * @return this {@link Builder} instance - */ - public Builder setContent(@Nonnull final String content, - final boolean isUrl) { - this.content = content; - this.isUrl = isUrl; - return this; - } - - /** - * Set the {@link MediaFormat} used by the {@link AudioStream}. - * - *

- * It should be one of the audio {@link MediaFormat}s ({@link MediaFormat#M4A M4A}, - * {@link MediaFormat#WEBMA WEBMA}, {@link MediaFormat#MP3 MP3}, {@link MediaFormat#OPUS - * OPUS}, {@link MediaFormat#OGG OGG}, or {@link MediaFormat#WEBMA_OPUS WEBMA_OPUS}) but - * can be {@code null} if the media format could not be determined. - *

- * - *

- * The default value is {@code null}. - *

- * - * @param mediaFormat the {@link MediaFormat} of the {@link AudioStream}, which can be null - * @return this {@link Builder} instance - */ - public Builder setMediaFormat(@Nullable final MediaFormat mediaFormat) { - this.mediaFormat = mediaFormat; - return this; - } - - /** - * Set the {@link DeliveryMethod} of the {@link AudioStream}. - * - *

- * It must not be null. - *

- * - *

- * The default delivery method is {@link DeliveryMethod#PROGRESSIVE_HTTP}. - *

- * - * @param deliveryMethod the {@link DeliveryMethod} of the {@link AudioStream}, which must - * not be null - * @return this {@link Builder} instance - */ - public Builder setDeliveryMethod(@Nonnull final DeliveryMethod deliveryMethod) { - this.deliveryMethod = deliveryMethod; - return this; - } - - /** - * Sets the URL of the manifest this stream comes from (if applicable, otherwise null). - * - * @param manifestUrl the URL of the manifest this stream comes from or {@code null} - * @return this {@link Builder} instance - */ - public Builder setManifestUrl(@Nullable final String manifestUrl) { - this.manifestUrl = manifestUrl; - return this; - } - - /** - * Set the average bitrate of the {@link AudioStream}. - * - *

- * The default value is {@link #UNKNOWN_BITRATE}. - *

- * - * @param averageBitrate the average bitrate of the {@link AudioStream}, which should - * positive - * @return this {@link Builder} instance - */ - public Builder setAverageBitrate(final int averageBitrate) { - this.averageBitrate = averageBitrate; - return this; - } - - /** - * Set the {@link ItagItem} corresponding to the {@link AudioStream}. - * - *

- * {@link ItagItem}s are YouTube specific objects, so they are only known for this service - * and can be null. - *

- * - *

- * The default value is {@code null}. - *

- * - * @param itagItem the {@link ItagItem} of the {@link AudioStream}, which can be null - * @return this {@link Builder} instance - */ - public Builder setItagItem(@Nullable final ItagItem itagItem) { - this.itagItem = itagItem; - return this; - } - - /** - * Build an {@link AudioStream} using the builder's current values. - * - *

- * The identifier and the content (and so the {@code isUrl} boolean) properties must have - * been set. - *

- * - * @return a new {@link AudioStream} using the builder's current values - * @throws IllegalStateException if {@code id}, {@code content} (and so {@code isUrl}) or - * {@code deliveryMethod} have been not set, or have been set as {@code null} - */ - @Nonnull - public AudioStream build() { - if (id == null) { - throw new IllegalStateException( - "The identifier of the audio stream has been not set or is null. If you " - + "are not able to get an identifier, use the static constant " - + "ID_UNKNOWN of the Stream class."); - } - - if (content == null) { - throw new IllegalStateException("The content of the audio stream has been not set " - + "or is null. Please specify a non-null one with setContent."); - } - - if (deliveryMethod == null) { - throw new IllegalStateException( - "The delivery method of the audio stream has been set as null, which is " - + "not allowed. Pass a valid one instead with setDeliveryMethod."); - } - - return new AudioStream(id, content, isUrl, mediaFormat, deliveryMethod, averageBitrate, - manifestUrl, itagItem); - } - } - - - /** - * Create a new audio stream. - * - * @param id the identifier which uniquely identifies the stream, e.g. for YouTube - * this would be the itag - * @param content the content or the URL of the stream, depending on whether isUrl is - * true - * @param isUrl whether content is the URL or the actual content of e.g. a DASH - * manifest - * @param format the {@link MediaFormat} used by the stream, which can be null - * @param deliveryMethod the {@link DeliveryMethod} of the stream - * @param averageBitrate the average bitrate of the stream (which can be unknown, see - * {@link #UNKNOWN_BITRATE}) - * @param itagItem the {@link ItagItem} corresponding to the stream, which cannot be null - * @param manifestUrl the URL of the manifest this stream comes from (if applicable, - * otherwise null) - */ - @SuppressWarnings("checkstyle:ParameterNumber") - private AudioStream(@Nonnull final String id, - @Nonnull final String content, - final boolean isUrl, - @Nullable final MediaFormat format, - @Nonnull final DeliveryMethod deliveryMethod, - final int averageBitrate, - @Nullable final String manifestUrl, - @Nullable final ItagItem itagItem) { - super(id, content, isUrl, format, deliveryMethod, manifestUrl); - if (itagItem != null) { - this.itagItem = itagItem; - this.itag = itagItem.id; - this.quality = itagItem.getQuality(); - this.bitrate = itagItem.getBitrate(); - this.initStart = itagItem.getInitStart(); - this.initEnd = itagItem.getInitEnd(); - this.indexStart = itagItem.getIndexStart(); - this.indexEnd = itagItem.getIndexEnd(); - this.codec = itagItem.getCodec(); - } - this.averageBitrate = averageBitrate; - } - - /** - * {@inheritDoc} - */ - @Override - public boolean equalStats(final Stream cmp) { - return super.equalStats(cmp) && cmp instanceof AudioStream - && averageBitrate == ((AudioStream) cmp).averageBitrate; - } - - /** - * Get the average bitrate of the stream. - * - * @return the average bitrate or {@link #UNKNOWN_BITRATE} if it is unknown - */ - public int getAverageBitrate() { - return averageBitrate; - } - - /** - * Get the itag identifier of the stream. - * - *

- * Always equals to {@link #ITAG_NOT_AVAILABLE_OR_NOT_APPLICABLE} for other streams than the - * ones of the YouTube service. - *

- * - * @return the number of the {@link ItagItem} passed in the constructor of the audio stream. - */ - public int getItag() { - return itag; - } - - /** - * Get the bitrate of the stream. - * - * @return the bitrate set from the {@link ItagItem} passed in the constructor of the stream. - */ - public int getBitrate() { - return bitrate; - } - - /** - * Get the initialization start of the stream. - * - * @return the initialization start value set from the {@link ItagItem} passed in the - * constructor of the stream. - */ - public int getInitStart() { - return initStart; - } - - /** - * Get the initialization end of the stream. - * - * @return the initialization end value set from the {@link ItagItem} passed in the constructor - * of the stream. - */ - public int getInitEnd() { - return initEnd; - } - - /** - * Get the index start of the stream. - * - * @return the index start value set from the {@link ItagItem} passed in the constructor of the - * stream. - */ - public int getIndexStart() { - return indexStart; - } - - /** - * Get the index end of the stream. - * - * @return the index end value set from the {@link ItagItem} passed in the constructor of the - * stream. - */ - public int getIndexEnd() { - return indexEnd; - } - - /** - * Get the quality of the stream. - * - * @return the quality label set from the {@link ItagItem} passed in the constructor of the - * stream. - */ - public String getQuality() { - return quality; - } - - /** - * Get the codec of the stream. - * - * @return the codec set from the {@link ItagItem} passed in the constructor of the stream. - */ - public String getCodec() { - return codec; - } - - /** - * {@inheritDoc} - */ - @Override - @Nullable - public ItagItem getItagItem() { - return itagItem; - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/DeliveryMethod.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/DeliveryMethod.java deleted file mode 100644 index ed98935725..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/DeliveryMethod.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.schabi.newpipe.extractor.stream; - -/** - * An enum to represent the different delivery methods of {@link Stream streams} which are returned - * by the extractor. - */ -public enum DeliveryMethod { - - /** - * Used for {@link Stream}s served using the progressive HTTP streaming method. - */ - PROGRESSIVE_HTTP, - - /** - * Used for {@link Stream}s served using the DASH (Dynamic Adaptive Streaming over HTTP) - * adaptive streaming method. - * - * @see the - * Dynamic Adaptive Streaming over HTTP Wikipedia page and - * DASH Industry Forum's website for more information about the DASH delivery method - */ - DASH, - - /** - * Used for {@link Stream}s served using the HLS (HTTP Live Streaming) adaptive streaming - * method. - * - * @see the HTTP Live Streaming - * page and Apple's developers website page - * about HTTP Live Streaming for more information about the HLS delivery method - */ - HLS, - - /** - * Used for {@link Stream}s served using the SmoothStreaming adaptive streaming method. - * - * @see Wikipedia's page about adaptive bitrate streaming, - * section Microsoft Smooth Streaming (MSS) for more information about the - * SmoothStreaming delivery method - */ - SS, - - /** - * Used for {@link Stream}s served via a torrent file. - * - * @see Wikipedia's BitTorrent's page, - * Wikipedia's page about torrent files - * and Bitorrent's website for more information - * about the BitTorrent protocol - */ - TORRENT -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Privacy.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Privacy.java new file mode 100644 index 0000000000..e02dd327b7 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Privacy.java @@ -0,0 +1,9 @@ +package org.schabi.newpipe.extractor.stream; + +public enum Privacy { + PUBLIC, + UNLISTED, + PRIVATE, + INTERNAL, + OTHER +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java deleted file mode 100644 index 04d2b3facb..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java +++ /dev/null @@ -1,207 +0,0 @@ -package org.schabi.newpipe.extractor.stream; - -import org.schabi.newpipe.extractor.MediaFormat; -import org.schabi.newpipe.extractor.services.youtube.ItagItem; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.io.Serializable; -import java.util.List; - -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - -/** - * Abstract class which represents streams in the extractor. - */ -public abstract class Stream implements Serializable { - public static final int FORMAT_ID_UNKNOWN = -1; - public static final String ID_UNKNOWN = " "; - - /** - * An integer to represent that the itag ID returned is not available (only for YouTube; this - * should never happen) or not applicable (for other services than YouTube). - * - *

- * An itag should not have a negative value, so {@code -1} is used for this constant. - *

- */ - public static final int ITAG_NOT_AVAILABLE_OR_NOT_APPLICABLE = -1; - - private final String id; - @Nullable private final MediaFormat mediaFormat; - private final String content; - private final boolean isUrl; - private final DeliveryMethod deliveryMethod; - @Nullable private final String manifestUrl; - - /** - * Instantiates a new {@code Stream} object. - * - * @param id the identifier which uniquely identifies the file, e.g. for YouTube - * this would be the itag - * @param content the content or URL, depending on whether isUrl is true - * @param isUrl whether content is the URL or the actual content of e.g. a DASH - * manifest - * @param format the {@link MediaFormat}, which can be null - * @param deliveryMethod the delivery method of the stream - * @param manifestUrl the URL of the manifest this stream comes from (if applicable, - * otherwise null) - */ - public Stream(final String id, - final String content, - final boolean isUrl, - @Nullable final MediaFormat format, - final DeliveryMethod deliveryMethod, - @Nullable final String manifestUrl) { - this.id = id; - this.content = content; - this.isUrl = isUrl; - this.mediaFormat = format; - this.deliveryMethod = deliveryMethod; - this.manifestUrl = manifestUrl; - } - - /** - * Checks if the list already contains a stream with the same statistics. - * - * @param stream the stream to be compared against the streams in the stream list - * @param streamList the list of {@link Stream}s which will be compared - * @return whether the list already contains one stream with equals stats - */ - public static boolean containSimilarStream(final Stream stream, - final List streamList) { - if (isNullOrEmpty(streamList)) { - return false; - } - for (final Stream cmpStream : streamList) { - if (stream.equalStats(cmpStream)) { - return true; - } - } - return false; - } - - /** - * Reveals whether two streams have the same statistics ({@link MediaFormat media format} and - * {@link DeliveryMethod delivery method}). - * - *

- * If the {@link MediaFormat media format} of the stream is unknown, the streams are compared - * by using only the {@link DeliveryMethod delivery method} and their ID. - *

- * - *

- * Note: This method always returns false if the stream passed is null. - *

- * - * @param other the stream object to be compared to this stream object - * @return whether the stream have the same stats or not, based on the criteria above - */ - public boolean equalStats(@Nullable final Stream other) { - if (other == null || mediaFormat == null || other.mediaFormat == null) { - return false; - } - return mediaFormat.id == other.mediaFormat.id && deliveryMethod == other.deliveryMethod - && isUrl == other.isUrl; - } - - /** - * Gets the identifier of this stream, e.g. the itag for YouTube. - * - *

- * It should normally be unique, but {@link #ID_UNKNOWN} may be returned as the identifier if - * the one used by the stream extractor cannot be extracted, which could happen if the - * extractor uses a value from a streaming service. - *

- * - * @return the identifier (which may be {@link #ID_UNKNOWN}) - */ - public String getId() { - return id; - } - - /** - * Gets the URL of this stream if the content is a URL, or {@code null} otherwise. - * - * @return the URL if the content is a URL, {@code null} otherwise - * @deprecated Use {@link #getContent()} instead. - */ - @Deprecated - @Nullable - public String getUrl() { - return isUrl ? content : null; - } - - /** - * Gets the content or URL. - * - * @return the content or URL - */ - public String getContent() { - return content; - } - - /** - * Returns whether the content is a URL or not. - * - * @return {@code true} if the content of this stream is a URL, {@code false} if it's the - * actual content - */ - public boolean isUrl() { - return isUrl; - } - - /** - * Gets the {@link MediaFormat}, which can be null. - * - * @return the format - */ - @Nullable - public MediaFormat getFormat() { - return mediaFormat; - } - - /** - * Gets the format ID, which can be unknown. - * - * @return the format ID or {@link #FORMAT_ID_UNKNOWN} - */ - public int getFormatId() { - if (mediaFormat != null) { - return mediaFormat.id; - } - return FORMAT_ID_UNKNOWN; - } - - /** - * Gets the {@link DeliveryMethod}. - * - * @return the delivery method - */ - @Nonnull - public DeliveryMethod getDeliveryMethod() { - return deliveryMethod; - } - - /** - * Gets the URL of the manifest this stream comes from (if applicable, otherwise null). - * - * @return the URL of the manifest this stream comes from or {@code null} - */ - @Nullable - public String getManifestUrl() { - return manifestUrl; - } - - /** - * Gets the {@link ItagItem} of a stream. - * - *

- * If the stream is not from YouTube, this value will always be null. - *

- * - * @return the {@link ItagItem} of the stream or {@code null} - */ - @Nullable - public abstract ItagItem getItagItem(); -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index ab922b1c98..8d9d8b5ffe 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -20,11 +20,10 @@ * along with NewPipe. If not, see . */ +import org.schabi.newpipe.extractor.Extractor; import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.InfoItemExtractor; -import org.schabi.newpipe.extractor.Extractor; -import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -32,16 +31,21 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.streamdata.stream.AudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.SubtitleStream; +import org.schabi.newpipe.extractor.streamdata.stream.VideoAudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.VideoStream; import org.schabi.newpipe.extractor.utils.Parser; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Locale; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + /** * Scrapes information from a video/audio streaming service (eg, YouTube). */ @@ -50,7 +54,7 @@ public abstract class StreamExtractor extends Extractor { public static final int NO_AGE_LIMIT = 0; public static final long UNKNOWN_SUBSCRIBER_COUNT = -1; - public StreamExtractor(final StreamingService service, final LinkHandler linkHandler) { + protected StreamExtractor(final StreamingService service, final LinkHandler linkHandler) { super(service, linkHandler); } @@ -254,10 +258,49 @@ public String getSubChannelAvatarUrl() throws ParsingException { } /** - * Get the dash mpd url. If you don't know what a dash MPD is you can read about it + * Defines how the current stream info should best be resolved. + * + *

+ * Service mostly offer different methods for streaming data. + * However the order is not always clearly defined. + * E.g. resolving a livestream might be better using the HLS master playlist. + *

+ * + * @return A list with the StreamResolutionMode order by priority (0 = highest priority) + */ + @Nonnull + public List getResolverStrategyPriority() { + if (isLive()) { + return Arrays.asList( + StreamResolvingStrategy.HLS_MASTER_PLAYLIST_URL, + StreamResolvingStrategy.DASH_MPD_URL, + StreamResolvingStrategy.VIDEO_ONLY_AND_AUDIO_STREAMS, + StreamResolvingStrategy.VIDEO_AUDIO_STREAMS + ); + } + return Arrays.asList( + StreamResolvingStrategy.VIDEO_ONLY_AND_AUDIO_STREAMS, + StreamResolvingStrategy.VIDEO_AUDIO_STREAMS, + StreamResolvingStrategy.HLS_MASTER_PLAYLIST_URL, + StreamResolvingStrategy.DASH_MPD_URL + ); + } + + /** + * Get the dash mpd url. + * + *

+ * If you don't know how DASH works take a look at + * + * here. + *

+ * + *

+ * If you don't know what a dash MPD is you can read about it * here. + *

* - * @return the url as a string or an empty string or an empty string if not available + * @return the url as a string or an empty string if not available * @throws ParsingException if an error occurs while reading */ @Nonnull @@ -266,15 +309,18 @@ public String getDashMpdUrl() throws ParsingException { } /** - * I am not sure if this is in use, and how this is used. However the frontend is missing - * support for HLS streams. Prove me if I am wrong. Please open an - * issue, - * or fix this description if you know whats up with this. + * Get the HLS master playlist url. + * + *

+ * If you don't know how HLS works take a look at + * + * here. + *

* * @return The Url to the hls stream or an empty string if not available. */ @Nonnull - public String getHlsUrl() throws ParsingException { + public String getHlsMasterPlaylistUrl() throws ParsingException { return ""; } @@ -286,10 +332,12 @@ public String getHlsUrl() throws ParsingException { * * @return a list of audio only streams in the format of AudioStream */ - public abstract List getAudioStreams() throws IOException, ExtractionException; + public List getAudioStreams() throws IOException, ExtractionException { + return Collections.emptyList(); + } /** - * This should return a list of available {@link VideoStream}s. + * This should return a list of available {@link VideoAudioStream}s. * Be aware this is the list of video streams which do contain an audio stream. * You can also return null or an empty list, however be aware that if you don't return anything * in getAudioStreams(), getVideoOnlyStreams() and getDashMpdUrl() either the Collector will @@ -297,7 +345,9 @@ public String getHlsUrl() throws ParsingException { * * @return a list of combined video and streams in the format of AudioStream */ - public abstract List getVideoStreams() throws IOException, ExtractionException; + public List getVideoStreams() throws IOException, ExtractionException { + return Collections.emptyList(); + } /** * This should return a list of available {@link VideoStream}s. @@ -308,38 +358,38 @@ public String getHlsUrl() throws ParsingException { * * @return a list of video and streams in the format of AudioStream */ - public abstract List getVideoOnlyStreams() throws IOException, ExtractionException; + public List getVideoOnlyStreams() throws IOException, ExtractionException { + return Collections.emptyList(); + } /** - * This will return a list of available {@link SubtitlesStream}s. + * This will return a list of available {@link SubtitleStream}s. * If no subtitles are available an empty list can be returned. * * @return a list of available subtitles or an empty list */ @Nonnull - public List getSubtitlesDefault() throws IOException, ExtractionException { + public List getSubtitles() throws IOException, ExtractionException { return Collections.emptyList(); } /** - * This will return a list of available {@link SubtitlesStream}s given by a specific type. - * If no subtitles in that specific format are available an empty list can be returned. + * This will return whenever there are only audio streams available. * - * @param format the media format by which the subtitles should be filtered - * @return a list of available subtitles or an empty list + * @return true when audio only otherwise false */ - @Nonnull - public List getSubtitles(final MediaFormat format) - throws IOException, ExtractionException { - return Collections.emptyList(); + public boolean isAudioOnly() { + return false; } /** - * Get the {@link StreamType}. + * This will return whenever the current stream is live. * - * @return the type of the stream + * @return true when live otherwise false */ - public abstract StreamType getStreamType() throws ParsingException; + public boolean isLive() { + return false; + } /** * Should return a list of streams related to the current handled. Many services show suggested @@ -351,8 +401,8 @@ public List getSubtitles(final MediaFormat format) * @return a list of InfoItems showing the related videos/streams */ @Nullable - public InfoItemsCollector - getRelatedItems() throws IOException, ExtractionException { + public InfoItemsCollector getRelatedItems() + throws IOException, ExtractionException { return null; } @@ -367,9 +417,8 @@ public StreamInfoItemsCollector getRelatedStreams() throws IOException, Extracti final InfoItemsCollector collector = getRelatedItems(); if (collector instanceof StreamInfoItemsCollector) { return (StreamInfoItemsCollector) collector; - } else { - return null; } + return null; } /** @@ -414,33 +463,33 @@ protected long getTimestampSeconds(final String regexPattern) throws ParsingExce return -2; } - if (!timestamp.isEmpty()) { + if (timestamp.isEmpty()) { + return 0; + } + + try { + String secondsString = ""; + String minutesString = ""; + String hoursString = ""; try { - String secondsString = ""; - String minutesString = ""; - String hoursString = ""; - try { - secondsString = Parser.matchGroup1("(\\d+)s", timestamp); - minutesString = Parser.matchGroup1("(\\d+)m", timestamp); - hoursString = Parser.matchGroup1("(\\d+)h", timestamp); - } catch (final Exception e) { - // it could be that time is given in another method - if (secondsString.isEmpty() && minutesString.isEmpty()) { - // if nothing was obtained, treat as unlabelled seconds - secondsString = Parser.matchGroup1("t=(\\d+)", timestamp); - } + secondsString = Parser.matchGroup1("(\\d+)s", timestamp); + minutesString = Parser.matchGroup1("(\\d+)m", timestamp); + hoursString = Parser.matchGroup1("(\\d+)h", timestamp); + } catch (final Exception e) { + // it could be that time is given in another method + if (secondsString.isEmpty() && minutesString.isEmpty()) { + // if nothing was obtained, treat as unlabelled seconds + secondsString = Parser.matchGroup1("t=(\\d+)", timestamp); } + } - final int seconds = secondsString.isEmpty() ? 0 : Integer.parseInt(secondsString); - final int minutes = minutesString.isEmpty() ? 0 : Integer.parseInt(minutesString); - final int hours = hoursString.isEmpty() ? 0 : Integer.parseInt(hoursString); + final int seconds = secondsString.isEmpty() ? 0 : Integer.parseInt(secondsString); + final int minutes = minutesString.isEmpty() ? 0 : Integer.parseInt(minutesString); + final int hours = hoursString.isEmpty() ? 0 : Integer.parseInt(hoursString); - return seconds + (60L * minutes) + (3600L * hours); - } catch (final ParsingException e) { - throw new ParsingException("Could not get timestamp.", e); - } - } else { - return 0; + return seconds + (60L * minutes) + (3600L * hours); + } catch (final ParsingException e) { + throw new ParsingException("Could not get timestamp.", e); } } @@ -553,12 +602,4 @@ public List getStreamSegments() throws ParsingException { public List getMetaInfo() throws ParsingException { return Collections.emptyList(); } - - public enum Privacy { - PUBLIC, - UNLISTED, - PRIVATE, - INTERNAL, - OTHER - } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 8d3f2c5224..ebdb2538a1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -1,5 +1,7 @@ package org.schabi.newpipe.extractor.stream; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import org.schabi.newpipe.extractor.Info; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.MetaInfo; @@ -9,6 +11,10 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.streamdata.stream.AudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.SubtitleStream; +import org.schabi.newpipe.extractor.streamdata.stream.VideoAudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.VideoStream; import org.schabi.newpipe.extractor.utils.ExtractorHelper; import java.io.IOException; @@ -16,11 +22,10 @@ import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.Objects; import javax.annotation.Nonnull; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - /* * Created by Christian Schabesberger on 26.08.15. * @@ -46,467 +51,174 @@ */ public class StreamInfo extends Info { - public static class StreamExtractException extends ExtractionException { - StreamExtractException(final String message) { - super(message); - } - } + private String thumbnailUrl = ""; + private String textualUploadDate; + private DateWrapper uploadDate; + private long duration = -1; + private int ageLimit; + private Description description; + + private long viewCount = -1; + private long likeCount = -1; + private long dislikeCount = -1; + + private String uploaderName = ""; + private String uploaderUrl = ""; + private String uploaderAvatarUrl = ""; + private boolean uploaderVerified = false; + private long uploaderSubscriberCount = -1; + + private String subChannelName = ""; + private String subChannelUrl = ""; + private String subChannelAvatarUrl = ""; + + private List streamResolvingStrategies = new ArrayList<>(); + private List videoStreams = new ArrayList<>(); + private List audioStreams = new ArrayList<>(); + private List videoOnlyStreams = new ArrayList<>(); + @Nonnull + private String dashMpdUrl = ""; + @Nonnull + private String hlsMasterPlaylistUrl = ""; + + private final boolean audioOnly; + private final boolean live; + + private List relatedItems = new ArrayList<>(); + + private long startPosition = 0; + private List subtitles = new ArrayList<>(); + + private String host = ""; + private Privacy privacy; + private String category = ""; + private String licence = ""; + private String supportInfo = ""; + private Locale language = null; + private List tags = new ArrayList<>(); + private List streamSegments = new ArrayList<>(); + private List metaInfo = new ArrayList<>(); + + @SuppressWarnings("checkstyle:ParameterNumber") public StreamInfo(final int serviceId, final String url, final String originalUrl, - final StreamType streamType, final String id, final String name, - final int ageLimit) { + final int ageLimit, + final boolean audioOnly, + final boolean live) { super(serviceId, id, url, originalUrl, name); - this.streamType = streamType; this.ageLimit = ageLimit; + this.audioOnly = audioOnly; + this.live = live; } - public static StreamInfo getInfo(final String url) throws IOException, ExtractionException { - return getInfo(NewPipe.getServiceByUrl(url), url); + /** + * Preview frames, e.g. for the storyboard / seekbar thumbnail preview + */ + private List previewFrames = Collections.emptyList(); + + /** + * Get the thumbnail url + * + * @return the thumbnail url as a string + */ + public String getThumbnailUrl() { + return thumbnailUrl; } - public static StreamInfo getInfo(@Nonnull final StreamingService service, - final String url) throws IOException, ExtractionException { - return getInfo(service.getStreamExtractor(url)); + public void setThumbnailUrl(final String thumbnailUrl) { + this.thumbnailUrl = thumbnailUrl; } - public static StreamInfo getInfo(@Nonnull final StreamExtractor extractor) - throws ExtractionException, IOException { - extractor.fetchPage(); - final StreamInfo streamInfo; - try { - streamInfo = extractImportantData(extractor); - extractStreams(streamInfo, extractor); - extractOptionalData(streamInfo, extractor); - return streamInfo; + public String getTextualUploadDate() { + return textualUploadDate; + } - } catch (final ExtractionException e) { - // Currently, YouTube does not distinguish between age restricted videos and videos - // blocked by country. This means that during the initialisation of the extractor, the - // extractor will assume that a video is age restricted while in reality it is blocked - // by country. - // - // We will now detect whether the video is blocked by country or not. + public void setTextualUploadDate(final String textualUploadDate) { + this.textualUploadDate = textualUploadDate; + } - final String errorMessage = extractor.getErrorMessage(); - if (isNullOrEmpty(errorMessage)) { - throw e; - } else { - throw new ContentNotAvailableException(errorMessage, e); - } - } + public DateWrapper getUploadDate() { + return uploadDate; } - @Nonnull - private static StreamInfo extractImportantData(@Nonnull final StreamExtractor extractor) - throws ExtractionException { - // Important data, without it the content can't be displayed. - // If one of these is not available, the frontend will receive an exception directly. + public void setUploadDate(final DateWrapper uploadDate) { + this.uploadDate = uploadDate; + } - final int serviceId = extractor.getServiceId(); - final String url = extractor.getUrl(); - final String originalUrl = extractor.getOriginalUrl(); - final StreamType streamType = extractor.getStreamType(); - final String id = extractor.getId(); - final String name = extractor.getName(); - final int ageLimit = extractor.getAgeLimit(); + /** + * Get the duration in seconds + * + * @return the duration in seconds + */ + public long getDuration() { + return duration; + } - // Suppress always-non-null warning as here we double-check it really is not null - //noinspection ConstantConditions - if (streamType == StreamType.NONE - || isNullOrEmpty(url) - || isNullOrEmpty(id) - || name == null /* but it can be empty of course */ - || ageLimit == -1) { - throw new ExtractionException("Some important stream information was not given."); - } + public void setDuration(final long duration) { + this.duration = duration; + } - return new StreamInfo(extractor.getServiceId(), url, extractor.getOriginalUrl(), - streamType, id, name, ageLimit); + public int getAgeLimit() { + return ageLimit; } + public void setAgeLimit(final int ageLimit) { + this.ageLimit = ageLimit; + } - private static void extractStreams(final StreamInfo streamInfo, - final StreamExtractor extractor) - throws ExtractionException { - /* ---- Stream extraction goes here ---- */ - // At least one type of stream has to be available, otherwise an exception will be thrown - // directly into the frontend. + public Description getDescription() { + return description; + } - try { - streamInfo.setDashMpdUrl(extractor.getDashMpdUrl()); - } catch (final Exception e) { - streamInfo.addError(new ExtractionException("Couldn't get DASH manifest", e)); - } + public void setDescription(final Description description) { + this.description = description; + } - try { - streamInfo.setHlsUrl(extractor.getHlsUrl()); - } catch (final Exception e) { - streamInfo.addError(new ExtractionException("Couldn't get HLS manifest", e)); - } + public long getViewCount() { + return viewCount; + } - /* Load and extract audio */ - try { - streamInfo.setAudioStreams(extractor.getAudioStreams()); - } catch (final ContentNotSupportedException e) { - throw e; - } catch (final Exception e) { - streamInfo.addError(new ExtractionException("Couldn't get audio streams", e)); - } + public void setViewCount(final long viewCount) { + this.viewCount = viewCount; + } - /* Extract video stream url */ - try { - streamInfo.setVideoStreams(extractor.getVideoStreams()); - } catch (final Exception e) { - streamInfo.addError(new ExtractionException("Couldn't get video streams", e)); - } + /** + * Get the number of likes. + * + * @return The number of likes or -1 if this information is not available + */ + public long getLikeCount() { + return likeCount; + } - /* Extract video only stream url */ - try { - streamInfo.setVideoOnlyStreams(extractor.getVideoOnlyStreams()); - } catch (final Exception e) { - streamInfo.addError(new ExtractionException("Couldn't get video only streams", e)); - } + public void setLikeCount(final long likeCount) { + this.likeCount = likeCount; + } - // Lists can be null if an exception was thrown during extraction - if (streamInfo.getVideoStreams() == null) { - streamInfo.setVideoStreams(Collections.emptyList()); - } - if (streamInfo.getVideoOnlyStreams() == null) { - streamInfo.setVideoOnlyStreams(Collections.emptyList()); - } - if (streamInfo.getAudioStreams() == null) { - streamInfo.setAudioStreams(Collections.emptyList()); - } + /** + * Get the number of dislikes. + * + * @return The number of likes or -1 if this information is not available + */ + public long getDislikeCount() { + return dislikeCount; + } - // Either audio or video has to be available, otherwise we didn't get a stream (since - // videoOnly are optional, they don't count). - if ((streamInfo.videoStreams.isEmpty()) && (streamInfo.audioStreams.isEmpty())) { - throw new StreamExtractException( - "Could not get any stream. See error variable to get further details."); - } + public void setDislikeCount(final long dislikeCount) { + this.dislikeCount = dislikeCount; } - @SuppressWarnings("MethodLength") - private static void extractOptionalData(final StreamInfo streamInfo, - final StreamExtractor extractor) { - /* ---- Optional data goes here: ---- */ - // If one of these fails, the frontend needs to handle that they are not available. - // Exceptions are therefore not thrown into the frontend, but stored into the error list, - // so the frontend can afterwards check where errors happened. + public String getUploaderName() { + return uploaderName; + } - try { - streamInfo.setThumbnailUrl(extractor.getThumbnailUrl()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setDuration(extractor.getLength()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setUploaderName(extractor.getUploaderName()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setUploaderUrl(extractor.getUploaderUrl()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setUploaderAvatarUrl(extractor.getUploaderAvatarUrl()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setUploaderVerified(extractor.isUploaderVerified()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setUploaderSubscriberCount(extractor.getUploaderSubscriberCount()); - } catch (final Exception e) { - streamInfo.addError(e); - } - - try { - streamInfo.setSubChannelName(extractor.getSubChannelName()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setSubChannelUrl(extractor.getSubChannelUrl()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setSubChannelAvatarUrl(extractor.getSubChannelAvatarUrl()); - } catch (final Exception e) { - streamInfo.addError(e); - } - - try { - streamInfo.setDescription(extractor.getDescription()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setViewCount(extractor.getViewCount()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setTextualUploadDate(extractor.getTextualUploadDate()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setUploadDate(extractor.getUploadDate()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setStartPosition(extractor.getTimeStamp()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setLikeCount(extractor.getLikeCount()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setDislikeCount(extractor.getDislikeCount()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setSubtitles(extractor.getSubtitlesDefault()); - } catch (final Exception e) { - streamInfo.addError(e); - } - - // Additional info - try { - streamInfo.setHost(extractor.getHost()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setPrivacy(extractor.getPrivacy()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setCategory(extractor.getCategory()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setLicence(extractor.getLicence()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setLanguageInfo(extractor.getLanguageInfo()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setTags(extractor.getTags()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setSupportInfo(extractor.getSupportInfo()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setStreamSegments(extractor.getStreamSegments()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setMetaInfo(extractor.getMetaInfo()); - } catch (final Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setPreviewFrames(extractor.getFrames()); - } catch (final Exception e) { - streamInfo.addError(e); - } - - streamInfo.setRelatedItems(ExtractorHelper.getRelatedItemsOrLogError(streamInfo, - extractor)); - } - - private StreamType streamType; - private String thumbnailUrl = ""; - private String textualUploadDate; - private DateWrapper uploadDate; - private long duration = -1; - private int ageLimit; - private Description description; - - private long viewCount = -1; - private long likeCount = -1; - private long dislikeCount = -1; - - private String uploaderName = ""; - private String uploaderUrl = ""; - private String uploaderAvatarUrl = ""; - private boolean uploaderVerified = false; - private long uploaderSubscriberCount = -1; - - private String subChannelName = ""; - private String subChannelUrl = ""; - private String subChannelAvatarUrl = ""; - - private List videoStreams = new ArrayList<>(); - private List audioStreams = new ArrayList<>(); - private List videoOnlyStreams = new ArrayList<>(); - - private String dashMpdUrl = ""; - private String hlsUrl = ""; - private List relatedItems = new ArrayList<>(); - - private long startPosition = 0; - private List subtitles = new ArrayList<>(); - - private String host = ""; - private StreamExtractor.Privacy privacy; - private String category = ""; - private String licence = ""; - private String supportInfo = ""; - private Locale language = null; - private List tags = new ArrayList<>(); - private List streamSegments = new ArrayList<>(); - private List metaInfo = new ArrayList<>(); - - /** - * Preview frames, e.g. for the storyboard / seekbar thumbnail preview - */ - private List previewFrames = Collections.emptyList(); - - /** - * Get the stream type - * - * @return the stream type - */ - public StreamType getStreamType() { - return streamType; - } - - public void setStreamType(final StreamType streamType) { - this.streamType = streamType; - } - - /** - * Get the thumbnail url - * - * @return the thumbnail url as a string - */ - public String getThumbnailUrl() { - return thumbnailUrl; - } - - public void setThumbnailUrl(final String thumbnailUrl) { - this.thumbnailUrl = thumbnailUrl; - } - - public String getTextualUploadDate() { - return textualUploadDate; - } - - public void setTextualUploadDate(final String textualUploadDate) { - this.textualUploadDate = textualUploadDate; - } - - public DateWrapper getUploadDate() { - return uploadDate; - } - - public void setUploadDate(final DateWrapper uploadDate) { - this.uploadDate = uploadDate; - } - - /** - * Get the duration in seconds - * - * @return the duration in seconds - */ - public long getDuration() { - return duration; - } - - public void setDuration(final long duration) { - this.duration = duration; - } - - public int getAgeLimit() { - return ageLimit; - } - - public void setAgeLimit(final int ageLimit) { - this.ageLimit = ageLimit; - } - - public Description getDescription() { - return description; - } - - public void setDescription(final Description description) { - this.description = description; - } - - public long getViewCount() { - return viewCount; - } - - public void setViewCount(final long viewCount) { - this.viewCount = viewCount; - } - - /** - * Get the number of likes. - * - * @return The number of likes or -1 if this information is not available - */ - public long getLikeCount() { - return likeCount; - } - - public void setLikeCount(final long likeCount) { - this.likeCount = likeCount; - } - - /** - * Get the number of dislikes. - * - * @return The number of likes or -1 if this information is not available - */ - public long getDislikeCount() { - return dislikeCount; - } - - public void setDislikeCount(final long dislikeCount) { - this.dislikeCount = dislikeCount; - } - - public String getUploaderName() { - return uploaderName; - } - - public void setUploaderName(final String uploaderName) { - this.uploaderName = uploaderName; - } + public void setUploaderName(final String uploaderName) { + this.uploaderName = uploaderName; + } public String getUploaderUrl() { return uploaderUrl; @@ -564,68 +276,75 @@ public void setSubChannelAvatarUrl(final String subChannelAvatarUrl) { this.subChannelAvatarUrl = subChannelAvatarUrl; } - public List getVideoStreams() { - return videoStreams; + @Nonnull + public List getStreamResolvingStrategies() { + return streamResolvingStrategies; } - public void setVideoStreams(final List videoStreams) { - this.videoStreams = videoStreams; + public void setStreamResolvingStrategies( + @Nonnull final List streamResolvingStrategies) { + this.streamResolvingStrategies = streamResolvingStrategies; } + @Nonnull + public List getVideoStreams() { + return videoStreams; + } + + public void setVideoStreams(@Nonnull final List videoStreams) { + this.videoStreams = Objects.requireNonNull(videoStreams); + } + + @Nonnull public List getAudioStreams() { return audioStreams; } - public void setAudioStreams(final List audioStreams) { - this.audioStreams = audioStreams; + public void setAudioStreams(@Nonnull final List audioStreams) { + this.audioStreams = Objects.requireNonNull(audioStreams); } + @Nonnull public List getVideoOnlyStreams() { return videoOnlyStreams; } - public void setVideoOnlyStreams(final List videoOnlyStreams) { - this.videoOnlyStreams = videoOnlyStreams; + public void setVideoOnlyStreams(@Nonnull final List videoOnlyStreams) { + this.videoOnlyStreams = Objects.requireNonNull(videoOnlyStreams); } + @Nonnull public String getDashMpdUrl() { return dashMpdUrl; } - public void setDashMpdUrl(final String dashMpdUrl) { - this.dashMpdUrl = dashMpdUrl; + public void setDashMpdUrl(@Nonnull final String dashMpdUrl) { + this.dashMpdUrl = Objects.requireNonNull(dashMpdUrl); } - public String getHlsUrl() { - return hlsUrl; + @Nonnull + public String getHlsMasterPlaylistUrl() { + return hlsMasterPlaylistUrl; } - public void setHlsUrl(final String hlsUrl) { - this.hlsUrl = hlsUrl; + public void setHlsMasterPlaylistUrl(@Nonnull final String hlsMasterPlaylistUrl) { + this.hlsMasterPlaylistUrl = Objects.requireNonNull(hlsMasterPlaylistUrl); } - public List getRelatedItems() { - return relatedItems; + public boolean isAudioOnly() { + return audioOnly; } - /** - * @deprecated Use {@link #getRelatedItems()} - */ - @Deprecated - public List getRelatedStreams() { - return getRelatedItems(); + public boolean isLive() { + return live; } - public void setRelatedItems(final List relatedItems) { - this.relatedItems = relatedItems; + public List getRelatedItems() { + return relatedItems; } - /** - * @deprecated Use {@link #setRelatedItems(List)} - */ - @Deprecated - public void setRelatedStreams(final List relatedItemsToSet) { - setRelatedItems(relatedItemsToSet); + public void setRelatedItems(final List relatedItems) { + this.relatedItems = relatedItems; } public long getStartPosition() { @@ -636,12 +355,13 @@ public void setStartPosition(final long startPosition) { this.startPosition = startPosition; } - public List getSubtitles() { + @Nonnull + public List getSubtitles() { return subtitles; } - public void setSubtitles(final List subtitles) { - this.subtitles = subtitles; + public void setSubtitles(@Nonnull final List subtitles) { + this.subtitles = Objects.requireNonNull(subtitles); } public String getHost() { @@ -652,11 +372,11 @@ public void setHost(final String host) { this.host = host; } - public StreamExtractor.Privacy getPrivacy() { + public Privacy getPrivacy() { return this.privacy; } - public void setPrivacy(final StreamExtractor.Privacy privacy) { + public void setPrivacy(final Privacy privacy) { this.privacy = privacy; } @@ -724,4 +444,299 @@ public void setPreviewFrames(final List previewFrames) { public List getMetaInfo() { return this.metaInfo; } + + public static StreamInfo getInfo(final String url) throws IOException, ExtractionException { + return getInfo(NewPipe.getServiceByUrl(url), url); + } + + public static StreamInfo getInfo(@Nonnull final StreamingService service, + final String url) throws IOException, ExtractionException { + return getInfo(service.getStreamExtractor(url)); + } + + public static StreamInfo getInfo(@Nonnull final StreamExtractor extractor) + throws ExtractionException, IOException { + extractor.fetchPage(); + + try { + final StreamInfo streamInfo = extractImportantData(extractor); + extractStreams(streamInfo, extractor); + extractOptionalData(streamInfo, extractor); + return streamInfo; + + } catch (final ExtractionException e) { + // Currently, YouTube does not distinguish between age restricted videos and videos + // blocked by country. This means that during the initialisation of the extractor, the + // extractor will assume that a video is age restricted while in reality it is blocked + // by country. + // + // We will now detect whether the video is blocked by country or not. + // TODO: An error message is not a valid indicator if the video a blocked in a country + final String errorMessage = extractor.getErrorMessage(); + if (isNullOrEmpty(errorMessage)) { + throw e; + } else { + throw new ContentNotAvailableException(errorMessage, e); + } + } + } + + @Nonnull + private static StreamInfo extractImportantData(@Nonnull final StreamExtractor extractor) + throws ExtractionException { + // Important data, without it the content can't be displayed. + // If one of these is not available, the frontend will receive an exception directly. + + extractor.getServiceId(); // Check if a exception is thrown + final String url = extractor.getUrl(); + extractor.getOriginalUrl(); // Check if a exception is thrown + final String id = extractor.getId(); + final String name = extractor.getName(); + final int ageLimit = extractor.getAgeLimit(); + + // Suppress always-non-null warning as here we double-check it really is not null + //noinspection ConstantConditions + if (isNullOrEmpty(url) + || isNullOrEmpty(id) + || name == null /* but it can be empty of course */ + || ageLimit == -1) { + throw new ExtractionException("Some important stream information was not given"); + } + + return new StreamInfo( + extractor.getServiceId(), + url, + extractor.getOriginalUrl(), + id, + name, + ageLimit, + extractor.isAudioOnly(), + extractor.isLive()); + } + + + private static void extractStreams(final StreamInfo streamInfo, + final StreamExtractor extractor) + throws ExtractionException { + streamInfo.setStreamResolvingStrategies(extractor.getResolverStrategyPriority()); + + /* ---- Stream extraction goes here ---- */ + // At least one type of stream has to be available, otherwise an exception will be thrown + // directly into the frontend. + + /* Load and extract audio */ + try { + streamInfo.setAudioStreams(extractor.getAudioStreams()); + } catch (final ContentNotSupportedException e) { + throw e; + } catch (final Exception e) { + streamInfo.addError(new ExtractionException("Couldn't get audio streams", e)); + } + + /* Extract video stream url */ + try { + streamInfo.setVideoStreams(extractor.getVideoStreams()); + } catch (final ContentNotSupportedException e) { + throw e; + } catch (final Exception e) { + streamInfo.addError(new ExtractionException("Couldn't get video streams", e)); + } + + /* Extract video only stream url */ + try { + streamInfo.setVideoOnlyStreams(extractor.getVideoOnlyStreams()); + } catch (final ContentNotSupportedException e) { + throw e; + } catch (final Exception e) { + streamInfo.addError(new ExtractionException("Couldn't get video only streams", e)); + } + + /* Extract DASH-MPD url */ + try { + streamInfo.setDashMpdUrl(extractor.getDashMpdUrl()); + } catch (final Exception e) { + streamInfo.addError(new ExtractionException("Couldn't get DASH-MPD url", e)); + } + + /* Extract hls master playlist url */ + try { + streamInfo.setHlsMasterPlaylistUrl(extractor.getHlsMasterPlaylistUrl()); + } catch (final Exception e) { + streamInfo.addError(new ExtractionException("Couldn't get HLS master playlist", e)); + } + + // Check if any data for streaming is available + if (streamInfo.getVideoStreams().isEmpty() + && streamInfo.getVideoOnlyStreams().isEmpty() + && streamInfo.getAudioStreams().isEmpty() + && streamInfo.getDashMpdUrl().trim().isEmpty() + && streamInfo.getHlsMasterPlaylistUrl().trim().isEmpty() + ) { + throw new StreamExtractException("Could not get any required streaming-data. " + + "See error variable to get further details."); + } + } + + @SuppressWarnings("MethodLength") + private static void extractOptionalData(final StreamInfo streamInfo, + final StreamExtractor extractor) { + /* ---- Optional data goes here: ---- */ + // If one of these fails, the frontend needs to handle that they are not available. + // Exceptions are therefore not thrown into the frontend, but stored into the error list, + // so the frontend can afterwards check where errors happened. + + try { + streamInfo.setThumbnailUrl(extractor.getThumbnailUrl()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setDuration(extractor.getLength()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setUploaderName(extractor.getUploaderName()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setUploaderUrl(extractor.getUploaderUrl()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setUploaderAvatarUrl(extractor.getUploaderAvatarUrl()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setUploaderVerified(extractor.isUploaderVerified()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setUploaderSubscriberCount(extractor.getUploaderSubscriberCount()); + } catch (final Exception e) { + streamInfo.addError(e); + } + + try { + streamInfo.setSubChannelName(extractor.getSubChannelName()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setSubChannelUrl(extractor.getSubChannelUrl()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setSubChannelAvatarUrl(extractor.getSubChannelAvatarUrl()); + } catch (final Exception e) { + streamInfo.addError(e); + } + + try { + streamInfo.setDescription(extractor.getDescription()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setViewCount(extractor.getViewCount()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setTextualUploadDate(extractor.getTextualUploadDate()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setUploadDate(extractor.getUploadDate()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setStartPosition(extractor.getTimeStamp()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setLikeCount(extractor.getLikeCount()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setDislikeCount(extractor.getDislikeCount()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setSubtitles(extractor.getSubtitles()); + } catch (final Exception e) { + streamInfo.addError(e); + } + + // Additional info + try { + streamInfo.setHost(extractor.getHost()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setPrivacy(extractor.getPrivacy()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setCategory(extractor.getCategory()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setLicence(extractor.getLicence()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setLanguageInfo(extractor.getLanguageInfo()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setTags(extractor.getTags()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setSupportInfo(extractor.getSupportInfo()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setStreamSegments(extractor.getStreamSegments()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setMetaInfo(extractor.getMetaInfo()); + } catch (final Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setPreviewFrames(extractor.getFrames()); + } catch (final Exception e) { + streamInfo.addError(e); + } + + streamInfo.setRelatedItems( + ExtractorHelper.getRelatedItemsOrLogError(streamInfo, extractor)); + } + + public static class StreamExtractException extends ExtractionException { + StreamExtractException(final String message) { + super(message); + } + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java index 72e1454cad..f3b45b6e63 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java @@ -29,7 +29,8 @@ * Info object for previews of unopened videos, eg search results, related videos */ public class StreamInfoItem extends InfoItem { - private final StreamType streamType; + private final boolean live; + private final boolean audioOnly; private String uploaderName; private String shortDescription; @@ -46,13 +47,19 @@ public class StreamInfoItem extends InfoItem { public StreamInfoItem(final int serviceId, final String url, final String name, - final StreamType streamType) { + final boolean live, + final boolean audioOnly) { super(InfoType.STREAM, serviceId, url, name); - this.streamType = streamType; + this.live = live; + this.audioOnly = audioOnly; } - public StreamType getStreamType() { - return streamType; + public boolean isAudioOnly() { + return audioOnly; + } + + public boolean isLive() { + return live; } public String getUploaderName() { @@ -133,7 +140,8 @@ public void setUploaderVerified(final boolean uploaderVerified) { @Override public String toString() { return "StreamInfoItem{" - + "streamType=" + streamType + + "audioOnly=" + audioOnly + + ", live='" + live + '\'' + ", uploaderName='" + uploaderName + '\'' + ", textualUploadDate='" + textualUploadDate + '\'' + ", viewCount=" + viewCount diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java index 131719d8c3..ea06e6345c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java @@ -30,12 +30,22 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { /** - * Get the stream type + * This will return whenever there are only audio streams available. * - * @return the stream type - * @throws ParsingException thrown if there is an error in the extraction + * @return true when audio only otherwise false */ - StreamType getStreamType() throws ParsingException; + default boolean isAudioOnly() { + return false; + } + + /** + * This will return whenever the current stream is live. + * + * @return true when live otherwise false + */ + default boolean isLive() { + return false; + } /** * Check if the stream is an ad. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java index 231a929e58..c294e18b77 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java @@ -45,7 +45,11 @@ public StreamInfoItem extract(final StreamInfoItemExtractor extractor) throws Pa } final StreamInfoItem resultItem = new StreamInfoItem( - getServiceId(), extractor.getUrl(), extractor.getName(), extractor.getStreamType()); + getServiceId(), + extractor.getUrl(), + extractor.getName(), + extractor.isLive(), + extractor.isAudioOnly()); // optional information try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamResolvingStrategy.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamResolvingStrategy.java new file mode 100644 index 0000000000..b024fcbad0 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamResolvingStrategy.java @@ -0,0 +1,31 @@ +package org.schabi.newpipe.extractor.stream; + +/** + * Defines what strategy of the extractor is used for playback. + */ +public enum StreamResolvingStrategy { + /** + * Uses video streams (with no audio) and separate audio streams. + * @see StreamExtractor#getVideoOnlyStreams() + * @see StreamExtractor#getAudioStreams() + */ + VIDEO_ONLY_AND_AUDIO_STREAMS, + /** + * Uses video streams that include audio data. + * + * @see StreamExtractor#getVideoStreams() + */ + VIDEO_AUDIO_STREAMS, + /** + * Uses the HLS master playlist url. + * + * @see StreamExtractor#getHlsMasterPlaylistUrl() + */ + HLS_MASTER_PLAYLIST_URL, + /** + * Uses the DASH MPD url. + * + * @see StreamExtractor#getDashMpdUrl() + */ + DASH_MPD_URL +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamType.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamType.java deleted file mode 100644 index 7e668cbd46..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamType.java +++ /dev/null @@ -1,74 +0,0 @@ -package org.schabi.newpipe.extractor.stream; - -/** - * An enum representing the stream type of a {@link StreamInfo} extracted by a {@link - * StreamExtractor}. - */ -public enum StreamType { - - /** - * Placeholder to check if the stream type was checked or not. It doesn't make sense to use this - * enum constant outside of the extractor as it will never be returned by an {@link - * org.schabi.newpipe.extractor.Extractor} and is only used internally. - */ - NONE, - - /** - * A normal video stream, usually with audio. Note that the {@link StreamInfo} can also - * provide audio-only {@link AudioStream}s in addition to video or video-only {@link - * VideoStream}s. - */ - VIDEO_STREAM, - - /** - * An audio-only stream. There should be no {@link VideoStream}s available! In order to prevent - * unexpected behaviors, when {@link StreamExtractor}s return this stream type, they should - * ensure that no video stream is returned in {@link StreamExtractor#getVideoStreams()} and - * {@link StreamExtractor#getVideoOnlyStreams()}. - */ - AUDIO_STREAM, - - /** - * A video live stream, usually with audio. Note that the {@link StreamInfo} can also - * provide audio-only {@link AudioStream}s in addition to video or video-only {@link - * VideoStream}s. - */ - LIVE_STREAM, - - /** - * An audio-only live stream. There should be no {@link VideoStream}s available! In order to - * prevent unexpected behaviors, when {@link StreamExtractor}s return this stream type, they - * should ensure that no video stream is returned in {@link StreamExtractor#getVideoStreams()} - * and {@link StreamExtractor#getVideoOnlyStreams()}. - */ - AUDIO_LIVE_STREAM, - - /** - * A video live stream that has just ended but has not yet been encoded into a normal video - * stream. Note that the {@link StreamInfo} can also provide audio-only {@link - * AudioStream}s in addition to video or video-only {@link VideoStream}s. - * - *

- * Note that most of the content of an ended live video (or audio) may be extracted as {@link - * #VIDEO_STREAM regular video contents} (or {@link #AUDIO_STREAM regular audio contents}) - * later, because the service may encode them again later as normal video/audio streams. That's - * the case on YouTube, for example. - *

- */ - POST_LIVE_STREAM, - - /** - * An audio live stream that has just ended but has not yet been encoded into a normal audio - * stream. There should be no {@link VideoStream}s available! In order to prevent unexpected - * behaviors, when {@link StreamExtractor}s return this stream type, they should ensure that no - * video stream is returned in {@link StreamExtractor#getVideoStreams()} and - * {@link StreamExtractor#getVideoOnlyStreams()}. - * - *

- * Note that most of ended live audio streams extracted with this value are processed as - * {@link #AUDIO_STREAM regular audio streams} later, because the service may encode them - * again later. - *

- */ - POST_LIVE_AUDIO_STREAM -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java deleted file mode 100644 index 778a85c93a..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java +++ /dev/null @@ -1,328 +0,0 @@ -package org.schabi.newpipe.extractor.stream; - -import org.schabi.newpipe.extractor.MediaFormat; -import org.schabi.newpipe.extractor.services.youtube.ItagItem; - -import java.util.Locale; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -public final class SubtitlesStream extends Stream { - private final MediaFormat format; - private final Locale locale; - private final boolean autoGenerated; - private final String code; - - /** - * Class to build {@link SubtitlesStream} objects. - */ - @SuppressWarnings("checkstyle:HiddenField") - public static final class Builder { - private String id; - private String content; - private boolean isUrl; - private DeliveryMethod deliveryMethod = DeliveryMethod.PROGRESSIVE_HTTP; - @Nullable - private MediaFormat mediaFormat; - @Nullable - private String manifestUrl; - private String languageCode; - // Use of the Boolean class instead of the primitive type needed for setter call check - private Boolean autoGenerated; - - /** - * Create a new {@link Builder} instance with default values. - */ - public Builder() { - } - - /** - * Set the identifier of the {@link SubtitlesStream}. - * - * @param id the identifier of the {@link SubtitlesStream}, which should not be null - * (otherwise the fallback to create the identifier will be used when building - * the builder) - * @return this {@link Builder} instance - */ - public Builder setId(@Nonnull final String id) { - this.id = id; - return this; - } - - /** - * Set the content of the {@link SubtitlesStream}. - * - *

- * It must not be null, and should be non empty. - *

- * - * @param content the content of the {@link SubtitlesStream}, which must not be null - * @param isUrl whether the content is a URL - * @return this {@link Builder} instance - */ - public Builder setContent(@Nonnull final String content, - final boolean isUrl) { - this.content = content; - this.isUrl = isUrl; - return this; - } - - /** - * Set the {@link MediaFormat} used by the {@link SubtitlesStream}. - * - *

- * It should be one of the subtitles {@link MediaFormat}s ({@link MediaFormat#SRT SRT}, - * {@link MediaFormat#TRANSCRIPT1 TRANSCRIPT1}, {@link MediaFormat#TRANSCRIPT2 - * TRANSCRIPT2}, {@link MediaFormat#TRANSCRIPT3 TRANSCRIPT3}, {@link MediaFormat#TTML - * TTML}, or {@link MediaFormat#VTT VTT}) but can be {@code null} if the media format could - * not be determined. - *

- * - *

- * The default value is {@code null}. - *

- * - * @param mediaFormat the {@link MediaFormat} of the {@link SubtitlesStream}, which can be - * null - * @return this {@link Builder} instance - */ - public Builder setMediaFormat(@Nullable final MediaFormat mediaFormat) { - this.mediaFormat = mediaFormat; - return this; - } - - /** - * Set the {@link DeliveryMethod} of the {@link SubtitlesStream}. - * - *

- * It must not be null. - *

- * - *

- * The default delivery method is {@link DeliveryMethod#PROGRESSIVE_HTTP}. - *

- * - * @param deliveryMethod the {@link DeliveryMethod} of the {@link SubtitlesStream}, which - * must not be null - * @return this {@link Builder} instance - */ - public Builder setDeliveryMethod(@Nonnull final DeliveryMethod deliveryMethod) { - this.deliveryMethod = deliveryMethod; - return this; - } - - /** - * Sets the URL of the manifest this stream comes from (if applicable, otherwise null). - * - * @param manifestUrl the URL of the manifest this stream comes from or {@code null} - * @return this {@link Builder} instance - */ - public Builder setManifestUrl(@Nullable final String manifestUrl) { - this.manifestUrl = manifestUrl; - return this; - } - - /** - * Set the language code of the {@link SubtitlesStream}. - * - *

- * It must not be null and should not be an empty string. - *

- * - * @param languageCode the language code of the {@link SubtitlesStream} - * @return this {@link Builder} instance - */ - public Builder setLanguageCode(@Nonnull final String languageCode) { - this.languageCode = languageCode; - return this; - } - - /** - * Set whether the subtitles have been auto-generated by the streaming service. - * - * @param autoGenerated whether the subtitles have been generated by the streaming - * service - * @return this {@link Builder} instance - */ - public Builder setAutoGenerated(final boolean autoGenerated) { - this.autoGenerated = autoGenerated; - return this; - } - - /** - * Build a {@link SubtitlesStream} using the builder's current values. - * - *

- * The content (and so the {@code isUrl} boolean), the language code and the {@code - * isAutoGenerated} properties must have been set. - *

- * - *

- * If no identifier has been set, an identifier will be generated using the language code - * and the media format suffix, if the media format is known. - *

- * - * @return a new {@link SubtitlesStream} using the builder's current values - * @throws IllegalStateException if {@code id}, {@code content} (and so {@code isUrl}), - * {@code deliveryMethod}, {@code languageCode} or the {@code isAutogenerated} have been - * not set, or have been set as {@code null} - */ - @Nonnull - public SubtitlesStream build() { - if (content == null) { - throw new IllegalStateException("No valid content was specified. Please specify a " - + "valid one with setContent."); - } - - if (deliveryMethod == null) { - throw new IllegalStateException( - "The delivery method of the subtitles stream has been set as null, which " - + "is not allowed. Pass a valid one instead with" - + "setDeliveryMethod."); - } - - if (languageCode == null) { - throw new IllegalStateException("The language code of the subtitles stream has " - + "been not set or is null. Make sure you specified an non null language " - + "code with setLanguageCode."); - } - - if (autoGenerated == null) { - throw new IllegalStateException("The subtitles stream has been not set as an " - + "autogenerated subtitles stream or not. Please specify this information " - + "with setIsAutoGenerated."); - } - - if (id == null) { - id = languageCode + (mediaFormat != null ? "." + mediaFormat.suffix - : ""); - } - - return new SubtitlesStream(id, content, isUrl, mediaFormat, deliveryMethod, - languageCode, autoGenerated, manifestUrl); - } - } - - /** - * Create a new subtitles stream. - * - * @param id the identifier which uniquely identifies the stream, e.g. for YouTube - * this would be the itag - * @param content the content or the URL of the stream, depending on whether isUrl is - * true - * @param isUrl whether content is the URL or the actual content of e.g. a DASH - * manifest - * @param mediaFormat the {@link MediaFormat} used by the stream - * @param deliveryMethod the {@link DeliveryMethod} of the stream - * @param languageCode the language code of the stream - * @param autoGenerated whether the subtitles are auto-generated by the streaming service - * @param manifestUrl the URL of the manifest this stream comes from (if applicable, - * otherwise null) - */ - @SuppressWarnings("checkstyle:ParameterNumber") - private SubtitlesStream(@Nonnull final String id, - @Nonnull final String content, - final boolean isUrl, - @Nullable final MediaFormat mediaFormat, - @Nonnull final DeliveryMethod deliveryMethod, - @Nonnull final String languageCode, - final boolean autoGenerated, - @Nullable final String manifestUrl) { - super(id, content, isUrl, mediaFormat, deliveryMethod, manifestUrl); - - /* - * Locale.forLanguageTag only for Android API >= 21 - * Locale.Builder only for Android API >= 21 - * Country codes doesn't work well without - */ - final String[] splits = languageCode.split("-"); - switch (splits.length) { - case 2: - this.locale = new Locale(splits[0], splits[1]); - break; - case 3: - // Complex variants don't work! - this.locale = new Locale(splits[0], splits[1], splits[2]); - break; - default: - this.locale = new Locale(splits[0]); - break; - } - - this.code = languageCode; - this.format = mediaFormat; - this.autoGenerated = autoGenerated; - } - - /** - * Get the extension of the subtitles. - * - * @return the extension of the subtitles - */ - public String getExtension() { - return format.suffix; - } - - /** - * Return whether if the subtitles are auto-generated. - *

- * Some streaming services can generate subtitles for their contents, like YouTube. - *

- * - * @return {@code true} if the subtitles are auto-generated, {@code false} otherwise - */ - public boolean isAutoGenerated() { - return autoGenerated; - } - - /** - * {@inheritDoc} - */ - @Override - public boolean equalStats(final Stream cmp) { - return super.equalStats(cmp) - && cmp instanceof SubtitlesStream - && code.equals(((SubtitlesStream) cmp).code) - && autoGenerated == ((SubtitlesStream) cmp).autoGenerated; - } - - /** - * Get the display language name of the subtitles. - * - * @return the display language name of the subtitles - */ - public String getDisplayLanguageName() { - return locale.getDisplayName(locale); - } - - /** - * Get the language tag of the subtitles. - * - * @return the language tag of the subtitles - */ - public String getLanguageTag() { - return code; - } - - /** - * Get the {@link Locale locale} of the subtitles. - * - * @return the {@link Locale locale} of the subtitles - */ - public Locale getLocale() { - return locale; - } - - /** - * No subtitles which are currently extracted use an {@link ItagItem}, so {@code null} is - * returned by this method. - * - * @return {@code null} - */ - @Nullable - @Override - public ItagItem getItagItem() { - return null; - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java deleted file mode 100644 index 14952ebd1a..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java +++ /dev/null @@ -1,485 +0,0 @@ -package org.schabi.newpipe.extractor.stream; - -/* - * Created by Christian Schabesberger on 04.03.16. - * - * Copyright (C) Christian Schabesberger 2016 - * VideoStream.java is part of NewPipe Extractor. - * - * NewPipe Extractor is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe Extractor is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe Extractor. If not, see . - */ - -import org.schabi.newpipe.extractor.MediaFormat; -import org.schabi.newpipe.extractor.services.youtube.ItagItem; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -public final class VideoStream extends Stream { - public static final String RESOLUTION_UNKNOWN = ""; - - /** @deprecated Use {@link #getResolution()} instead. */ - @Deprecated - public final String resolution; - - /** @deprecated Use {@link #isVideoOnly()} instead. */ - @Deprecated - public final boolean isVideoOnly; - - // Fields for DASH - private int itag = ITAG_NOT_AVAILABLE_OR_NOT_APPLICABLE; - private int bitrate; - private int initStart; - private int initEnd; - private int indexStart; - private int indexEnd; - private int width; - private int height; - private int fps; - private String quality; - private String codec; - @Nullable private ItagItem itagItem; - - /** - * Class to build {@link VideoStream} objects. - */ - @SuppressWarnings("checkstyle:hiddenField") - public static final class Builder { - private String id; - private String content; - private boolean isUrl; - private DeliveryMethod deliveryMethod = DeliveryMethod.PROGRESSIVE_HTTP; - @Nullable - private MediaFormat mediaFormat; - @Nullable - private String manifestUrl; - // Use of the Boolean class instead of the primitive type needed for setter call check - private Boolean isVideoOnly; - private String resolution; - @Nullable - private ItagItem itagItem; - - /** - * Create a new {@link Builder} instance with its default values. - */ - public Builder() { - } - - /** - * Set the identifier of the {@link VideoStream}. - * - *

- * It must not be null, and should be non empty. - *

- * - *

- * If you are not able to get an identifier, use the static constant {@link - * Stream#ID_UNKNOWN ID_UNKNOWN} of the {@link Stream} class. - *

- * - * @param id the identifier of the {@link VideoStream}, which must not be null - * @return this {@link Builder} instance - */ - public Builder setId(@Nonnull final String id) { - this.id = id; - return this; - } - - /** - * Set the content of the {@link VideoStream}. - * - *

- * It must not be null, and should be non empty. - *

- * - * @param content the content of the {@link VideoStream} - * @param isUrl whether the content is a URL - * @return this {@link Builder} instance - */ - public Builder setContent(@Nonnull final String content, - final boolean isUrl) { - this.content = content; - this.isUrl = isUrl; - return this; - } - - /** - * Set the {@link MediaFormat} used by the {@link VideoStream}. - * - *

- * It should be one of the video {@link MediaFormat}s ({@link MediaFormat#MPEG_4 MPEG_4}, - * {@link MediaFormat#v3GPP v3GPP}, or {@link MediaFormat#WEBM WEBM}) but can be {@code - * null} if the media format could not be determined. - *

- * - *

- * The default value is {@code null}. - *

- * - * @param mediaFormat the {@link MediaFormat} of the {@link VideoStream}, which can be null - * @return this {@link Builder} instance - */ - public Builder setMediaFormat(@Nullable final MediaFormat mediaFormat) { - this.mediaFormat = mediaFormat; - return this; - } - - /** - * Set the {@link DeliveryMethod} of the {@link VideoStream}. - * - *

- * It must not be null. - *

- * - *

- * The default delivery method is {@link DeliveryMethod#PROGRESSIVE_HTTP}. - *

- * - * @param deliveryMethod the {@link DeliveryMethod} of the {@link VideoStream}, which must - * not be null - * @return this {@link Builder} instance - */ - public Builder setDeliveryMethod(@Nonnull final DeliveryMethod deliveryMethod) { - this.deliveryMethod = deliveryMethod; - return this; - } - - /** - * Sets the URL of the manifest this stream comes from (if applicable, otherwise null). - * - * @param manifestUrl the URL of the manifest this stream comes from or {@code null} - * @return this {@link Builder} instance - */ - public Builder setManifestUrl(@Nullable final String manifestUrl) { - this.manifestUrl = manifestUrl; - return this; - } - - /** - * Set whether the {@link VideoStream} is video-only. - * - *

- * This property must be set before building the {@link VideoStream}. - *

- * - * @param isVideoOnly whether the {@link VideoStream} is video-only - * @return this {@link Builder} instance - */ - public Builder setIsVideoOnly(final boolean isVideoOnly) { - this.isVideoOnly = isVideoOnly; - return this; - } - - /** - * Set the resolution of the {@link VideoStream}. - * - *

- * This resolution can be used by clients to know the quality of the video stream. - *

- * - *

- * If you are not able to know the resolution, you should use {@link #RESOLUTION_UNKNOWN} - * as the resolution of the video stream. - *

- * - *

- * It must be set before building the builder and not null. - *

- * - * @param resolution the resolution of the {@link VideoStream} - * @return this {@link Builder} instance - */ - public Builder setResolution(@Nonnull final String resolution) { - this.resolution = resolution; - return this; - } - - /** - * Set the {@link ItagItem} corresponding to the {@link VideoStream}. - * - *

- * {@link ItagItem}s are YouTube specific objects, so they are only known for this service - * and can be null. - *

- * - *

- * The default value is {@code null}. - *

- * - * @param itagItem the {@link ItagItem} of the {@link VideoStream}, which can be null - * @return this {@link Builder} instance - */ - public Builder setItagItem(@Nullable final ItagItem itagItem) { - this.itagItem = itagItem; - return this; - } - - /** - * Build a {@link VideoStream} using the builder's current values. - * - *

- * The identifier, the content (and so the {@code isUrl} boolean), the {@code isVideoOnly} - * and the {@code resolution} properties must have been set. - *

- * - * @return a new {@link VideoStream} using the builder's current values - * @throws IllegalStateException if {@code id}, {@code content} (and so {@code isUrl}), - * {@code deliveryMethod}, {@code isVideoOnly} or {@code resolution} have been not set, or - * have been set as {@code null} - */ - @Nonnull - public VideoStream build() { - if (id == null) { - throw new IllegalStateException( - "The identifier of the video stream has been not set or is null. If you " - + "are not able to get an identifier, use the static constant " - + "ID_UNKNOWN of the Stream class."); - } - - if (content == null) { - throw new IllegalStateException("The content of the video stream has been not set " - + "or is null. Please specify a non-null one with setContent."); - } - - if (deliveryMethod == null) { - throw new IllegalStateException( - "The delivery method of the video stream has been set as null, which is " - + "not allowed. Pass a valid one instead with setDeliveryMethod."); - } - - if (isVideoOnly == null) { - throw new IllegalStateException("The video stream has been not set as a " - + "video-only stream or as a video stream with embedded audio. Please " - + "specify this information with setIsVideoOnly."); - } - - if (resolution == null) { - throw new IllegalStateException( - "The resolution of the video stream has been not set. Please specify it " - + "with setResolution (use an empty string if you are not able to " - + "get it)."); - } - - return new VideoStream(id, content, isUrl, mediaFormat, deliveryMethod, resolution, - isVideoOnly, manifestUrl, itagItem); - } - } - - /** - * Create a new video stream. - * - * @param id the identifier which uniquely identifies the stream, e.g. for YouTube - * this would be the itag - * @param content the content or the URL of the stream, depending on whether isUrl is - * true - * @param isUrl whether content is the URL or the actual content of e.g. a DASH - * manifest - * @param format the {@link MediaFormat} used by the stream, which can be null - * @param deliveryMethod the {@link DeliveryMethod} of the stream - * @param resolution the resolution of the stream - * @param isVideoOnly whether the stream is video-only - * @param itagItem the {@link ItagItem} corresponding to the stream, which cannot be null - * @param manifestUrl the URL of the manifest this stream comes from (if applicable, - * otherwise null) - */ - @SuppressWarnings("checkstyle:ParameterNumber") - private VideoStream(@Nonnull final String id, - @Nonnull final String content, - final boolean isUrl, - @Nullable final MediaFormat format, - @Nonnull final DeliveryMethod deliveryMethod, - @Nonnull final String resolution, - final boolean isVideoOnly, - @Nullable final String manifestUrl, - @Nullable final ItagItem itagItem) { - super(id, content, isUrl, format, deliveryMethod, manifestUrl); - if (itagItem != null) { - this.itagItem = itagItem; - this.itag = itagItem.id; - this.bitrate = itagItem.getBitrate(); - this.initStart = itagItem.getInitStart(); - this.initEnd = itagItem.getInitEnd(); - this.indexStart = itagItem.getIndexStart(); - this.indexEnd = itagItem.getIndexEnd(); - this.codec = itagItem.getCodec(); - this.height = itagItem.getHeight(); - this.width = itagItem.getWidth(); - this.quality = itagItem.getQuality(); - this.fps = itagItem.getFps(); - } - this.resolution = resolution; - this.isVideoOnly = isVideoOnly; - } - - /** - * {@inheritDoc} - */ - @Override - public boolean equalStats(final Stream cmp) { - return super.equalStats(cmp) - && cmp instanceof VideoStream - && resolution.equals(((VideoStream) cmp).resolution) - && isVideoOnly == ((VideoStream) cmp).isVideoOnly; - } - - /** - * Get the video resolution. - * - *

- * It can be unknown for some streams, like for HLS master playlists. In this case, - * {@link #RESOLUTION_UNKNOWN} is returned by this method. - *

- * - * @return the video resolution or {@link #RESOLUTION_UNKNOWN} - */ - @Nonnull - public String getResolution() { - return resolution; - } - - /** - * Return whether the stream is video-only. - * - *

- * Video-only streams have no audio. - *

- * - * @return {@code true} if this stream is video-only, {@code false} otherwise - */ - public boolean isVideoOnly() { - return isVideoOnly; - } - - /** - * Get the itag identifier of the stream. - * - *

- * Always equals to {@link #ITAG_NOT_AVAILABLE_OR_NOT_APPLICABLE} for other streams than the - * ones of the YouTube service. - *

- * - * @return the number of the {@link ItagItem} passed in the constructor of the video stream. - */ - public int getItag() { - return itag; - } - - /** - * Get the bitrate of the stream. - * - * @return the bitrate set from the {@link ItagItem} passed in the constructor of the stream. - */ - public int getBitrate() { - return bitrate; - } - - /** - * Get the initialization start of the stream. - * - * @return the initialization start value set from the {@link ItagItem} passed in the - * constructor of the - * stream. - */ - public int getInitStart() { - return initStart; - } - - /** - * Get the initialization end of the stream. - * - * @return the initialization end value set from the {@link ItagItem} passed in the constructor - * of the stream. - */ - public int getInitEnd() { - return initEnd; - } - - /** - * Get the index start of the stream. - * - * @return the index start value set from the {@link ItagItem} passed in the constructor of the - * stream. - */ - public int getIndexStart() { - return indexStart; - } - - /** - * Get the index end of the stream. - * - * @return the index end value set from the {@link ItagItem} passed in the constructor of the - * stream. - */ - public int getIndexEnd() { - return indexEnd; - } - - /** - * Get the width of the video stream. - * - * @return the width set from the {@link ItagItem} passed in the constructor of the - * stream. - */ - public int getWidth() { - return width; - } - - /** - * Get the height of the video stream. - * - * @return the height set from the {@link ItagItem} passed in the constructor of the - * stream. - */ - public int getHeight() { - return height; - } - - /** - * Get the frames per second of the video stream. - * - * @return the frames per second set from the {@link ItagItem} passed in the constructor of the - * stream. - */ - public int getFps() { - return fps; - } - - /** - * Get the quality of the stream. - * - * @return the quality label set from the {@link ItagItem} passed in the constructor of the - * stream. - */ - public String getQuality() { - return quality; - } - - /** - * Get the codec of the stream. - * - * @return the codec set from the {@link ItagItem} passed in the constructor of the stream. - */ - public String getCodec() { - return codec; - } - - /** - * {@inheritDoc} - */ - @Override - @Nullable - public ItagItem getItagItem() { - return itagItem; - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/DASHDeliveryData.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/DASHDeliveryData.java new file mode 100644 index 0000000000..42d7e452d5 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/DASHDeliveryData.java @@ -0,0 +1,5 @@ +package org.schabi.newpipe.extractor.streamdata.delivery; + +public interface DASHDeliveryData extends DeliveryData { + // Just a marker interface +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/DASHManifestDeliveryData.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/DASHManifestDeliveryData.java new file mode 100644 index 0000000000..3360ca9e68 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/DASHManifestDeliveryData.java @@ -0,0 +1,24 @@ +package org.schabi.newpipe.extractor.streamdata.delivery; + +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreator; + +import javax.annotation.Nonnull; + +public interface DASHManifestDeliveryData extends DASHDeliveryData, DownloadableDeliveryData { + @Nonnull + DashManifestCreator dashManifestCreator(); + + String getCachedDashManifestAsString(); + + @Nonnull + @Override + default String downloadUrl() { + return dashManifestCreator().downloadUrl(); + } + + @Override + default long getExpectedContentLength(final Downloader downloader) { + return dashManifestCreator().getExpectedContentLength(downloader); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/DASHUrlDeliveryData.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/DASHUrlDeliveryData.java new file mode 100644 index 0000000000..9204bb2b37 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/DASHUrlDeliveryData.java @@ -0,0 +1,5 @@ +package org.schabi.newpipe.extractor.streamdata.delivery; + +public interface DASHUrlDeliveryData extends DownloadableUrlBasedDeliveryData, DASHDeliveryData { + // Nothing to implement additionally +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/DeliveryData.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/DeliveryData.java new file mode 100644 index 0000000000..dfb2a158ab --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/DeliveryData.java @@ -0,0 +1,5 @@ +package org.schabi.newpipe.extractor.streamdata.delivery; + +public interface DeliveryData { + // Just a marker +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/DownloadableDeliveryData.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/DownloadableDeliveryData.java new file mode 100644 index 0000000000..c4c57340c0 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/DownloadableDeliveryData.java @@ -0,0 +1,22 @@ +package org.schabi.newpipe.extractor.streamdata.delivery; + +import org.schabi.newpipe.extractor.downloader.Downloader; + +import javax.annotation.Nonnull; + +/** + * Provides information for downloading a stream. + */ +public interface DownloadableDeliveryData extends DeliveryData { + + @Nonnull + String downloadUrl(); + + /** + * Returns the expected content length/size of the data. + * + * @param downloader The downloader that may be used for fetching (HTTP HEAD). + * @return the expected size/content length or -1 if unknown + */ + long getExpectedContentLength(Downloader downloader); +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/DownloadableUrlBasedDeliveryData.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/DownloadableUrlBasedDeliveryData.java new file mode 100644 index 0000000000..e00b735289 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/DownloadableUrlBasedDeliveryData.java @@ -0,0 +1,19 @@ +package org.schabi.newpipe.extractor.streamdata.delivery; + +import org.schabi.newpipe.extractor.downloader.Downloader; + +import javax.annotation.Nonnull; + +public interface DownloadableUrlBasedDeliveryData + extends UrlBasedDeliveryData, DownloadableDeliveryData { + @Nonnull + @Override + default String downloadUrl() { + return url(); + } + + @Override + default long getExpectedContentLength(final Downloader downloader) { + return downloader.getContentLength(url()); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/HLSDeliveryData.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/HLSDeliveryData.java new file mode 100644 index 0000000000..abe46f86d5 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/HLSDeliveryData.java @@ -0,0 +1,5 @@ +package org.schabi.newpipe.extractor.streamdata.delivery; + +public interface HLSDeliveryData extends DownloadableUrlBasedDeliveryData { + // Nothing to implement additionally +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/ProgressiveHTTPDeliveryData.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/ProgressiveHTTPDeliveryData.java new file mode 100644 index 0000000000..672ad1c512 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/ProgressiveHTTPDeliveryData.java @@ -0,0 +1,5 @@ +package org.schabi.newpipe.extractor.streamdata.delivery; + +public interface ProgressiveHTTPDeliveryData extends DownloadableUrlBasedDeliveryData { + // Nothing to implement additionally +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/TorrentDeliveryData.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/TorrentDeliveryData.java new file mode 100644 index 0000000000..b7112a37b9 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/TorrentDeliveryData.java @@ -0,0 +1,5 @@ +package org.schabi.newpipe.extractor.streamdata.delivery; + +public interface TorrentDeliveryData extends UrlBasedDeliveryData { + // Nothing to implement additionally +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/UrlBasedDeliveryData.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/UrlBasedDeliveryData.java new file mode 100644 index 0000000000..a0df54f913 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/UrlBasedDeliveryData.java @@ -0,0 +1,8 @@ +package org.schabi.newpipe.extractor.streamdata.delivery; + +import javax.annotation.Nonnull; + +public interface UrlBasedDeliveryData extends DeliveryData { + @Nonnull + String url(); +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/dashmanifestcreator/DashManifestCreationException.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/dashmanifestcreator/DashManifestCreationException.java new file mode 100644 index 0000000000..4c60e7fc73 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/dashmanifestcreator/DashManifestCreationException.java @@ -0,0 +1,57 @@ +package org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator; + +import javax.annotation.Nonnull; + +/** + * Exception that is thrown when a DASH manifest creator encounters a problem + * while creating a manifest. + */ +public class DashManifestCreationException extends RuntimeException { + + public DashManifestCreationException(final String message) { + super(message); + } + + public DashManifestCreationException(final String message, final Exception cause) { + super(message, cause); + } + + // Methods to create exceptions easily without having to use big exception messages and to + // reduce duplication + + /** + * Create a new {@link DashManifestCreationException} with a cause and the following detail + * message format: + *
+ * {@code "Could not add " + element + " element", cause}, where {@code element} is an element + * of a DASH manifest. + * + * @param element the element which was not added to the DASH document + * @param cause the exception which prevented addition of the element to the DASH document + * @return a new {@link DashManifestCreationException} + */ + @Nonnull + public static DashManifestCreationException couldNotAddElement(final String element, + final Exception cause) { + return new DashManifestCreationException("Could not add " + element + " element", cause); + } + + /** + * Create a new {@link DashManifestCreationException} with a cause and the following detail + * message format: + *
+ * {@code "Could not add " + element + " element: " + reason}, where {@code element} is an + * element of a DASH manifest and {@code reason} the reason why this element cannot be added to + * the DASH document. + * + * @param element the element which was not added to the DASH document + * @param reason the reason message of why the element has been not added to the DASH document + * @return a new {@link DashManifestCreationException} + */ + @Nonnull + public static DashManifestCreationException couldNotAddElement(final String element, + final String reason) { + return new DashManifestCreationException( + "Could not add " + element + " element: " + reason); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/dashmanifestcreator/DashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/dashmanifestcreator/DashManifestCreator.java new file mode 100644 index 0000000000..77a39b5299 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/dashmanifestcreator/DashManifestCreator.java @@ -0,0 +1,28 @@ +package org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator; + +import org.schabi.newpipe.extractor.downloader.Downloader; + +import javax.annotation.Nonnull; + +public interface DashManifestCreator { + + /** + * Generates the DASH manifest. + * + * @return The dash manifest as string. + * @throws DashManifestCreationException May throw a CreationException + */ + @Nonnull + String generateManifest(); + + @Nonnull + String downloadUrl(); + + // CHECKSTYLE:OFF - Link is too long + /** + * See + * {@link org.schabi.newpipe.extractor.streamdata.delivery.DownloadableDeliveryData#getExpectedContentLength(Downloader)} + */ + // CHECKSTYLE:ON + long getExpectedContentLength(Downloader downloader); +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/dashmanifestcreator/DashManifestCreatorConstants.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/dashmanifestcreator/DashManifestCreatorConstants.java new file mode 100644 index 0000000000..01c494dcc4 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/dashmanifestcreator/DashManifestCreatorConstants.java @@ -0,0 +1,21 @@ +package org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator; + +public final class DashManifestCreatorConstants { + private DashManifestCreatorConstants() { + // No impl + } + + // XML elements of DASH MPD manifests + // see https://www.brendanlong.com/the-structure-of-an-mpeg-dash-mpd.html + public static final String MPD = "MPD"; + public static final String PERIOD = "Period"; + public static final String ADAPTATION_SET = "AdaptationSet"; + public static final String ROLE = "Role"; + public static final String REPRESENTATION = "Representation"; + public static final String AUDIO_CHANNEL_CONFIGURATION = "AudioChannelConfiguration"; + public static final String SEGMENT_TEMPLATE = "SegmentTemplate"; + public static final String SEGMENT_TIMELINE = "SegmentTimeline"; + public static final String BASE_URL = "BaseURL"; + public static final String SEGMENT_BASE = "SegmentBase"; + public static final String INITIALIZATION = "Initialization"; +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/AbstractDeliveryDataImpl.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/AbstractDeliveryDataImpl.java new file mode 100644 index 0000000000..c6c70163b6 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/AbstractDeliveryDataImpl.java @@ -0,0 +1,7 @@ +package org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl; + +import org.schabi.newpipe.extractor.streamdata.delivery.DeliveryData; + +public abstract class AbstractDeliveryDataImpl implements DeliveryData { + // Nothing to implement so far +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/AbstractUrlBasedDeliveryDataImpl.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/AbstractUrlBasedDeliveryDataImpl.java new file mode 100644 index 0000000000..81e4d6a4ff --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/AbstractUrlBasedDeliveryDataImpl.java @@ -0,0 +1,25 @@ +package org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl; + +import org.schabi.newpipe.extractor.streamdata.delivery.UrlBasedDeliveryData; + +import java.util.Objects; + +import javax.annotation.Nonnull; + +public abstract class AbstractUrlBasedDeliveryDataImpl extends AbstractDeliveryDataImpl + implements UrlBasedDeliveryData { + + @Nonnull + private final String url; + + protected AbstractUrlBasedDeliveryDataImpl(@Nonnull final String url) { + this.url = Objects.requireNonNull(url); + } + + @Nonnull + @Override + public String url() { + return url; + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/SimpleDASHManifestDeliveryDataImpl.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/SimpleDASHManifestDeliveryDataImpl.java new file mode 100644 index 0000000000..a32e6b86d6 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/SimpleDASHManifestDeliveryDataImpl.java @@ -0,0 +1,36 @@ +package org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl; + +import org.schabi.newpipe.extractor.streamdata.delivery.DASHManifestDeliveryData; +import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreator; + +import java.util.Objects; + +import javax.annotation.Nonnull; + +public class SimpleDASHManifestDeliveryDataImpl extends AbstractDeliveryDataImpl + implements DASHManifestDeliveryData { + @Nonnull + private final DashManifestCreator dashManifestCreator; + + private String cachedDashManifest; + + public SimpleDASHManifestDeliveryDataImpl( + @Nonnull final DashManifestCreator dashManifestCreator + ) { + this.dashManifestCreator = Objects.requireNonNull(dashManifestCreator); + } + + @Override + @Nonnull + public DashManifestCreator dashManifestCreator() { + return dashManifestCreator; + } + + @Override + public String getCachedDashManifestAsString() { + if (cachedDashManifest == null) { + cachedDashManifest = dashManifestCreator().generateManifest(); + } + return cachedDashManifest; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/SimpleDASHUrlDeliveryDataImpl.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/SimpleDASHUrlDeliveryDataImpl.java new file mode 100644 index 0000000000..263097915e --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/SimpleDASHUrlDeliveryDataImpl.java @@ -0,0 +1,12 @@ +package org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl; + +import org.schabi.newpipe.extractor.streamdata.delivery.DASHUrlDeliveryData; + +import javax.annotation.Nonnull; + +public class SimpleDASHUrlDeliveryDataImpl extends AbstractUrlBasedDeliveryDataImpl + implements DASHUrlDeliveryData { + public SimpleDASHUrlDeliveryDataImpl(@Nonnull final String url) { + super(url); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/SimpleHLSDeliveryDataImpl.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/SimpleHLSDeliveryDataImpl.java new file mode 100644 index 0000000000..9496825bfc --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/SimpleHLSDeliveryDataImpl.java @@ -0,0 +1,12 @@ +package org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl; + +import org.schabi.newpipe.extractor.streamdata.delivery.HLSDeliveryData; + +import javax.annotation.Nonnull; + +public class SimpleHLSDeliveryDataImpl extends AbstractUrlBasedDeliveryDataImpl + implements HLSDeliveryData { + public SimpleHLSDeliveryDataImpl(@Nonnull final String url) { + super(url); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/SimpleProgressiveHTTPDeliveryDataImpl.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/SimpleProgressiveHTTPDeliveryDataImpl.java new file mode 100644 index 0000000000..94b09b4ab7 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/SimpleProgressiveHTTPDeliveryDataImpl.java @@ -0,0 +1,12 @@ +package org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl; + +import org.schabi.newpipe.extractor.streamdata.delivery.ProgressiveHTTPDeliveryData; + +import javax.annotation.Nonnull; + +public class SimpleProgressiveHTTPDeliveryDataImpl extends AbstractUrlBasedDeliveryDataImpl + implements ProgressiveHTTPDeliveryData { + public SimpleProgressiveHTTPDeliveryDataImpl(@Nonnull final String url) { + super(url); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/SimpleTorrentDeliveryDataImpl.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/SimpleTorrentDeliveryDataImpl.java new file mode 100644 index 0000000000..f3a3b36dbf --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/simpleimpl/SimpleTorrentDeliveryDataImpl.java @@ -0,0 +1,12 @@ +package org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl; + +import org.schabi.newpipe.extractor.streamdata.delivery.TorrentDeliveryData; + +import javax.annotation.Nonnull; + +public class SimpleTorrentDeliveryDataImpl extends AbstractUrlBasedDeliveryDataImpl + implements TorrentDeliveryData { + public SimpleTorrentDeliveryDataImpl(@Nonnull final String url) { + super(url); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/AbstractMediaFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/AbstractMediaFormat.java new file mode 100644 index 0000000000..68f91e02c5 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/AbstractMediaFormat.java @@ -0,0 +1,67 @@ +package org.schabi.newpipe.extractor.streamdata.format; + +import java.util.Objects; + +import javax.annotation.Nonnull; + +public abstract class AbstractMediaFormat implements MediaFormat { + private final int id; + private final String name; + private final String suffix; + private final String mimeType; + + protected AbstractMediaFormat( + final int id, + @Nonnull final String name, + @Nonnull final String suffix, + @Nonnull final String mimeType + ) { + this.id = id; + this.name = Objects.requireNonNull(name); + this.suffix = Objects.requireNonNull(suffix); + this.mimeType = Objects.requireNonNull(mimeType); + } + + @Override + public int id() { + return id; + } + + @Nonnull + @Override + public String name() { + return name; + } + + @Nonnull + @Override + public String suffix() { + return suffix; + } + + @Nonnull + @Override + public String mimeType() { + return mimeType; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof AbstractMediaFormat)) { + return false; + } + final AbstractMediaFormat that = (AbstractMediaFormat) o; + return id() == that.id() + && Objects.equals(name(), that.name()) + && Objects.equals(suffix(), that.suffix()) + && Objects.equals(mimeType(), that.mimeType()); + } + + @Override + public int hashCode() { + return Objects.hash(id(), name(), suffix(), mimeType()); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/AudioMediaFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/AudioMediaFormat.java new file mode 100644 index 0000000000..bce4fd7f14 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/AudioMediaFormat.java @@ -0,0 +1,14 @@ +package org.schabi.newpipe.extractor.streamdata.format; + +import javax.annotation.Nonnull; + +public class AudioMediaFormat extends AbstractMediaFormat { + public AudioMediaFormat( + final int id, + @Nonnull final String name, + @Nonnull final String suffix, + @Nonnull final String mimeType + ) { + super(id, name, suffix, mimeType); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/MediaFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/MediaFormat.java new file mode 100644 index 0000000000..9448916890 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/MediaFormat.java @@ -0,0 +1,17 @@ +package org.schabi.newpipe.extractor.streamdata.format; + +import javax.annotation.Nonnull; + +public interface MediaFormat { + + int id(); + + @Nonnull + String name(); + + @Nonnull + String suffix(); + + @Nonnull + String mimeType(); +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/SubtitleMediaFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/SubtitleMediaFormat.java new file mode 100644 index 0000000000..62c693d20d --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/SubtitleMediaFormat.java @@ -0,0 +1,14 @@ +package org.schabi.newpipe.extractor.streamdata.format; + +import javax.annotation.Nonnull; + +public class SubtitleMediaFormat extends AbstractMediaFormat { + public SubtitleMediaFormat( + final int id, + @Nonnull final String name, + @Nonnull final String suffix, + @Nonnull final String mimeType + ) { + super(id, name, suffix, mimeType); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/VideoAudioMediaFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/VideoAudioMediaFormat.java new file mode 100644 index 0000000000..8120b08558 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/VideoAudioMediaFormat.java @@ -0,0 +1,14 @@ +package org.schabi.newpipe.extractor.streamdata.format; + +import javax.annotation.Nonnull; + +public class VideoAudioMediaFormat extends AbstractMediaFormat { + public VideoAudioMediaFormat( + final int id, + @Nonnull final String name, + @Nonnull final String suffix, + @Nonnull final String mimeType + ) { + super(id, name, suffix, mimeType); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/registry/AudioFormatRegistry.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/registry/AudioFormatRegistry.java new file mode 100644 index 0000000000..55e1d67c45 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/registry/AudioFormatRegistry.java @@ -0,0 +1,23 @@ +package org.schabi.newpipe.extractor.streamdata.format.registry; + +import org.schabi.newpipe.extractor.streamdata.format.AudioMediaFormat; + +public class AudioFormatRegistry extends MediaFormatRegistry { + + public static final AudioMediaFormat M4A = + new AudioMediaFormat(0x100, "m4a", "m4a", "audio/mp4"); + public static final AudioMediaFormat WEBMA = + new AudioMediaFormat(0x200, "WebM", "webm", "audio/webm"); + public static final AudioMediaFormat MP3 = + new AudioMediaFormat(0x300, "MP3", "mp3", "audio/mpeg"); + public static final AudioMediaFormat OPUS = + new AudioMediaFormat(0x400, "opus", "opus", "audio/opus"); + public static final AudioMediaFormat OGG = + new AudioMediaFormat(0x500, "ogg", "ogg", "audio/ogg"); + public static final AudioMediaFormat WEBMA_OPUS = + new AudioMediaFormat(0x200, "WebM Opus", "webm", "audio/webm"); + + public AudioFormatRegistry() { + super(new AudioMediaFormat[]{M4A, WEBMA, MP3, OPUS, OGG, WEBMA_OPUS}); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/registry/MediaFormatRegistry.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/registry/MediaFormatRegistry.java new file mode 100644 index 0000000000..53d981a766 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/registry/MediaFormatRegistry.java @@ -0,0 +1,85 @@ +package org.schabi.newpipe.extractor.streamdata.format.registry; + +import org.schabi.newpipe.extractor.streamdata.format.AbstractMediaFormat; + +import java.util.Arrays; +import java.util.function.Function; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public abstract class MediaFormatRegistry { + + protected final F[] values; + + protected MediaFormatRegistry(final F[] values) { + this.values = values; + } + + public F[] values() { + return values; + } + + public T getById(final int id, + final Function field, + final T orElse) { + return Arrays.stream(values()) + .filter(mediaFormat -> mediaFormat.id() == id) + .map(field) + .findFirst() + .orElse(orElse); + } + + /** + * Return the friendly name of the media format with the supplied id + * + * @param id the id of the media format. Currently an arbitrary, NewPipe-specific number. + * @return the friendly name of the MediaFormat associated with this ids, + * or an empty String if none match it. + */ + @Nonnull + public String getNameById(final int id) { + return getById(id, AbstractMediaFormat::name, ""); + } + + /** + * Return the MIME type of the media format with the supplied id + * + * @param id the id of the media format. Currently an arbitrary, NewPipe-specific number. + * @return the MIME type of the MediaFormat associated with this ids, + * or an empty String if none match it. + */ + @Nullable + public String getMimeById(final int id) { + return getById(id, AbstractMediaFormat::mimeType, null); + } + + /** + * Return the MediaFormat with the supplied mime type + * + * @return MediaFormat associated with this mime type + * @throws IllegalStateException if there is no matching MediaFormat + */ + @Nonnull + public F getFromMimeTypeOrThrow(final String mimeType) { + return Arrays.stream(values()) + .filter(mediaFormat -> mediaFormat.mimeType().equals(mimeType)) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No matching MediaFormat")); + } + + /** + * Return the MediaFormat with the supplied suffix + * + * @return MediaFormat associated with this suffix + * @throws IllegalStateException if there is no matching MediaFormat + */ + @Nonnull + public F getFromSuffixOrThrow(final String suffix) { + return Arrays.stream(values()) + .filter(mediaFormat -> mediaFormat.suffix().equals(suffix)) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No matching MediaFormat")); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/registry/SubtitleFormatRegistry.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/registry/SubtitleFormatRegistry.java new file mode 100644 index 0000000000..8de47e3b77 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/registry/SubtitleFormatRegistry.java @@ -0,0 +1,25 @@ +package org.schabi.newpipe.extractor.streamdata.format.registry; + +import org.schabi.newpipe.extractor.streamdata.format.SubtitleMediaFormat; + +public class SubtitleFormatRegistry extends MediaFormatRegistry { + + public static final SubtitleMediaFormat VTT = + new SubtitleMediaFormat(0x1000, "WebVTT", "vtt", "text/vtt"); + public static final SubtitleMediaFormat TTML = + new SubtitleMediaFormat(0x2000, "Timed Text Markup Language", "ttml", + "application/ttml+xml"); + public static final SubtitleMediaFormat TRANSCRIPT1 = + new SubtitleMediaFormat(0x3000, "TranScript v1", "srv1", "text/xml"); + public static final SubtitleMediaFormat TRANSCRIPT2 = + new SubtitleMediaFormat(0x4000, "TranScript v2", "srv2", "text/xml"); + public static final SubtitleMediaFormat TRANSCRIPT3 = + new SubtitleMediaFormat(0x5000, "TranScript v3", "srv3", "text/xml"); + public static final SubtitleMediaFormat SRT = + new SubtitleMediaFormat(0x6000, "SubRip file format", "srt", "text/srt"); + + + public SubtitleFormatRegistry() { + super(new SubtitleMediaFormat[]{VTT, TTML, TRANSCRIPT1, TRANSCRIPT2, TRANSCRIPT3, SRT}); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/registry/VideoAudioFormatRegistry.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/registry/VideoAudioFormatRegistry.java new file mode 100644 index 0000000000..aeccfd1696 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/format/registry/VideoAudioFormatRegistry.java @@ -0,0 +1,17 @@ +package org.schabi.newpipe.extractor.streamdata.format.registry; + +import org.schabi.newpipe.extractor.streamdata.format.VideoAudioMediaFormat; + +public class VideoAudioFormatRegistry extends MediaFormatRegistry { + + public static final VideoAudioMediaFormat MPEG_4 = + new VideoAudioMediaFormat(0x0, "MPEG-4", "mp4", "video/mp4"); + public static final VideoAudioMediaFormat V3GPP = + new VideoAudioMediaFormat(0x10, "3GPP", "3gp", "video/3gpp"); + public static final VideoAudioMediaFormat WEBM = + new VideoAudioMediaFormat(0x20, "WebM", "webm", "video/webm"); + + public VideoAudioFormatRegistry() { + super(new VideoAudioMediaFormat[]{MPEG_4, V3GPP, WEBM}); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/AudioStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/AudioStream.java new file mode 100644 index 0000000000..dcb9fdce52 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/AudioStream.java @@ -0,0 +1,10 @@ +package org.schabi.newpipe.extractor.streamdata.stream; + +import org.schabi.newpipe.extractor.streamdata.format.AudioMediaFormat; + +/** + * Represents a audio (only) stream. + */ +public interface AudioStream extends Stream, BaseAudioStream { + // Nothing +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/BaseAudioStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/BaseAudioStream.java new file mode 100644 index 0000000000..ae37692b23 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/BaseAudioStream.java @@ -0,0 +1,14 @@ +package org.schabi.newpipe.extractor.streamdata.stream; + +public interface BaseAudioStream { + int UNKNOWN_AVG_BITRATE = -1; + + /** + * Average audio bitrate in KBit/s. + * + * @return the average bitrate or -1 if unknown + */ + default int averageBitrate() { + return UNKNOWN_AVG_BITRATE; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/Stream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/Stream.java new file mode 100644 index 0000000000..ac3618811e --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/Stream.java @@ -0,0 +1,20 @@ +package org.schabi.newpipe.extractor.streamdata.stream; + +import org.schabi.newpipe.extractor.streamdata.delivery.DeliveryData; +import org.schabi.newpipe.extractor.streamdata.format.MediaFormat; + +import javax.annotation.Nonnull; + +public interface Stream { + + /** + * The (container) media format, e.g. mp3 for audio streams or webm for video(+audio) streams. + * + * @return The (container) media format + */ + @Nonnull + M mediaFormat(); + + @Nonnull + DeliveryData deliveryData(); +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/SubtitleStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/SubtitleStream.java new file mode 100644 index 0000000000..47aedce7d6 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/SubtitleStream.java @@ -0,0 +1,48 @@ +package org.schabi.newpipe.extractor.streamdata.stream; + +import org.schabi.newpipe.extractor.streamdata.format.SubtitleMediaFormat; + +import java.util.Locale; + +import javax.annotation.Nonnull; + +/** + * Represents a subtitle (only) stream. + */ +public interface SubtitleStream extends Stream { + /** + * Return whether if the subtitles are auto-generated. + *

+ * Some streaming services can generate subtitles for their contents, like YouTube. + *

+ * + * @return {@code true} if the subtitles are auto-generated, {@code false} otherwise + */ + default boolean autoGenerated() { + return false; + } + + /** + * Get the language code of the subtitles. + * + * @return the language code of the subtitles + */ + @Nonnull + String languageCode(); + + /** + * Get the {@link Locale locale} of the subtitles. + * + *

+ * Note: The locale is directly derived from the language-code. + *

+ * + * @return the {@link Locale locale} of the subtitles + */ + @Nonnull + Locale locale(); + + default String getDisplayLanguageName() { + return locale().getDisplayName(locale()); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/VideoAudioStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/VideoAudioStream.java new file mode 100644 index 0000000000..88904461be --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/VideoAudioStream.java @@ -0,0 +1,8 @@ +package org.schabi.newpipe.extractor.streamdata.stream; + +/** + * Represents a combined video+audio stream. + */ +public interface VideoAudioStream extends VideoStream, BaseAudioStream { + // Nothing +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/VideoStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/VideoStream.java new file mode 100644 index 0000000000..115d4bbd12 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/VideoStream.java @@ -0,0 +1,14 @@ +package org.schabi.newpipe.extractor.streamdata.stream; + +import org.schabi.newpipe.extractor.streamdata.format.VideoAudioMediaFormat; +import org.schabi.newpipe.extractor.streamdata.stream.quality.VideoQualityData; + +import javax.annotation.Nonnull; + +/** + * Represents a video (only) stream. + */ +public interface VideoStream extends Stream { + @Nonnull + VideoQualityData qualityData(); +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/quality/VideoQualityData.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/quality/VideoQualityData.java new file mode 100644 index 0000000000..86bef7ab1f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/quality/VideoQualityData.java @@ -0,0 +1,50 @@ +package org.schabi.newpipe.extractor.streamdata.stream.quality; + +public class VideoQualityData { + public static final int UNKNOWN = -1; + + private final int height; + private final int width; + private final int fps; + + public VideoQualityData(final int height, final int width, final int fps) { + this.height = height; + this.width = width; + this.fps = fps; + } + + + public int height() { + return height; + } + + public int width() { + return width; + } + + public int fps() { + return fps; + } + + public boolean equalsVideoQualityData(final VideoQualityData other) { + return height() == other.height() + && width() == other.width() + && fps() == other.fps(); + } + + public static VideoQualityData fromHeightWidth(final int height, final int width) { + return new VideoQualityData(height, width, UNKNOWN); + } + + public static VideoQualityData fromHeightFps(final int height, final int fps) { + return new VideoQualityData(height, UNKNOWN, fps); + } + + public static VideoQualityData fromHeight(final int height) { + return new VideoQualityData(height, UNKNOWN, UNKNOWN); + } + + public static VideoQualityData fromUnknown() { + return new VideoQualityData(UNKNOWN, UNKNOWN, UNKNOWN); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/simpleimpl/AbstractStreamImpl.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/simpleimpl/AbstractStreamImpl.java new file mode 100644 index 0000000000..72438779f4 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/simpleimpl/AbstractStreamImpl.java @@ -0,0 +1,35 @@ +package org.schabi.newpipe.extractor.streamdata.stream.simpleimpl; + +import org.schabi.newpipe.extractor.streamdata.delivery.DeliveryData; +import org.schabi.newpipe.extractor.streamdata.format.MediaFormat; +import org.schabi.newpipe.extractor.streamdata.stream.Stream; + +import java.util.Objects; + +import javax.annotation.Nonnull; + +public abstract class AbstractStreamImpl implements Stream { + @Nonnull + private final M mediaFormat; + @Nonnull + private final DeliveryData deliveryData; + + protected AbstractStreamImpl( + @Nonnull final M mediaFormat, + @Nonnull final DeliveryData deliveryData) { + this.mediaFormat = Objects.requireNonNull(mediaFormat); + this.deliveryData = Objects.requireNonNull(deliveryData); + } + + @Nonnull + @Override + public M mediaFormat() { + return mediaFormat; + } + + @Nonnull + @Override + public DeliveryData deliveryData() { + return deliveryData; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/simpleimpl/SimpleAudioStreamImpl.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/simpleimpl/SimpleAudioStreamImpl.java new file mode 100644 index 0000000000..c87cd39d90 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/simpleimpl/SimpleAudioStreamImpl.java @@ -0,0 +1,33 @@ +package org.schabi.newpipe.extractor.streamdata.stream.simpleimpl; + +import org.schabi.newpipe.extractor.streamdata.delivery.DeliveryData; +import org.schabi.newpipe.extractor.streamdata.format.AudioMediaFormat; +import org.schabi.newpipe.extractor.streamdata.stream.AudioStream; + +import javax.annotation.Nonnull; + +public class SimpleAudioStreamImpl extends AbstractStreamImpl + implements AudioStream { + private final int averageBitrate; + + public SimpleAudioStreamImpl( + @Nonnull final AudioMediaFormat mediaFormat, + @Nonnull final DeliveryData deliveryData, + final int averageBitrate + ) { + super(mediaFormat, deliveryData); + this.averageBitrate = averageBitrate; + } + + public SimpleAudioStreamImpl( + @Nonnull final AudioMediaFormat mediaFormat, + @Nonnull final DeliveryData deliveryData + ) { + this(mediaFormat, deliveryData, UNKNOWN_AVG_BITRATE); + } + + @Override + public int averageBitrate() { + return averageBitrate; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/simpleimpl/SimpleSubtitleStreamImpl.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/simpleimpl/SimpleSubtitleStreamImpl.java new file mode 100644 index 0000000000..fe102bc18e --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/simpleimpl/SimpleSubtitleStreamImpl.java @@ -0,0 +1,63 @@ +package org.schabi.newpipe.extractor.streamdata.stream.simpleimpl; + +import org.schabi.newpipe.extractor.streamdata.delivery.DeliveryData; +import org.schabi.newpipe.extractor.streamdata.format.SubtitleMediaFormat; +import org.schabi.newpipe.extractor.streamdata.stream.SubtitleStream; + +import java.util.Locale; +import java.util.Objects; + +import javax.annotation.Nonnull; + +public class SimpleSubtitleStreamImpl extends AbstractStreamImpl + implements SubtitleStream { + private final boolean autogenerated; + @Nonnull + private final String languageCode; + private final Locale locale; + + public SimpleSubtitleStreamImpl( + @Nonnull final SubtitleMediaFormat subtitleMediaFormat, + @Nonnull final DeliveryData deliveryData, + final boolean autogenerated, + @Nonnull final String languageCode + ) { + super(subtitleMediaFormat, deliveryData); + this.autogenerated = autogenerated; + this.languageCode = Objects.requireNonNull(languageCode); + /* + * Locale.forLanguageTag only for Android API >= 21 + * Locale.Builder only for Android API >= 21 + * Country codes doesn't work well without + */ + final String[] splits = languageCode.split("-"); + switch (splits.length) { + case 2: + this.locale = new Locale(splits[0], splits[1]); + break; + case 3: + // Complex variants don't work! + this.locale = new Locale(splits[0], splits[1], splits[2]); + break; + default: + this.locale = new Locale(splits[0]); + break; + } + } + + @Override + public boolean autoGenerated() { + return autogenerated; + } + + @Nonnull + @Override + public String languageCode() { + return languageCode; + } + + @Override + public Locale locale() { + return locale; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/simpleimpl/SimpleVideoAudioStreamImpl.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/simpleimpl/SimpleVideoAudioStreamImpl.java new file mode 100644 index 0000000000..7dd0e266df --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/simpleimpl/SimpleVideoAudioStreamImpl.java @@ -0,0 +1,49 @@ +package org.schabi.newpipe.extractor.streamdata.stream.simpleimpl; + +import org.schabi.newpipe.extractor.streamdata.delivery.DeliveryData; +import org.schabi.newpipe.extractor.streamdata.format.VideoAudioMediaFormat; +import org.schabi.newpipe.extractor.streamdata.stream.VideoAudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.quality.VideoQualityData; + +import java.util.Objects; + +import javax.annotation.Nonnull; + +public class SimpleVideoAudioStreamImpl extends AbstractStreamImpl + implements VideoAudioStream { + + @Nonnull + private final VideoQualityData videoQualityData; + + private final int averageBitrate; + + public SimpleVideoAudioStreamImpl( + @Nonnull final VideoAudioMediaFormat mediaFormat, + @Nonnull final DeliveryData deliveryData, + @Nonnull final VideoQualityData videoQualityData, + final int averageBitrate + ) { + super(mediaFormat, deliveryData); + this.videoQualityData = Objects.requireNonNull(videoQualityData); + this.averageBitrate = averageBitrate; + } + + public SimpleVideoAudioStreamImpl( + @Nonnull final VideoAudioMediaFormat mediaFormat, + @Nonnull final DeliveryData deliveryData, + @Nonnull final VideoQualityData videoQualityData + ) { + this(mediaFormat, deliveryData, videoQualityData, UNKNOWN_AVG_BITRATE); + } + + @Nonnull + @Override + public VideoQualityData qualityData() { + return videoQualityData; + } + + @Override + public int averageBitrate() { + return averageBitrate; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/simpleimpl/SimpleVideoStreamImpl.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/simpleimpl/SimpleVideoStreamImpl.java new file mode 100644 index 0000000000..1ea04b58de --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/stream/simpleimpl/SimpleVideoStreamImpl.java @@ -0,0 +1,38 @@ +package org.schabi.newpipe.extractor.streamdata.stream.simpleimpl; + +import org.schabi.newpipe.extractor.streamdata.delivery.DeliveryData; +import org.schabi.newpipe.extractor.streamdata.format.VideoAudioMediaFormat; +import org.schabi.newpipe.extractor.streamdata.stream.VideoStream; +import org.schabi.newpipe.extractor.streamdata.stream.quality.VideoQualityData; + +import java.util.Objects; + +import javax.annotation.Nonnull; + +public class SimpleVideoStreamImpl extends AbstractStreamImpl + implements VideoStream { + @Nonnull + private final VideoQualityData videoQualityData; + + public SimpleVideoStreamImpl( + @Nonnull final VideoAudioMediaFormat mediaFormat, + @Nonnull final DeliveryData deliveryData, + @Nonnull final VideoQualityData videoQualityData + ) { + super(mediaFormat, deliveryData); + this.videoQualityData = Objects.requireNonNull(videoQualityData); + } + + public SimpleVideoStreamImpl( + @Nonnull final VideoAudioMediaFormat mediaFormat, + @Nonnull final DeliveryData deliveryData + ) { + this(mediaFormat, deliveryData, VideoQualityData.fromUnknown()); + } + + @Nonnull + @Override + public VideoQualityData qualityData() { + return videoQualityData; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index 850eba778e..2002fc1b06 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -161,4 +161,9 @@ public static List getStringListFromJsonArray(@Nonnull final JsonArray a .map(String.class::cast) .collect(Collectors.toList()); } + + public static Integer getNullableInteger(@Nonnull final JsonObject jsonObject, + @Nonnull final String key) { + return (Integer) jsonObject.getNumber(key); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCache.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCache.java deleted file mode 100644 index ac12f83f95..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCache.java +++ /dev/null @@ -1,255 +0,0 @@ -package org.schabi.newpipe.extractor.utils; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * A {@link Serializable serializable} cache class used by the extractor to cache manifests - * generated with extractor's manifests generators. - * - *

- * It relies internally on a {@link ConcurrentHashMap} to allow concurrent access to the cache. - *

- * - * @param the type of cache keys, which must be {@link Serializable serializable} - * @param the type of the second element of {@link Pair pairs} used as values of the cache, - * which must be {@link Serializable serializable} - */ -public final class ManifestCreatorCache - implements Serializable { - - /** - * The default maximum size of a manifest cache. - */ - public static final int DEFAULT_MAXIMUM_SIZE = Integer.MAX_VALUE; - - /** - * The default clear factor of a manifest cache. - */ - public static final double DEFAULT_CLEAR_FACTOR = 0.75; - - /** - * The {@link ConcurrentHashMap} used internally as the cache of manifests. - */ - private final ConcurrentHashMap> concurrentHashMap; - - /** - * The maximum size of the cache. - * - *

- * The default value is {@link #DEFAULT_MAXIMUM_SIZE}. - *

- */ - private int maximumSize = DEFAULT_MAXIMUM_SIZE; - - /** - * The clear factor of the cache, which is a double between {@code 0} and {@code 1} excluded. - * - *

- * The default value is {@link #DEFAULT_CLEAR_FACTOR}. - *

- */ - private double clearFactor = DEFAULT_CLEAR_FACTOR; - - /** - * Creates a new {@link ManifestCreatorCache}. - */ - public ManifestCreatorCache() { - concurrentHashMap = new ConcurrentHashMap<>(); - } - - /** - * Tests if the specified key is in the cache. - * - * @param key the key to test its presence in the cache - * @return {@code true} if the key is in the cache, {@code false} otherwise. - */ - public boolean containsKey(final K key) { - return concurrentHashMap.containsKey(key); - } - - /** - * Returns the value to which the specified key is mapped, or {@code null} if the cache - * contains no mapping for the key. - * - * @param key the key to which getting its value - * @return the value to which the specified key is mapped, or {@code null} - */ - @Nullable - public Pair get(final K key) { - return concurrentHashMap.get(key); - } - - /** - * Adds a new element to the cache. - * - *

- * If the cache limit is reached, oldest elements will be cleared first using the load factor - * and the maximum size. - *

- * - * @param key the key to put - * @param value the value to associate to the key - * - * @return the previous value associated with the key, or {@code null} if there was no mapping - * for the key (note that a null return can also indicate that the cache previously associated - * {@code null} with the key). - */ - @Nullable - public V put(final K key, final V value) { - if (!concurrentHashMap.containsKey(key) && concurrentHashMap.size() == maximumSize) { - final int newCacheSize = (int) Math.round(maximumSize * clearFactor); - keepNewestEntries(newCacheSize != 0 ? newCacheSize : 1); - } - - final Pair returnValue = concurrentHashMap.put(key, - new Pair<>(concurrentHashMap.size(), value)); - return returnValue == null ? null : returnValue.getSecond(); - } - - /** - * Clears the cached manifests. - * - *

- * The cache will be empty after this method is called. - *

- */ - public void clear() { - concurrentHashMap.clear(); - } - - /** - * Resets the cache. - * - *

- * The cache will be empty and the clear factor and the maximum size will be reset to their - * default values. - *

- * - * @see #clear() - * @see #resetClearFactor() - * @see #resetMaximumSize() - */ - public void reset() { - clear(); - resetClearFactor(); - resetMaximumSize(); - } - - /** - * @return the number of cached manifests in the cache - */ - public int size() { - return concurrentHashMap.size(); - } - - /** - * @return the maximum size of the cache - */ - public long getMaximumSize() { - return maximumSize; - } - - /** - * Sets the maximum size of the cache. - * - * If the current cache size is more than the new maximum size, the percentage of one less the - * clear factor of the maximum new size of manifests in the cache will be removed. - * - * @param maximumSize the new maximum size of the cache - * @throws IllegalArgumentException if {@code maximumSize} is less than or equal to 0 - */ - public void setMaximumSize(final int maximumSize) { - if (maximumSize <= 0) { - throw new IllegalArgumentException("Invalid maximum size"); - } - - if (maximumSize < this.maximumSize && !concurrentHashMap.isEmpty()) { - final int newCacheSize = (int) Math.round(maximumSize * clearFactor); - keepNewestEntries(newCacheSize != 0 ? newCacheSize : 1); - } - - this.maximumSize = maximumSize; - } - - /** - * Resets the maximum size of the cache to its {@link #DEFAULT_MAXIMUM_SIZE default value}. - */ - public void resetMaximumSize() { - this.maximumSize = DEFAULT_MAXIMUM_SIZE; - } - - /** - * @return the current clear factor of the cache, used when the cache limit size is reached - */ - public double getClearFactor() { - return clearFactor; - } - - /** - * Sets the clear factor of the cache, used when the cache limit size is reached. - * - *

- * The clear factor must be a double between {@code 0} excluded and {@code 1} excluded. - *

- * - *

- * Note that it will be only used the next time the cache size limit is reached. - *

- * - * @param clearFactor the new clear factor of the cache - * @throws IllegalArgumentException if the clear factor passed a parameter is invalid - */ - public void setClearFactor(final double clearFactor) { - if (clearFactor <= 0 || clearFactor >= 1) { - throw new IllegalArgumentException("Invalid clear factor"); - } - - this.clearFactor = clearFactor; - } - - /** - * Resets the clear factor to its {@link #DEFAULT_CLEAR_FACTOR default value}. - */ - public void resetClearFactor() { - this.clearFactor = DEFAULT_CLEAR_FACTOR; - } - - @Nonnull - @Override - public String toString() { - return "ManifestCreatorCache[clearFactor=" + clearFactor + ", maximumSize=" + maximumSize - + ", concurrentHashMap=" + concurrentHashMap + "]"; - } - - /** - * Keeps only the newest entries in a cache. - * - *

- * This method will first collect the entries to remove by looping through the concurrent hash - * map - *

- * - * @param newLimit the new limit of the cache - */ - private void keepNewestEntries(final int newLimit) { - final int difference = concurrentHashMap.size() - newLimit; - final ArrayList>> entriesToRemove = new ArrayList<>(); - - concurrentHashMap.entrySet().forEach(entry -> { - final Pair value = entry.getValue(); - if (value.getFirst() < difference) { - entriesToRemove.add(entry); - } else { - value.setFirst(value.getFirst() - difference); - } - }); - - entriesToRemove.forEach(entry -> concurrentHashMap.remove(entry.getKey(), - entry.getValue())); - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Pair.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Pair.java index 6efb66b209..943f9ddac2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Pair.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Pair.java @@ -1,25 +1,20 @@ package org.schabi.newpipe.extractor.utils; -import java.io.Serializable; import java.util.Objects; /** - * Serializable class to create a pair of objects. + * Class to create a pair of objects. * - *

- * The two objects of the pair must be {@link Serializable serializable} and can be of the same - * type. - *

* *

* Note that this class is not intended to be used as a general-purpose pair and should only be * used when interfacing with the extractor. *

* - * @param the type of the first object, which must be {@link Serializable} - * @param the type of the second object, which must be {@link Serializable} + * @param the type of the first object + * @param the type of the second object */ -public class Pair implements Serializable { +public class Pair { /** * The first object of the pair. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java index 64707cd3a1..c3e3ace294 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java @@ -93,13 +93,16 @@ public static boolean isMatch(@Nonnull final Pattern pattern, final String input } @Nonnull - public static Map compatParseMap(@Nonnull final String input) - throws UnsupportedEncodingException { + public static Map compatParseMap(@Nonnull final String input) { final Map map = new HashMap<>(); for (final String arg : input.split("&")) { final String[] splitArg = arg.split("="); if (splitArg.length > 1) { - map.put(splitArg[0], URLDecoder.decode(splitArg[1], UTF_8)); + try { + map.put(splitArg[0], URLDecoder.decode(splitArg[1], UTF_8)); + } catch (final UnsupportedEncodingException ignored) { + // No UTF-8 = Not possible + } } else { map.put(splitArg[0], ""); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java index da418660a1..5d94777c91 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java @@ -1,7 +1,6 @@ package org.schabi.newpipe.extractor.services; public interface BaseStreamExtractorTest extends BaseExtractorTest { - void testStreamType() throws Exception; void testUploaderName() throws Exception; void testUploaderUrl() throws Exception; void testUploaderAvatarUrl() throws Exception; @@ -22,7 +21,8 @@ public interface BaseStreamExtractorTest extends BaseExtractorTest { void testAgeLimit() throws Exception; void testErrorMessage() throws Exception; void testAudioStreams() throws Exception; - void testVideoStreams() throws Exception; + void testVideoOnlyStreams() throws Exception; + void testVideoAudioStreams() throws Exception; void testSubtitles() throws Exception; void testGetDashMpdUrl() throws Exception; void testFrames() throws Exception; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java index d9b4e6cde2..24a74d5611 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java @@ -1,40 +1,44 @@ package org.schabi.newpipe.extractor.services; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEqualsOrderIndependent; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertGreaterOrEqual; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestListOfItems; +import static org.schabi.newpipe.extractor.stream.StreamExtractor.UNKNOWN_SUBSCRIBER_COUNT; + import org.junit.jupiter.api.Test; import org.schabi.newpipe.extractor.ExtractorAsserts; import org.schabi.newpipe.extractor.InfoItemsCollector; -import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.Frameset; +import org.schabi.newpipe.extractor.stream.Privacy; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.stream.SubtitlesStream; -import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.streamdata.delivery.DASHManifestDeliveryData; +import org.schabi.newpipe.extractor.streamdata.delivery.DeliveryData; +import org.schabi.newpipe.extractor.streamdata.delivery.UrlBasedDeliveryData; +import org.schabi.newpipe.extractor.streamdata.stream.AudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.SubtitleStream; +import org.schabi.newpipe.extractor.streamdata.stream.VideoAudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.VideoStream; -import javax.annotation.Nullable; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; import java.net.MalformedURLException; import java.net.URL; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.stream.Collectors; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertGreaterOrEqual; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEqualsOrderIndependent; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl; -import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestListOfItems; -import static org.schabi.newpipe.extractor.stream.StreamExtractor.UNKNOWN_SUBSCRIBER_COUNT; +import javax.annotation.Nullable; /** * Test for {@link StreamExtractor} @@ -42,7 +46,8 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest implements BaseStreamExtractorTest { - public abstract StreamType expectedStreamType(); + public boolean expectedIsLive() { return false; } + public boolean expectedIsAudioOnly() { return false; } public abstract String expectedUploaderName(); public abstract String expectedUploaderUrl(); public boolean expectedUploaderVerified() { return false; } @@ -61,13 +66,14 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest expectedMetaInfo() throws MalformedURLException { return Collections.emptyList(); } // default: no metadata info available - @Test - @Override - public void testStreamType() throws Exception { - assertEquals(expectedStreamType(), extractor().getStreamType()); - } - @Test @Override public void testUploaderName() throws Exception { @@ -193,12 +193,13 @@ public void testViewCount() throws Exception { public void testUploadDate() throws Exception { final DateWrapper dateWrapper = extractor().getUploadDate(); - if (expectedUploadDate() == null) { + final String expectedUploadDate = expectedUploadDate(); + if (expectedUploadDate == null) { assertNull(dateWrapper); } else { assertNotNull(dateWrapper); - final LocalDateTime expectedDateTime = LocalDateTime.parse(expectedUploadDate(), + final LocalDateTime expectedDateTime = LocalDateTime.parse(expectedUploadDate, DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")); final LocalDateTime actualDateTime = dateWrapper.offsetDateTime().toLocalDateTime(); @@ -260,34 +261,39 @@ public void testErrorMessage() throws Exception { @Test @Override - public void testVideoStreams() throws Exception { - final List videoStreams = extractor().getVideoStreams(); + public void testVideoOnlyStreams() throws Exception { final List videoOnlyStreams = extractor().getVideoOnlyStreams(); - assertNotNull(videoStreams); assertNotNull(videoOnlyStreams); - videoStreams.addAll(videoOnlyStreams); - if (expectedHasVideoStreams()) { - assertFalse(videoStreams.isEmpty()); + if (expectedHasVideoOnlyStreams()) { + assertFalse(videoOnlyStreams.isEmpty()); - for (final VideoStream stream : videoStreams) { - if (stream.isUrl()) { - assertIsSecureUrl(stream.getContent()); - } - final StreamType streamType = extractor().getStreamType(); - // On some video streams, the resolution can be empty and the format be unknown, - // especially on livestreams (like streams with HLS master playlists) - if (streamType != StreamType.LIVE_STREAM - && streamType != StreamType.AUDIO_LIVE_STREAM) { - assertFalse(stream.getResolution().isEmpty()); - final int formatId = stream.getFormatId(); - // see MediaFormat: video stream formats range from 0 to 0x100 - assertTrue(0 <= formatId && formatId < 0x100, - "Format id does not fit a video stream: " + formatId); - } + for (final VideoStream stream : videoOnlyStreams) { + assertNotNull(stream.mediaFormat()); + assertNotNull(stream.qualityData()); + checkDeliveryData(stream.deliveryData()); + } + } else { + assertTrue(videoOnlyStreams.isEmpty()); + } + } + + @Test + @Override + public void testVideoAudioStreams() throws Exception { + final List videoAudioStreams = extractor().getVideoStreams(); + assertNotNull(videoAudioStreams); + + if (expectedHasVideoAndAudioStreams()) { + assertFalse(videoAudioStreams.isEmpty()); + + for (final VideoAudioStream stream : videoAudioStreams) { + assertNotNull(stream.mediaFormat()); + assertNotNull(stream.qualityData()); + checkDeliveryData(stream.deliveryData()); } } else { - assertTrue(videoStreams.isEmpty()); + assertTrue(videoAudioStreams.isEmpty()); } } @@ -301,17 +307,8 @@ public void testAudioStreams() throws Exception { assertFalse(audioStreams.isEmpty()); for (final AudioStream stream : audioStreams) { - if (stream.isUrl()) { - assertIsSecureUrl(stream.getContent()); - } - - // The media format can be unknown on some audio streams - if (stream.getFormat() != null) { - final int formatId = stream.getFormat().id; - // see MediaFormat: audio stream formats range from 0x100 to 0x1000 - assertTrue(0x100 <= formatId && formatId < 0x1000, - "Format id does not fit an audio stream: " + formatId); - } + assertNotNull(stream.mediaFormat()); + checkDeliveryData(stream.deliveryData()); } } else { assertTrue(audioStreams.isEmpty()); @@ -321,32 +318,29 @@ public void testAudioStreams() throws Exception { @Test @Override public void testSubtitles() throws Exception { - final List subtitles = extractor().getSubtitlesDefault(); + final List subtitles = extractor().getSubtitles(); assertNotNull(subtitles); if (expectedHasSubtitles()) { assertFalse(subtitles.isEmpty()); - for (final SubtitlesStream stream : subtitles) { - if (stream.isUrl()) { - assertIsSecureUrl(stream.getContent()); - } - - final int formatId = stream.getFormatId(); - // see MediaFormat: video stream formats range from 0x1000 to 0x10000 - assertTrue(0x1000 <= formatId && formatId < 0x10000, - "Format id does not fit a subtitles stream: " + formatId); + for (final SubtitleStream stream : subtitles) { + assertNotNull(stream.languageCode()); + assertNotNull(stream.mediaFormat()); + checkDeliveryData(stream.deliveryData()); } } else { assertTrue(subtitles.isEmpty()); + } + } - final MediaFormat[] formats = {MediaFormat.VTT, MediaFormat.TTML, MediaFormat.SRT, - MediaFormat.TRANSCRIPT1, MediaFormat.TRANSCRIPT2, MediaFormat.TRANSCRIPT3}; - for (final MediaFormat format : formats) { - final List formatSubtitles = extractor().getSubtitles(format); - assertNotNull(formatSubtitles); - assertTrue(formatSubtitles.isEmpty()); - } + private void checkDeliveryData(final DeliveryData deliveryData) { + if (deliveryData instanceof UrlBasedDeliveryData) { + assertIsSecureUrl(((UrlBasedDeliveryData) deliveryData).url()); + } else if (deliveryData instanceof DASHManifestDeliveryData) { + final DASHManifestDeliveryData dashManifestDD = + (DASHManifestDeliveryData) deliveryData; + assertNotNull(dashManifestDD.dashManifestCreator()); } } @@ -461,6 +455,15 @@ public void testMetaInfo() throws Exception { assertTrue(urls.contains(expectedUrl)); } } + } + @Test + public void testIsLive() throws Exception { + assertEquals(expectedIsLive(), extractor().isLive()); + } + + @Test + public void testIsAudioOnly() throws Exception { + assertEquals(expectedIsAudioOnly(), extractor().isAudioOnly()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java index 01ab77bfef..fbd921de0a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java @@ -1,5 +1,11 @@ package org.schabi.newpipe.extractor.services.bandcamp; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; + import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.schabi.newpipe.downloader.DownloaderTestImpl; @@ -11,7 +17,6 @@ import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioStreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; import java.io.IOException; import java.util.Calendar; @@ -19,9 +24,6 @@ import java.util.List; import java.util.TimeZone; -import static org.junit.jupiter.api.Assertions.*; -import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; - public class BandcampRadioStreamExtractorTest extends DefaultStreamExtractorTest { private static StreamExtractor extractor; @@ -47,11 +49,12 @@ public void testGettingCorrectStreamExtractor() throws ExtractionException { @Override public String expectedId() throws Exception { return "230"; } @Override public String expectedUrlContains() throws Exception { return URL; } @Override public String expectedOriginalUrlContains() throws Exception { return URL; } - @Override public boolean expectedHasVideoStreams() { return false; } + @Override public boolean expectedHasVideoOnlyStreams() { return false; } + @Override public boolean expectedHasVideoAndAudioStreams() { return false; } @Override public boolean expectedHasSubtitles() { return false; } @Override public boolean expectedHasFrames() { return false; } @Override public boolean expectedHasRelatedItems() { return false; } - @Override public StreamType expectedStreamType() { return StreamType.AUDIO_STREAM; } + @Override public boolean expectedIsAudioOnly() { return true; } @Override public StreamingService expectedService() { return Bandcamp; } @Override public String expectedUploaderName() { return "Andrew Jervis"; } @Override public int expectedStreamSegmentsCount() { return 30; } @@ -80,6 +83,7 @@ public List expectedDescriptionContains() { @Override public String expectedUploadDate() { return "16 May 2017 00:00:00 GMT"; } @Override public String expectedTextualUploadDate() { return "16 May 2017 00:00:00 GMT"; } + @Override @Test public void testUploadDate() throws ParsingException { final Calendar expectedCalendar = Calendar.getInstance(); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java index 3a582db287..94ef94e48c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -13,7 +13,6 @@ import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; import java.io.IOException; import java.util.Collections; @@ -70,8 +69,8 @@ public String expectedOriginalUrlContains() { } @Override - public StreamType expectedStreamType() { - return StreamType.AUDIO_STREAM; + public boolean expectedIsAudioOnly() { + return true; } @Override @@ -120,7 +119,12 @@ public long expectedDislikeCountAtLeast() { } @Override - public boolean expectedHasVideoStreams() { + public boolean expectedHasVideoOnlyStreams() { + return false; + } + + @Override + public boolean expectedHasVideoAndAudioStreams() { return false; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java index ed8d2ef16e..00032cc5be 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java @@ -5,8 +5,8 @@ import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; -import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.streamdata.stream.AudioStream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; @@ -34,7 +34,7 @@ public void getAudioStreamsCount() throws Exception { @Test public void getAudioStreamsContainOgg() throws Exception { for (AudioStream stream : extractor.getAudioStreams()) { - assertEquals("OGG", stream.getFormat().toString()); + assertEquals("ogg", stream.mediaFormat().name()); } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java index 458946ba3e..eb7e6fc263 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -8,7 +8,6 @@ import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; import javax.annotation.Nullable; @@ -43,7 +42,6 @@ public static void setUp() throws Exception { @Override public String expectedId() { return ID; } @Override public String expectedUrlContains() { return URL; } @Override public String expectedOriginalUrlContains() { return URL; } - @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "gpn18"; } @Override public String expectedUploaderUrl() { return "https://media.ccc.de/c/gpn18"; } @Override public List expectedDescriptionContains() { return Arrays.asList("SSH-Sessions", "\"Terminal Multiplexer\""); } @@ -54,6 +52,7 @@ public static void setUp() throws Exception { @Override public long expectedLikeCountAtLeast() { return -1; } @Override public long expectedDislikeCountAtLeast() { return -1; } @Override public boolean expectedHasRelatedItems() { return false; } + @Override public boolean expectedHasVideoOnlyStreams() { return false; } @Override public boolean expectedHasSubtitles() { return false; } @Override public boolean expectedHasFrames() { return false; } @Override public List expectedTags() { return Arrays.asList("gpn18", "105"); } @@ -76,8 +75,8 @@ public void testUploaderAvatarUrl() throws Exception { @Override @Test - public void testVideoStreams() throws Exception { - super.testVideoStreams(); + public void testVideoAudioStreams() throws Exception { + super.testVideoAudioStreams(); assertEquals(4, extractor.getVideoStreams().size()); } @@ -115,7 +114,6 @@ public static void setUp() throws Exception { } @Override public String expectedUrlContains() { return URL; } @Override public String expectedOriginalUrlContains() { return URL; } - @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "36c3"; } @Override public String expectedUploaderUrl() { return "https://media.ccc.de/c/36c3"; } @Override public List expectedDescriptionContains() { return Arrays.asList("WhatsApp", "Signal"); } @@ -126,6 +124,7 @@ public static void setUp() throws Exception { @Override public long expectedLikeCountAtLeast() { return -1; } @Override public long expectedDislikeCountAtLeast() { return -1; } @Override public boolean expectedHasRelatedItems() { return false; } + @Override public boolean expectedHasVideoOnlyStreams() { return false; } @Override public boolean expectedHasSubtitles() { return false; } @Override public boolean expectedHasFrames() { return false; } @Override public List expectedTags() { return Arrays.asList("36c3", "10565", "2019", "Security", "Main"); } @@ -146,8 +145,8 @@ public void testUploaderAvatarUrl() throws Exception { @Override @Test - public void testVideoStreams() throws Exception { - super.testVideoStreams(); + public void testVideoAudioStreams() throws Exception { + super.testVideoAudioStreams(); assertEquals(8, extractor.getVideoStreams().size()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java index 63a7839fbf..39578f6597 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java @@ -10,7 +10,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; import java.io.IOException; import java.util.Arrays; @@ -25,6 +24,7 @@ public abstract class PeertubeStreamExtractorTest extends DefaultStreamExtractorTest { private static final String BASE_URL = "/videos/watch/"; + @Override public boolean expectedHasVideoOnlyStreams() { return false; } @Override public boolean expectedHasAudioStreams() { return false; } @Override public boolean expectedHasFrames() { return false; } @@ -57,7 +57,6 @@ public void testGetLanguageInformation() throws ParsingException { @Override public String expectedUrlContains() { return INSTANCE + BASE_URL + ID; } @Override public String expectedOriginalUrlContains() { return URL; } - @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "Framasoft"; } @Override public String expectedUploaderUrl() { return "https://framatube.org/accounts/framasoft@framatube.org"; } @Override public String expectedSubChannelName() { return "A propos de PeerTube"; } @@ -118,7 +117,6 @@ public static void setUp() throws Exception { @Override public String expectedUrlContains() { return INSTANCE + BASE_URL + ID; } @Override public String expectedOriginalUrlContains() { return URL; } - @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "Marinauts"; } @Override public String expectedUploaderUrl() { return "https://tilvids.com/accounts/marinauts@tilvids.com"; } @Override public String expectedSubChannelName() { return "Main marinauts channel"; } @@ -163,7 +161,6 @@ public static void setUp() throws Exception { @Override public String expectedUrlContains() { return INSTANCE + BASE_URL + ID; } @Override public String expectedOriginalUrlContains() { return URL; } - @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "Résilience humaine"; } @Override public String expectedUploaderUrl() { return "https://nocensoring.net/accounts/gmt@nocensoring.net"; } @Override public String expectedSubChannelName() { return "SYSTEM FAILURE Quel à-venir ?"; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java index da2c0511dd..63835ca469 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java @@ -1,20 +1,24 @@ package org.schabi.newpipe.extractor.services.soundcloud; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; + import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.ExtractorAsserts; -import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException; import org.schabi.newpipe.extractor.exceptions.SoundCloudGoPlusContentException; import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; -import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.streamdata.delivery.ProgressiveHTTPDeliveryData; +import org.schabi.newpipe.extractor.streamdata.delivery.UrlBasedDeliveryData; +import org.schabi.newpipe.extractor.streamdata.format.registry.AudioFormatRegistry; +import org.schabi.newpipe.extractor.streamdata.stream.AudioStream; import java.util.Arrays; import java.util.Collections; @@ -22,9 +26,6 @@ import javax.annotation.Nullable; -import static org.junit.jupiter.api.Assertions.*; -import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; - public class SoundcloudStreamExtractorTest { private static final String SOUNDCLOUD = "https://soundcloud.com/"; @@ -53,7 +54,7 @@ public static void setUp() throws Exception { @Override public String expectedUrlContains() { return UPLOADER + "/" + ID; } @Override public String expectedOriginalUrlContains() { return URL; } - @Override public StreamType expectedStreamType() { return StreamType.AUDIO_STREAM; } + @Override public boolean expectedIsAudioOnly() { return true; } @Override public String expectedUploaderName() { return "Jess Glynne"; } @Override public String expectedUploaderUrl() { return UPLOADER; } @Override public boolean expectedUploaderVerified() { return true; } @@ -67,7 +68,8 @@ public static void setUp() throws Exception { @Override public long expectedLikeCountAtLeast() { return -1; } @Override public long expectedDislikeCountAtLeast() { return -1; } @Override public boolean expectedHasAudioStreams() { return false; } - @Override public boolean expectedHasVideoStreams() { return false; } + @Override public boolean expectedHasVideoOnlyStreams() { return false; } + @Override public boolean expectedHasVideoAndAudioStreams() { return false; } @Override public boolean expectedHasSubtitles() { return false; } @Override public boolean expectedHasFrames() { return false; } @Override public int expectedStreamSegmentsCount() { return 0; } @@ -116,7 +118,7 @@ public void testRelatedItems() throws Exception { @Override public String expectedUrlContains() { return UPLOADER + "/" + ID; } @Override public String expectedOriginalUrlContains() { return URL; } - @Override public StreamType expectedStreamType() { return StreamType.AUDIO_STREAM; } + @Override public boolean expectedIsAudioOnly() { return true; } @Override public String expectedUploaderName() { return "martinsolveig"; } @Override public String expectedUploaderUrl() { return UPLOADER; } @Override public boolean expectedUploaderVerified() { return true; } @@ -130,7 +132,8 @@ public void testRelatedItems() throws Exception { @Override public long expectedLikeCountAtLeast() { return -1; } @Override public long expectedDislikeCountAtLeast() { return -1; } @Override public boolean expectedHasAudioStreams() { return false; } - @Override public boolean expectedHasVideoStreams() { return false; } + @Override public boolean expectedHasVideoOnlyStreams() { return false; } + @Override public boolean expectedHasVideoAndAudioStreams() { return false; } @Override public boolean expectedHasRelatedItems() { return true; } @Override public boolean expectedHasSubtitles() { return false; } @Override public boolean expectedHasFrames() { return false; } @@ -160,7 +163,7 @@ public static void setUp() throws Exception { @Override public String expectedUrlContains() { return UPLOADER + "/" + ID; } @Override public String expectedOriginalUrlContains() { return URL; } - @Override public StreamType expectedStreamType() { return StreamType.AUDIO_STREAM; } + @Override public boolean expectedIsAudioOnly() { return true; } @Override public String expectedUploaderName() { return "Creative Commons"; } @Override public String expectedUploaderUrl() { return UPLOADER; } @Override public List expectedDescriptionContains() { return Arrays.asList("Stigmergy is a mechanism of indirect coordination", @@ -172,7 +175,8 @@ public static void setUp() throws Exception { @Nullable @Override public String expectedTextualUploadDate() { return "2019-03-28 13:36:18"; } @Override public long expectedLikeCountAtLeast() { return -1; } @Override public long expectedDislikeCountAtLeast() { return -1; } - @Override public boolean expectedHasVideoStreams() { return false; } + @Override public boolean expectedHasVideoOnlyStreams() { return false; } + @Override public boolean expectedHasVideoAndAudioStreams() { return false; } @Override public boolean expectedHasSubtitles() { return false; } @Override public boolean expectedHasFrames() { return false; } @Override public int expectedStreamSegmentsCount() { return 0; } @@ -186,29 +190,23 @@ public static void setUp() throws Exception { @Test public void testAudioStreams() throws Exception { super.testAudioStreams(); + final List audioStreams = extractor.getAudioStreams(); - assertEquals(2, audioStreams.size()); - audioStreams.forEach(audioStream -> { - final DeliveryMethod deliveryMethod = audioStream.getDeliveryMethod(); - final String mediaUrl = audioStream.getContent(); - if (audioStream.getFormat() == MediaFormat.OPUS) { - // Assert that it's an OPUS 64 kbps media URL with a single range which comes - // from an HLS SoundCloud CDN - ExtractorAsserts.assertContains("-hls-opus-media.sndcdn.com", mediaUrl); - ExtractorAsserts.assertContains(".64.opus", mediaUrl); - assertSame(DeliveryMethod.HLS, deliveryMethod, - "Wrong delivery method for stream " + audioStream.getId() + ": " - + deliveryMethod); - } else if (audioStream.getFormat() == MediaFormat.MP3) { - // Assert that it's a MP3 128 kbps media URL which comes from a progressive - // SoundCloud CDN - ExtractorAsserts.assertContains("-media.sndcdn.com/bKOA7Pwbut93.128.mp3", - mediaUrl); - assertSame(DeliveryMethod.PROGRESSIVE_HTTP, deliveryMethod, - "Wrong delivery method for stream " + audioStream.getId() + ": " - + deliveryMethod); - } - }); + assertEquals(3, audioStreams.size()); + + for (final AudioStream audioStream : audioStreams) { + assertTrue(audioStream.deliveryData() instanceof UrlBasedDeliveryData, + "Wrong delivery method for mediaFormat=" + audioStream.mediaFormat() + + " , avgBR=" + audioStream.averageBitrate() + + " , deliverDataType=" + audioStream.deliveryData().getClass() + ); + + final UrlBasedDeliveryData deliveryData = + (UrlBasedDeliveryData) audioStream.deliveryData(); + + final String mediaUrl = deliveryData.url(); + ExtractorAsserts.assertContains("-media.sndcdn.com", mediaUrl); + } } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java index 533ad7ee94..5c01bc5452 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java @@ -1,12 +1,18 @@ package org.schabi.newpipe.extractor.services.youtube; -import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.function.Executable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.schabi.newpipe.downloader.DownloaderFactory; -import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.localization.DateWrapper; @@ -15,129 +21,112 @@ import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; -import java.util.LinkedHashMap; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Stream; /** * A class that tests multiple channels and ranges of "time ago". */ -public class YoutubeChannelLocalizationTest { - private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/channel/"; - private static final boolean DEBUG = false; - private final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm"); - - @Test - public void testAllSupportedLocalizations() throws Exception { +@Disabled("There is currently only one localization supported for YT") +class YoutubeChannelLocalizationTest { + private static final String RESOURCE_PATH = + DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/channel/"; + private static final DateTimeFormatter DTF = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm"); + private static final List CHANNEL_URLS = Arrays.asList( + "https://www.youtube.com/user/NBCNews", + "https://www.youtube.com/channel/UCcmpeVbSSQlZRvHfdC-CRwg/videos", + "https://www.youtube.com/channel/UC65afEgL62PGFWXY7n6CUbA", + "https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg"); + + private static final Map> REFERENCES = new HashMap<>(); + + + @BeforeAll + static void setUp() throws Exception { YoutubeTestsUtils.ensureStateless(); NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "localization")); - testLocalizationsFor("https://www.youtube.com/user/NBCNews"); - testLocalizationsFor("https://www.youtube.com/channel/UCcmpeVbSSQlZRvHfdC-CRwg/videos"); - testLocalizationsFor("https://www.youtube.com/channel/UC65afEgL62PGFWXY7n6CUbA"); - testLocalizationsFor("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg"); + for (final String url : CHANNEL_URLS) { + REFERENCES.put(url, getItemsPage(url, Localization.DEFAULT)); + } } - private void testLocalizationsFor(final String channelUrl) throws Exception { - - final List supportedLocalizations = YouTube.getSupportedLocalizations(); - // final List supportedLocalizations = Arrays.asList(Localization.DEFAULT, new Localization("sr")); - final Map> results = new LinkedHashMap<>(); - - for (Localization currentLocalization : supportedLocalizations) { - if (DEBUG) System.out.println("Testing localization = " + currentLocalization); - - ListExtractor.InfoItemsPage itemsPage; - try { - final ChannelExtractor extractor = YouTube.getChannelExtractor(channelUrl); - extractor.forceLocalization(currentLocalization); - extractor.fetchPage(); - itemsPage = defaultTestRelatedItems(extractor); - } catch (final Throwable e) { - System.out.println("[!] " + currentLocalization + " → failed"); - throw e; - } - - final List items = itemsPage.getItems(); - for (int i = 0; i < items.size(); i++) { - final StreamInfoItem item = items.get(i); - - String debugMessage = "[" + String.format("%02d", i) + "] " - + currentLocalization.getLocalizationCode() + " → " + item.getName() - + "\n:::: " + item.getStreamType() + ", views = " + item.getViewCount(); - final DateWrapper uploadDate = item.getUploadDate(); - if (uploadDate != null) { - String dateAsText = dateTimeFormatter.format(uploadDate.offsetDateTime()); - debugMessage += "\n:::: " + item.getTextualUploadDate() + - "\n:::: " + dateAsText; - } - if (DEBUG) System.out.println(debugMessage + "\n"); - } - results.put(currentLocalization, itemsPage.getItems()); - - if (DEBUG) System.out.println("\n===============================\n"); - } + static Stream provideDataForSupportedLocalizations() { + final List localizations = + new ArrayList<>(YouTube.getSupportedLocalizations()); + // Will already be checked in the references + localizations.remove(Localization.DEFAULT); + return CHANNEL_URLS.stream() + .flatMap(url -> localizations.stream().map(l -> Arguments.of(url, l))); + } - // Check results - final List referenceList = results.get(Localization.DEFAULT); - boolean someFail = false; - - for (Map.Entry> currentResultEntry : results.entrySet()) { - if (currentResultEntry.getKey().equals(Localization.DEFAULT)) { - continue; - } - - final String currentLocalizationCode = currentResultEntry.getKey().getLocalizationCode(); - final String referenceLocalizationCode = Localization.DEFAULT.getLocalizationCode(); - if (DEBUG) { - System.out.println("Comparing " + referenceLocalizationCode + " with " + - currentLocalizationCode); - } - - final List currentList = currentResultEntry.getValue(); - if (referenceList.size() != currentList.size()) { - if (DEBUG) System.out.println("[!] " + currentLocalizationCode + " → Lists are not equal"); - someFail = true; - continue; - } - - for (int i = 0; i < referenceList.size() - 1; i++) { - final StreamInfoItem referenceItem = referenceList.get(i); - final StreamInfoItem currentItem = currentList.get(i); - - final DateWrapper referenceUploadDate = referenceItem.getUploadDate(); - final DateWrapper currentUploadDate = currentItem.getUploadDate(); - - final String referenceDateString = referenceUploadDate == null ? "null" : - dateTimeFormatter.format(referenceUploadDate.offsetDateTime()); - final String currentDateString = currentUploadDate == null ? "null" : - dateTimeFormatter.format(currentUploadDate.offsetDateTime()); - - long difference = -1; - if (referenceUploadDate != null && currentUploadDate != null) { - difference = ChronoUnit.MILLIS.between(referenceUploadDate.offsetDateTime(), currentUploadDate.offsetDateTime()); - } - - final boolean areTimeEquals = difference < 5 * 60 * 1000L; - - if (!areTimeEquals) { - System.out.println("" + - " [!] " + currentLocalizationCode + " → [" + i + "] dates are not equal\n" + - " " + referenceLocalizationCode + ": " + - referenceDateString + " → " + referenceItem.getTextualUploadDate() + - "\n " + currentLocalizationCode + ": " + - currentDateString + " → " + currentItem.getTextualUploadDate()); - } - - } - } + @ParameterizedTest + @MethodSource("provideDataForSupportedLocalizations") + void testSupportedLocalizations( + final String channelUrl, + final Localization localization + ) throws Exception { + final List currentItems = getItemsPage(channelUrl, localization); + + final List refItems = REFERENCES.get(channelUrl); + + assertAll( + Stream.concat( + // Check if the lists match + Stream.of(() -> assertEquals( + refItems.size(), + currentItems.size(), + "Number of returned items doesn't match reference list")), + // Check all items + refItems.stream() + .map(refItem -> { + final StreamInfoItem curItem = + currentItems.get(refItems.indexOf(refItem)); + return checkItemAgainstReference(refItem, curItem); + }) + ) + ); + } - if (someFail) { - fail("Some localization failed"); - } else { - if (DEBUG) System.out.print("All tests passed" + - "\n\n===============================\n\n"); - } + private Executable checkItemAgainstReference( + final StreamInfoItem refItem, + final StreamInfoItem curItem + ) { + final DateWrapper refUploadDate = refItem.getUploadDate(); + final DateWrapper curUploadDate = curItem.getUploadDate(); + + final long difference = + refUploadDate == null || curUploadDate == null + ? -1 + : ChronoUnit.MINUTES.between( + refUploadDate.offsetDateTime(), + curUploadDate.offsetDateTime()); + return () -> assertTrue( + difference < 5, + () -> { + final String refDateStr = refUploadDate == null + ? "null" + : DTF.format(refUploadDate.offsetDateTime()); + final String curDateStr = curUploadDate == null + ? "null" + : DTF.format(curUploadDate.offsetDateTime()); + + return "Difference between reference '" + refDateStr + + "' and current '" + curDateStr + "' is too great"; + }); + } + + private static List getItemsPage( + final String channelUrl, + final Localization localization) throws Exception { + final ChannelExtractor extractor = YouTube.getChannelExtractor(channelUrl); + extractor.forceLocalization(localization); + extractor.fetchPage(); + return defaultTestRelatedItems(extractor).getItems(); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java index 0d276f9013..a5f79d59e6 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java @@ -1,53 +1,60 @@ package org.schabi.newpipe.extractor.services.youtube; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertGreater; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertNotBlank; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.ADAPTATION_SET; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.AUDIO_CHANNEL_CONFIGURATION; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.BASE_URL; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.INITIALIZATION; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.MPD; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.PERIOD; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.REPRESENTATION; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.ROLE; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.SEGMENT_BASE; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.SEGMENT_TEMPLATE; +import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.SEGMENT_TIMELINE; +import static org.schabi.newpipe.extractor.utils.Utils.isBlank; + import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.function.Executable; import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.CreationException; -import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeOtfDashManifestCreator; -import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeProgressiveDashManifestCreator; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeOtfDashManifestCreator; +import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeProgressiveDashManifestCreator; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; -import org.schabi.newpipe.extractor.stream.DeliveryMethod; -import org.schabi.newpipe.extractor.stream.Stream; +import org.schabi.newpipe.extractor.streamdata.delivery.DASHManifestDeliveryData; +import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreator; +import org.schabi.newpipe.extractor.streamdata.stream.AudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.BaseAudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.Stream; +import org.schabi.newpipe.extractor.streamdata.stream.VideoAudioStream; +import org.schabi.newpipe.extractor.streamdata.stream.VideoStream; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NodeList; import org.xml.sax.InputSource; -import javax.annotation.Nonnull; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import java.io.StringReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; -import java.util.Random; -import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.IntStream; -import static org.junit.jupiter.api.Assertions.assertAll; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertGreater; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertGreaterOrEqual; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertNotBlank; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ADAPTATION_SET; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.AUDIO_CHANNEL_CONFIGURATION; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.BASE_URL; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.INITIALIZATION; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.MPD; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.PERIOD; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.REPRESENTATION; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ROLE; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_BASE; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TEMPLATE; -import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TIMELINE; -import static org.schabi.newpipe.extractor.utils.Utils.isBlank; +import javax.annotation.Nonnull; +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; /** * Test for YouTube DASH manifest creators. @@ -80,126 +87,100 @@ */ class YoutubeDashManifestCreatorsTest { // Setting a higher number may let Google video servers return 403s - private static final int MAX_STREAMS_TO_TEST_PER_METHOD = 3; - private static final String url = "https://www.youtube.com/watch?v=DJ8GQUNUXGM"; + private static final int MAX_STREAMS_TO_TEST_PER_METHOD = 5; + private static final String URL = "https://www.youtube.com/watch?v=DJ8GQUNUXGM"; + private static YoutubeStreamExtractor extractor; - private static long videoLength; @BeforeAll public static void setUp() throws Exception { - YoutubeParsingHelper.resetClientVersionAndKey(); - YoutubeParsingHelper.setNumberGenerator(new Random(1)); + YoutubeTestsUtils.ensureStateless(); + // Has to be done with a real downloader otherwise because there are secondary requests when + // building a DASHManifest which require valid requests with a real IP NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (YoutubeStreamExtractor) YouTube.getStreamExtractor(url); + extractor = (YoutubeStreamExtractor) YouTube.getStreamExtractor(URL); extractor.fetchPage(); - videoLength = extractor.getLength(); } @Test - void testOtfStreams() throws Exception { - assertDashStreams(extractor.getVideoOnlyStreams()); - assertDashStreams(extractor.getAudioStreams()); - - // no video stream with audio uses the DASH delivery method (YouTube OTF stream type) - assertEquals(0, assertFilterStreams(extractor.getVideoStreams(), - DeliveryMethod.DASH).size()); + void testVideoOnlyStreams() throws ExtractionException { + final List videoStreams = getDashStreams(extractor.getVideoOnlyStreams()); + assertTrue(videoStreams.size() > 0); + checkDashStreams(videoStreams); } @Test - void testProgressiveStreams() throws Exception { - assertProgressiveStreams(extractor.getVideoOnlyStreams()); - assertProgressiveStreams(extractor.getAudioStreams()); - - // we are not able to generate DASH manifests of video formats with audio - assertThrows(CreationException.class, - () -> assertProgressiveStreams(extractor.getVideoStreams())); + void testVideoStreams() throws ExtractionException { + final List videoAudioStreams = getDashStreams(extractor.getVideoStreams()); + assertEquals(0, videoAudioStreams.size(), "There should be no dash streams for video-audio streams"); } - private void assertDashStreams(final List streams) throws Exception { - - for (final Stream stream : assertFilterStreams(streams, DeliveryMethod.DASH)) { - //noinspection ConstantConditions - final String manifest = YoutubeOtfDashManifestCreator.fromOtfStreamingUrl( - stream.getContent(), stream.getItagItem(), videoLength); - assertNotBlank(manifest); - - assertManifestGenerated( - manifest, - stream.getItagItem(), - document -> assertAll( - () -> assertSegmentTemplateElement(document), - () -> assertSegmentTimelineAndSElements(document) - ) - ); - } + @Test + void testAudioStreams() throws ExtractionException { + final List audioStreams = getDashStreams(extractor.getAudioStreams()); + assertTrue(audioStreams.size() > 0); + checkDashStreams(audioStreams); } - private void assertProgressiveStreams(final List streams) throws Exception { - - for (final Stream stream : assertFilterStreams(streams, DeliveryMethod.PROGRESSIVE_HTTP)) { - //noinspection ConstantConditions - final String manifest = - YoutubeProgressiveDashManifestCreator.fromProgressiveStreamingUrl( - stream.getContent(), stream.getItagItem(), videoLength); - assertNotBlank(manifest); - - assertManifestGenerated( - manifest, - stream.getItagItem(), - document -> assertAll( - () -> assertBaseUrlElement(document), - () -> assertSegmentBaseElement(document, stream.getItagItem()), - () -> assertInitializationElement(document, stream.getItagItem()) - ) - ); - } + private > List getDashStreams(final List streams) { + return streams.stream() + .filter(s -> s.deliveryData() instanceof DASHManifestDeliveryData) + .collect(Collectors.toList()); } - @Nonnull - private List assertFilterStreams( - @Nonnull final List streams, - final DeliveryMethod deliveryMethod) { - - final List filteredStreams = streams.stream() - .filter(stream -> stream.getDeliveryMethod() == deliveryMethod) + private > void checkDashStreams(final Collection streams) { + assertAll(streams.stream() .limit(MAX_STREAMS_TO_TEST_PER_METHOD) - .collect(Collectors.toList()); - - assertAll(filteredStreams.stream() - .flatMap(stream -> java.util.stream.Stream.of( - () -> assertNotBlank(stream.getContent()), - () -> assertNotNull(stream.getItagItem()) - )) + .map(s -> + () -> checkManifest(s, + ((DASHManifestDeliveryData) s.deliveryData()) + .getCachedDashManifestAsString())) ); - - return filteredStreams; } - private void assertManifestGenerated(final String dashManifest, - final ItagItem itagItem, - final Consumer additionalAsserts) - throws Exception { + private void checkManifest( + final Stream stream, + final String manifestAsString) throws Exception { + assertNotBlank(manifestAsString, "Generated manifest string is blank"); + checkGeneratedManifest(manifestAsString, stream); + } - final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory - .newInstance(); + private void checkGeneratedManifest( + final String dashManifest, + final Stream stream + ) throws Exception { + final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance(); + documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); + documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); final DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder(); - final Document document = documentBuilder.parse(new InputSource( - new StringReader(dashManifest))); + final Document document = + documentBuilder.parse(new InputSource(new StringReader(dashManifest))); - assertAll( + final List asserts = new ArrayList<>(Arrays.asList( () -> assertMpdElement(document), () -> assertPeriodElement(document), - () -> assertAdaptationSetElement(document, itagItem), + () -> assertAdaptationSetElement(document), () -> assertRoleElement(document), - () -> assertRepresentationElement(document, itagItem), - () -> { - if (itagItem.itagType.equals(ItagItem.ItagType.AUDIO)) { - assertAudioChannelConfigurationElement(document, itagItem); - } - }, - () -> additionalAsserts.accept(document) - ); + () -> assertRepresentationElement(document, stream) + )); + + if (stream instanceof BaseAudioStream) { + asserts.add(() -> assertAudioChannelConfigurationElement(document)); + } + + final DashManifestCreator dashManifestCreator = + ((DASHManifestDeliveryData) stream.deliveryData()).dashManifestCreator(); + if (dashManifestCreator instanceof YoutubeOtfDashManifestCreator) { + asserts.add(() -> assertSegmentTemplateElement(document)); + asserts.add(() -> assertSegmentTimelineAndSElements(document)); + } else if (dashManifestCreator instanceof YoutubeProgressiveDashManifestCreator) { + asserts.add(() -> assertBaseUrlElement(document)); + asserts.add(() -> assertSegmentBaseElement(document)); + asserts.add(() -> assertInitializationElement(document)); + } + + assertAll(asserts); } private void assertMpdElement(@Nonnull final Document document) { @@ -216,10 +197,9 @@ private void assertPeriodElement(@Nonnull final Document document) { assertGetElement(document, PERIOD, MPD); } - private void assertAdaptationSetElement(@Nonnull final Document document, - @Nonnull final ItagItem itagItem) { + private void assertAdaptationSetElement(@Nonnull final Document document) { final Element element = assertGetElement(document, ADAPTATION_SET, PERIOD); - assertAttrEquals(itagItem.getMediaFormat().getMimeType(), element, "mimeType"); + assertAttrNotBlank(element, "mimeType"); } private void assertRoleElement(@Nonnull final Document document) { @@ -227,27 +207,24 @@ private void assertRoleElement(@Nonnull final Document document) { } private void assertRepresentationElement(@Nonnull final Document document, - @Nonnull final ItagItem itagItem) { + @Nonnull final Stream stream) { final Element element = assertGetElement(document, REPRESENTATION, ADAPTATION_SET); - assertAttrEquals(itagItem.getBitrate(), element, "bandwidth"); - assertAttrEquals(itagItem.getCodec(), element, "codecs"); + assertAttrNotBlank(element, "bandwidth"); + assertAttrNotBlank(element, "codecs"); - if (itagItem.itagType == ItagItem.ItagType.VIDEO_ONLY - || itagItem.itagType == ItagItem.ItagType.VIDEO) { - assertAttrEquals(itagItem.getFps(), element, "frameRate"); - assertAttrEquals(itagItem.getHeight(), element, "height"); - assertAttrEquals(itagItem.getWidth(), element, "width"); + if (stream instanceof VideoStream) { + assertAttrNotBlank(element, "frameRate"); + assertAttrNotBlank(element, "height"); + assertAttrNotBlank(element, "width"); } - assertAttrEquals(itagItem.id, element, "id"); } - private void assertAudioChannelConfigurationElement(@Nonnull final Document document, - @Nonnull final ItagItem itagItem) { + private void assertAudioChannelConfigurationElement(@Nonnull final Document document) { final Element element = assertGetElement(document, AUDIO_CHANNEL_CONFIGURATION, REPRESENTATION); - assertAttrEquals(itagItem.getAudioChannels(), element, "value"); + assertAttrNotBlank(element, "value"); } private void assertSegmentTemplateElement(@Nonnull final Document document) { @@ -293,59 +270,20 @@ private void assertBaseUrlElement(@Nonnull final Document document) { assertIsValidUrl(element.getTextContent()); } - private void assertSegmentBaseElement(@Nonnull final Document document, - @Nonnull final ItagItem itagItem) { + private void assertSegmentBaseElement(@Nonnull final Document document) { final Element element = assertGetElement(document, SEGMENT_BASE, REPRESENTATION); - assertRangeEquals(itagItem.getIndexStart(), itagItem.getIndexEnd(), element, "indexRange"); + assertAttrNotBlank(element, "indexRange"); } - private void assertInitializationElement(@Nonnull final Document document, - @Nonnull final ItagItem itagItem) { + private void assertInitializationElement(@Nonnull final Document document) { final Element element = assertGetElement(document, INITIALIZATION, SEGMENT_BASE); - assertRangeEquals(itagItem.getInitStart(), itagItem.getInitEnd(), element, "range"); - } - - - private void assertAttrEquals(final int expected, - @Nonnull final Element element, - final String attribute) { - - final int actual = Integer.parseInt(element.getAttribute(attribute)); - assertAll( - () -> assertGreater(0, actual), - () -> assertEquals(expected, actual) - ); + assertAttrNotBlank(element, "range"); } - private void assertAttrEquals(final String expected, - @Nonnull final Element element, - final String attribute) { - final String actual = element.getAttribute(attribute); - assertAll( - () -> assertNotBlank(actual), - () -> assertEquals(expected, actual) - ); + private void assertAttrNotBlank(@Nonnull final Element element, final String attribute) { + assertNotBlank(element.getAttribute(attribute), "Attribute '" + attribute + "' is blank"); } - private void assertRangeEquals(final int expectedStart, - final int expectedEnd, - @Nonnull final Element element, - final String attribute) { - final String range = element.getAttribute(attribute); - assertNotBlank(range); - final String[] rangeParts = range.split("-"); - assertEquals(2, rangeParts.length); - - final int actualStart = Integer.parseInt(rangeParts[0]); - final int actualEnd = Integer.parseInt(rangeParts[1]); - - assertAll( - () -> assertGreaterOrEqual(0, actualStart), - () -> assertEquals(expectedStart, actualStart), - () -> assertGreater(0, actualEnd), - () -> assertEquals(expectedEnd, actualEnd) - ); - } @Nonnull private Element assertGetElement(@Nonnull final Document document, @@ -353,7 +291,7 @@ private Element assertGetElement(@Nonnull final Document document, final String expectedParentTagName) { final Element element = (Element) document.getElementsByTagName(tagName).item(0); - assertNotNull(element); + assertNotNull(element, "Could not get first element with tagName=" + tagName); assertTrue(element.getParentNode().isEqualNode( document.getElementsByTagName(expectedParentTagName).item(0)), "Element with tag name \"" + tagName + "\" does not have a parent node" diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java index f72f598c9c..7a905a6705 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java @@ -9,7 +9,6 @@ import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; import org.schabi.newpipe.extractor.services.youtube.YoutubeTestsUtils; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; import java.util.Collections; import java.util.List; @@ -38,7 +37,6 @@ public static void setUp() throws Exception { @Override public String expectedUrlContains() { return YoutubeStreamExtractorDefaultTest.BASE_URL + ID; } @Override public String expectedOriginalUrlContains() { return URL; } - @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "DAN TV"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCcQHIVL83g5BEQe2IJFb-6w"; } @Override public long expectedUploaderSubscriberCountAtLeast() { return 50; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java index 085fce5be8..9ece3917a0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java @@ -10,7 +10,6 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeTestsUtils; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; import java.util.Arrays; import java.util.List; @@ -42,7 +41,6 @@ public static void setUp() throws Exception { @Override public String expectedUrlContains() { return URL; } @Override public String expectedOriginalUrlContains() { return URL; } - @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "Amazing Atheist"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCjNxszyFPasDdRoD9J6X-sw"; } @Override public long expectedUploaderSubscriberCountAtLeast() { return 900_000; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index b220c3d190..17683f4b74 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -43,9 +43,9 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeTestsUtils; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.stream.Description; +import org.schabi.newpipe.extractor.stream.Privacy; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamSegment; -import org.schabi.newpipe.extractor.stream.StreamType; import java.io.IOException; import java.net.MalformedURLException; @@ -133,7 +133,6 @@ public static void setUp() throws Exception { @Override public String expectedUrlContains() { return BASE_URL + ID; } @Override public String expectedOriginalUrlContains() { return URL; } - @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "PewDiePie"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw"; } @Override public long expectedUploaderSubscriberCountAtLeast() { return 110_000_000; } @@ -176,7 +175,6 @@ public static void setUp() throws Exception { @Override public String expectedUrlContains() { return URL; } @Override public String expectedOriginalUrlContains() { return URL; } - @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "Unbox Therapy"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCsTcErHg8oDvUnTzoqsYeNw"; } @Override public long expectedUploaderSubscriberCountAtLeast() { return 18_000_000; } @@ -229,7 +227,6 @@ public static void setUp() throws Exception { @Override public String expectedUrlContains() { return BASE_URL + ID; } @Override public String expectedOriginalUrlContains() { return URL; } - @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "YouTuber PrinceOfFALLEN"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCQT2yul0lr6Ie9qNQNmw-sg"; } @Override public List expectedDescriptionContains() { return Arrays.asList("dislikes", "Alpha", "wrong"); } @@ -265,7 +262,6 @@ public static void setUp() throws Exception { @Override public String expectedUrlContains() { return BASE_URL + ID; } @Override public String expectedOriginalUrlContains() { return URL; } - @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "tagesschau"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UC5NOEUbkLheQcaaRldYW5GA"; } @Override public long expectedUploaderSubscriberCountAtLeast() { return 1_000_000; } @@ -326,7 +322,6 @@ public static void setUp() throws Exception { @Override public String expectedUrlContains() { return BASE_URL + ID; } @Override public String expectedOriginalUrlContains() { return URL; } - @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "maiLab"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCyHDQ5C6z1NDmJ4g6SerW8g"; } @Override public long expectedUploaderSubscriberCountAtLeast() { return 1_400_000; } @@ -394,7 +389,6 @@ public static void setUp() throws Exception { @Override public String expectedUrlContains() { return BASE_URL + ID; } @Override public String expectedOriginalUrlContains() { return URL; } - @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "Dinge Erklärt – Kurzgesagt"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCwRH985XgMYXQ6NxXDo8npw"; } @Override public long expectedUploaderSubscriberCountAtLeast() { return 1_500_000; } @@ -442,7 +436,7 @@ public static void setUp() throws Exception { @Test void testGetUnlisted() { - assertEquals(StreamExtractor.Privacy.UNLISTED, extractor.getPrivacy()); + assertEquals(Privacy.UNLISTED, extractor.getPrivacy()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java index 6b26a0d04e..ac5446f2b6 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java @@ -10,7 +10,6 @@ import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; import org.schabi.newpipe.extractor.services.youtube.YoutubeTestsUtils; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; import java.util.Arrays; import java.util.List; @@ -45,7 +44,7 @@ public void testUploaderName() throws Exception { @Override public String expectedUrlContains() { return YoutubeStreamExtractorDefaultTest.BASE_URL + ID; } @Override public String expectedOriginalUrlContains() { return URL; } - @Override public StreamType expectedStreamType() { return StreamType.LIVE_STREAM; } + @Override public boolean expectedIsLive() { return true; } @Override public String expectedUploaderName() { return "Lofi Girl"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCSJ4gkVC6NrvII8umztf0Ow"; } @Override public long expectedUploaderSubscriberCountAtLeast() { return 9_800_000; } @@ -61,6 +60,7 @@ public void testUploaderName() throws Exception { @Nullable @Override public String expectedTextualUploadDate() { return "2022-07-12"; } @Override public long expectedLikeCountAtLeast() { return 340_000; } @Override public long expectedDislikeCountAtLeast() { return -1; } + @Override public boolean expectedHasVideoAndAudioStreams() { return false; } @Override public boolean expectedHasSubtitles() { return false; } @Nullable @Override public String expectedDashMpdUrlContains() { return "https://manifest.googlevideo.com/api/manifest/dash/"; } @Override public boolean expectedHasFrames() { return false; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorRelatedMixTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorRelatedMixTest.java index cd7feb3527..a1f56b1691 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorRelatedMixTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorRelatedMixTest.java @@ -20,7 +20,6 @@ import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; import org.schabi.newpipe.extractor.services.youtube.YoutubeTestsUtils; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; import java.util.Arrays; import java.util.List; @@ -52,7 +51,6 @@ public static void setUp() throws Exception { @Override public String expectedUrlContains() { return URL; } @Override public String expectedOriginalUrlContains() { return URL; } - @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "NoCopyrightSounds"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UC_aEa8K-EOJ3D6gOs7HcyNg"; } @Override public List expectedDescriptionContains() { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java index 691ec59167..57df83f9a3 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java @@ -1,7 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.stream; import static org.schabi.newpipe.extractor.ServiceList.YouTube; -import static org.schabi.newpipe.extractor.stream.StreamExtractor.Privacy.UNLISTED; +import static org.schabi.newpipe.extractor.stream.Privacy.UNLISTED; import org.junit.jupiter.api.BeforeAll; import org.schabi.newpipe.downloader.DownloaderFactory; @@ -9,8 +9,8 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; import org.schabi.newpipe.extractor.services.youtube.YoutubeTestsUtils; +import org.schabi.newpipe.extractor.stream.Privacy; import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamType; import java.util.Arrays; import java.util.List; @@ -38,7 +38,6 @@ public static void setUp() throws Exception { @Override public String expectedUrlContains() { return URL; } @Override public String expectedOriginalUrlContains() { return URL; } - @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } @Override public String expectedUploaderName() { return "Hooked"; } @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCPysfiuOv4VKBeXFFPhKXyw"; } @Override public long expectedUploaderSubscriberCountAtLeast() { return 24_300; } @@ -52,7 +51,7 @@ public static void setUp() throws Exception { @Nullable @Override public String expectedTextualUploadDate() { return "2017-09-22"; } @Override public long expectedLikeCountAtLeast() { return 110; } @Override public long expectedDislikeCountAtLeast() { return -1; } - @Override public StreamExtractor.Privacy expectedPrivacy() { return UNLISTED; } + @Override public Privacy expectedPrivacy() { return UNLISTED; } @Override public String expectedLicence() { return "YouTube licence"; } @Override public String expectedCategory() { return "Gaming"; } @Override public List expectedTags() { return Arrays.asList("dark souls", "hooked", "praise the casual"); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCacheTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCacheTest.java deleted file mode 100644 index 83c5c1dfb1..0000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCacheTest.java +++ /dev/null @@ -1,74 +0,0 @@ -package org.schabi.newpipe.extractor.utils; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -class ManifestCreatorCacheTest { - @Test - void basicMaximumSizeAndResetTest() { - final ManifestCreatorCache cache = new ManifestCreatorCache<>(); - - // 30 elements set -> cache resized to 23 -> 5 new elements set to the cache -> 28 - cache.setMaximumSize(30); - setCacheContent(cache); - assertEquals(28, cache.size(), - "Wrong cache size with default clear factor and 30 as the maximum size"); - cache.reset(); - - assertEquals(0, cache.size(), - "The cache has been not cleared after a reset call (wrong cache size)"); - assertEquals(ManifestCreatorCache.DEFAULT_MAXIMUM_SIZE, cache.getMaximumSize(), - "Wrong maximum size after cache reset"); - assertEquals(ManifestCreatorCache.DEFAULT_CLEAR_FACTOR, cache.getClearFactor(), - "Wrong clear factor after cache reset"); - } - - @Test - void maximumSizeAndClearFactorSettersAndResettersTest() { - final ManifestCreatorCache cache = new ManifestCreatorCache<>(); - cache.setMaximumSize(20); - cache.setClearFactor(0.5); - - setCacheContent(cache); - // 30 elements set -> cache resized to 10 -> 5 new elements set to the cache -> 15 - assertEquals(15, cache.size(), - "Wrong cache size with 0.5 as the clear factor and 20 as the maximum size"); - - // Clear factor and maximum size getters tests - assertEquals(0.5, cache.getClearFactor(), - "Wrong clear factor gotten from clear factor getter"); - assertEquals(20, cache.getMaximumSize(), - "Wrong maximum cache size gotten from maximum size getter"); - - // Resetters tests - cache.resetMaximumSize(); - assertEquals(ManifestCreatorCache.DEFAULT_MAXIMUM_SIZE, cache.getMaximumSize(), - "Wrong maximum cache size gotten from maximum size getter after maximum size " - + "resetter call"); - - cache.resetClearFactor(); - assertEquals(ManifestCreatorCache.DEFAULT_CLEAR_FACTOR, cache.getClearFactor(), - "Wrong clear factor gotten from clear factor getter after clear factor resetter " - + "call"); - } - - /** - * Adds sample strings to the provided manifest creator cache, in order to test clear factor and - * maximum size. - * @param cache the cache to fill with some data - */ - private static void setCacheContent(final ManifestCreatorCache cache) { - int i = 0; - while (i < 26) { - cache.put(String.valueOf((char) ('a' + i)), "V"); - ++i; - } - - i = 0; - while (i < 9) { - cache.put("a" + (char) ('a' + i), "V"); - ++i; - } - } -}