Rewrote extractors part 1

Author: litetex

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java

@@ -12,30 +12,31 @@
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
-import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector;
-import org.schabi.newpipe.extractor.stream.AudioStream;
 import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
-import org.schabi.newpipe.extractor.stream.VideoStream;
+import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleProgressiveHTTPDeliveryDataImpl;
+import org.schabi.newpipe.extractor.streamdata.format.registry.AudioFormatRegistry;
+import org.schabi.newpipe.extractor.streamdata.stream.AudioStream;
+import org.schabi.newpipe.extractor.streamdata.stream.simpleimpl.SimpleAudioStreamImpl;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Utils;
 
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
 public class BandcampStreamExtractor extends StreamExtractor {
     private JsonObject albumJson;
     private JsonObject current;
@@ -149,17 +150,17 @@ public Description getDescription() {
 
     @Override
     public List<AudioStream> getAudioStreams() {
-        final List<AudioStream> audioStreams = new ArrayList<>();
-        audioStreams.add(new AudioStream.Builder()
-                .setId("mp3-128")
-                .setContent(albumJson.getArray("trackinfo")
-                        .getObject(0)
-                        .getObject("file")
-                        .getString("mp3-128"), true)
-                .setMediaFormat(MediaFormat.MP3)
-                .setAverageBitrate(128)
-                .build());
-        return audioStreams;
+        return Collections.singletonList(
+                new SimpleAudioStreamImpl(
+                        new SimpleProgressiveHTTPDeliveryDataImpl(albumJson
+                                .getArray("trackinfo")
+                                .getObject(0)
+                                .getObject("file")
+                                .getString("mp3-128")),
+                        AudioFormatRegistry.MP3,
+                        128
+                )
+        );
     }
 
     @Override
@@ -168,16 +169,6 @@ public long getLength() throws ParsingException {
                 .getDouble("duration");
     }
 
-    @Override
-    public List<VideoStream> getVideoStreams() {
-        return Collections.emptyList();
-    }
-
-    @Override
-    public List<VideoStream> getVideoOnlyStreams() {
-        return Collections.emptyList();
-    }
-
     @Override
     public StreamType getStreamType() {
         return StreamType.AUDIO_STREAM;

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java

@@ -3,32 +3,32 @@
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 
-import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
-import org.schabi.newpipe.extractor.stream.AudioStream;
-import org.schabi.newpipe.extractor.stream.DeliveryMethod;
 import org.schabi.newpipe.extractor.stream.Description;
-import org.schabi.newpipe.extractor.stream.Stream;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
-import org.schabi.newpipe.extractor.stream.VideoStream;
+import org.schabi.newpipe.extractor.streamdata.delivery.DeliveryData;
+import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleDASHUrlDeliveryDataImpl;
+import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleHLSDeliveryDataImpl;
+import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleProgressiveHTTPDeliveryDataImpl;
+import org.schabi.newpipe.extractor.streamdata.format.registry.AudioFormatRegistry;
+import org.schabi.newpipe.extractor.streamdata.format.registry.VideoAudioFormatRegistry;
+import org.schabi.newpipe.extractor.streamdata.stream.AudioStream;
+import org.schabi.newpipe.extractor.streamdata.stream.VideoAudioStream;
+import org.schabi.newpipe.extractor.streamdata.stream.simpleimpl.SimpleAudioStreamImpl;
+import org.schabi.newpipe.extractor.streamdata.stream.simpleimpl.SimpleVideoAudioStreamImpl;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.List;
-import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import javax.annotation.Nonnull;
 
-import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE;
-import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN;
-import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
-
 public class MediaCCCLiveStreamExtractor extends StreamExtractor {
     private static final String STREAMS = "streams";
     private static final String URLS = "urls";
@@ -46,7 +46,8 @@ public MediaCCCLiveStreamExtractor(final StreamingService service,
     @Override
     public void onFetchPage(@Nonnull final Downloader downloader)
             throws IOException, ExtractionException {
-        final JsonArray doc = MediaCCCParsingHelper.getLiveStreams(downloader,
+        final JsonArray doc = MediaCCCParsingHelper.getLiveStreams(
+                downloader,
                 getExtractorLocalization());
         // Find the correct room
         for (int c = 0; c < doc.size(); c++) {
@@ -138,7 +139,7 @@ public String getDashMpdUrl() throws ParsingException {
      */
     @Nonnull
     @Override
-    public String getHlsUrl() {
+    public String getHlsMasterPlaylistUrl() {
         return getManifestOfDeliveryMethodWanted("hls");
     }
 
@@ -149,84 +150,61 @@ private String getManifestOfDeliveryMethodWanted(@Nonnull final String deliveryM
                 .map(JsonObject.class::cast)
                 .map(streamObject -> streamObject.getObject(URLS))
                 .filter(urls -> urls.has(deliveryMethod))
-                .map(urls -> urls.getObject(deliveryMethod).getString(URL, EMPTY_STRING))
+                .map(urls -> urls.getObject(deliveryMethod).getString(URL, ""))
                 .findFirst()
-                .orElse(EMPTY_STRING);
+                .orElse("");
     }
 
     @Override
     public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
-        return getStreams("audio",
-                dto -> {
-                    final AudioStream.Builder builder = new AudioStream.Builder()
-                            .setId(dto.urlValue.getString("tech", ID_UNKNOWN))
-                            .setContent(dto.urlValue.getString(URL), true)
-                            .setAverageBitrate(UNKNOWN_BITRATE);
-
-                    if ("hls".equals(dto.urlKey)) {
-                        // We don't know with the type string what media format will
-                        // have HLS streams.
-                        // However, the tech string may contain some information
-                        // about the media format used.
-                        return builder.setDeliveryMethod(DeliveryMethod.HLS)
-                                .build();
+        return getStreamDTOs("audio")
+                .map(dto -> {
+                    final String url = dto.getUrlValue().getString(URL);
+                    final DeliveryData deliveryData;
+                    if ("hls".equals(dto.getUrlKey())) {
+                        deliveryData = new SimpleHLSDeliveryDataImpl(url);
+                    } else if ("dash".equals(dto.getUrlKey())) {
+                        deliveryData = new SimpleDASHUrlDeliveryDataImpl(url);
+                    } else {
+                        deliveryData = new SimpleProgressiveHTTPDeliveryDataImpl(url);
                     }
 
-                    return builder.setMediaFormat(MediaFormat.getFromSuffix(dto.urlKey))
-                            .build();
-                });
+                    return new SimpleAudioStreamImpl(
+                            deliveryData,
+                            // TODO: This looks wrong
+                            new AudioFormatRegistry().getFromSuffix(dto.getUrlKey())
+                    );
+                })
+                .collect(Collectors.toList());
     }
 
     @Override
-    public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
-        return getStreams("video",
-                dto -> {
-                    final JsonArray videoSize = dto.streamJsonObj.getArray("videoSize");
-
-                    final VideoStream.Builder builder = new VideoStream.Builder()
-                            .setId(dto.urlValue.getString("tech", ID_UNKNOWN))
-                            .setContent(dto.urlValue.getString(URL), true)
-                            .setIsVideoOnly(false)
-                            .setResolution(videoSize.getInt(0) + "x" + videoSize.getInt(1));
-
-                    if ("hls".equals(dto.urlKey)) {
-                        // We don't know with the type string what media format will
-                        // have HLS streams.
-                        // However, the tech string may contain some information
-                        // about the media format used.
-                        return builder.setDeliveryMethod(DeliveryMethod.HLS)
-                                .build();
+    public List<VideoAudioStream> getVideoStreams() throws IOException, ExtractionException {
+        return getStreamDTOs("video")
+                .map(dto -> {
+                    final String url = dto.getUrlValue().getString(URL);
+                    final DeliveryData deliveryData;
+                    if ("hls".equals(dto.getUrlKey())) {
+                        deliveryData = new SimpleHLSDeliveryDataImpl(url);
+                    } else if ("dash".equals(dto.getUrlKey())) {
+                        deliveryData = new SimpleDASHUrlDeliveryDataImpl(url);
+                    } else {
+                        deliveryData = new SimpleProgressiveHTTPDeliveryDataImpl(url);
                     }
 
-                    return builder.setMediaFormat(MediaFormat.getFromSuffix(dto.urlKey))
-                            .build();
-                });
-    }
-
+                    final JsonArray videoSize = dto.getStreamJsonObj().getArray("videoSize");
 
-    /**
-     * This is just an internal class used in {@link #getStreams(String, Function)} to tie together
-     * the stream json object, its URL key and its URL value. An object of this class would be
-     * temporary and the three values it holds would be <b>convert</b>ed to a proper {@link Stream}
-     * object based on the wanted stream type.
-     */
-    private static final class MediaCCCLiveStreamMapperDTO {
-        final JsonObject streamJsonObj;
-        final String urlKey;
-        final JsonObject urlValue;
-
-        MediaCCCLiveStreamMapperDTO(final JsonObject streamJsonObj,
-                                    final String urlKey,
-                                    final JsonObject urlValue) {
-            this.streamJsonObj = streamJsonObj;
-            this.urlKey = urlKey;
-            this.urlValue = urlValue;
-        }
+                    return new SimpleVideoAudioStreamImpl(
+                            deliveryData,
+                            // TODO: This looks wrong
+                            new VideoAudioFormatRegistry().getFromSuffix(dto.getUrlKey()),
+                            videoSize.getInt(0) + "x" + videoSize.getInt(1)
+                    );
+                })
+                .collect(Collectors.toList());
     }
 
-    private <T extends Stream> List<T> getStreams(
-            @Nonnull final String streamType,
-            @Nonnull final Function<MediaCCCLiveStreamMapperDTO, T> converter) {
+    private Stream<MediaCCCLiveStreamMapperDTO> getStreamDTOs(@Nonnull final String streamType) {
         return room.getArray(STREAMS).stream()
                 // Ensure that we use only process JsonObjects
                 .filter(JsonObject.class::isInstance)
@@ -239,22 +217,17 @@ private <T extends Stream> List<T> getStreams(
                         .map(e -> new MediaCCCLiveStreamMapperDTO(
                                 streamJsonObj,
                                 e.getKey(),
-                                (JsonObject) e.getValue())))
-                // The DASH manifest will be extracted with getDashMpdUrl
-                .filter(dto -> !"dash".equals(dto.urlKey))
-                // Convert
-                .map(converter)
-                .collect(Collectors.toList());
+                                (JsonObject) e.getValue())));
     }
 
     @Override
-    public List<VideoStream> getVideoOnlyStreams() {
-        return Collections.emptyList();
+    public StreamType getStreamType() throws ParsingException {
+        return StreamType.LIVE_STREAM;
     }
 
     @Override
-    public StreamType getStreamType() throws ParsingException {
-        return StreamType.LIVE_STREAM;
+    public boolean isLive() {
+        return true;
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java

@@ -3,6 +3,7 @@
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
+
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -43,13 +44,21 @@ public static boolean isLiveStreamId(final String id) {
     /**
      * Get currently available live streams from
      * <a href="https://streaming.media.ccc.de/streams/v2.json">
-     *     https://streaming.media.ccc.de/streams/v2.json</a>.
+     * https://streaming.media.ccc.de/streams/v2.json</a>.
+     * <p>
      * Use this method to cache requests, because they can get quite big.
+     * </p>
+     *
+     * <p>
+     * For more information see also: <a href="https://github.com/voc/streaming-website#json-api">
+     * https://github.com/voc/streaming-website#json-api</a>.
+     * </p>
      * TODO: implement better caching policy (max-age: 3 min)
-     * @param downloader The downloader to use for making the request
+     *
+     * @param downloader   The downloader to use for making the request
      * @param localization The localization to be used. Will most likely be ignored.
      * @return {@link JsonArray} containing current conferences and info about their rooms and
-     *         streams.
+     * streams.
      * @throws ExtractionException if the data could not be fetched or the retrieved data could not
      *                             be parsed to a {@link JsonArray}
      */
@@ -58,13 +67,14 @@ public static JsonArray getLiveStreams(final Downloader downloader,
             throws ExtractionException {
         if (liveStreams == null) {
             try {
-                final String site = downloader.get("https://streaming.media.ccc.de/streams/v2.json",
-                        localization).responseBody();
+                final String site = downloader
+                        .get("https://streaming.media.ccc.de/streams/v2.json", localization)
+                        .responseBody();
                 liveStreams = JsonParser.array().from(site);
             } catch (final IOException | ReCaptchaException e) {
-                throw new ExtractionException("Could not get live stream JSON.", e);
+                throw new ExtractionException("Could not get live stream JSON", e);
             } catch (final JsonParserException e) {
-                throw new ExtractionException("Could not parse JSON.", e);
+                throw new ExtractionException("Could not parse JSON", e);
             }
         }
         return liveStreams;