More refactoring + started implementing ItagInfo

Author: litetex

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java

@@ -152,12 +152,12 @@ public Description getDescription() {
     public List<AudioStream> getAudioStreams() {
         return Collections.singletonList(
                 new SimpleAudioStreamImpl(
+                        AudioFormatRegistry.MP3,
                         new SimpleProgressiveHTTPDeliveryDataImpl(albumJson
                                 .getArray("trackinfo")
                                 .getObject(0)
                                 .getObject("file")
                                 .getString("mp3-128")),
-                        AudioFormatRegistry.MP3,
                         128
                 )
         );

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java

@@ -171,9 +171,9 @@ public List<AudioStream> getAudioStreams() throws IOException, ExtractionExcepti
                     }
 
                     return new SimpleAudioStreamImpl(
-                            deliveryData,
                             // TODO: This looks wrong
-                            new AudioFormatRegistry().getFromSuffix(dto.getUrlKey())
+                            new AudioFormatRegistry().getFromSuffix(dto.getUrlKey()),
+                            deliveryData
                     );
                 })
                 .collect(Collectors.toList());
@@ -196,9 +196,9 @@ public List<VideoAudioStream> getVideoStreams() throws IOException, ExtractionEx
                     final JsonArray videoSize = dto.getStreamJsonObj().getArray("videoSize");
 
                     return new SimpleVideoAudioStreamImpl(
-                            deliveryData,
                             // TODO: This looks wrong
                             new VideoAudioFormatRegistry().getFromSuffix(dto.getUrlKey()),
+                            deliveryData,
                             VideoQualityData.fromHeightWidth(
                                     /*height=*/videoSize.getInt(1, VideoQualityData.UNKNOWN),
                                     /*width=*/videoSize.getInt(0, VideoQualityData.UNKNOWN))

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java

@@ -99,8 +99,8 @@ public String getUploaderAvatarUrl() {
     public List<AudioStream> getAudioStreams() throws ExtractionException {
         return getRecordingsByMimeType("audio")
                 .map(o -> new SimpleAudioStreamImpl(
-                        new SimpleProgressiveHTTPDeliveryDataImpl(o.getString("recording_url")),
-                        new AudioFormatRegistry().getFromMimeType(o.getString("mime_type"))
+                        new AudioFormatRegistry().getFromMimeType(o.getString("mime_type")),
+                        new SimpleProgressiveHTTPDeliveryDataImpl(o.getString("recording_url"))
                 ))
                 .collect(Collectors.toList());
     }
@@ -109,10 +109,8 @@ public List<AudioStream> getAudioStreams() throws ExtractionException {
     public List<VideoAudioStream> getVideoStreams() throws ExtractionException {
         return getRecordingsByMimeType("video")
                 .map(o -> new SimpleVideoAudioStreamImpl(
-                        new SimpleProgressiveHTTPDeliveryDataImpl(o.getString("recording_url")),
-                        null,
-                        AudioStream.UNKNOWN_BITRATE,
                         new VideoAudioFormatRegistry().getFromMimeType(o.getString("mime_type")),
+                        new SimpleProgressiveHTTPDeliveryDataImpl(o.getString("recording_url")),
                         VideoQualityData.fromHeightWidth(
                                 o.getInt("height", VideoQualityData.UNKNOWN),
                                 o.getInt("width", VideoQualityData.UNKNOWN))

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java

@@ -403,14 +403,15 @@ private void tryExtractSubtitles() {
                     .map(JsonObject.class::cast)
                     .map(caption -> {
                         try {
-                            final String url = baseUrl + JsonUtils.getString(caption,
-                                    "captionPath");
+                            final String url =
+                                    baseUrl + JsonUtils.getString(caption, "captionPath");
 
                             return new SimpleSubtitleStreamImpl(
-                                    new SimpleProgressiveHTTPDeliveryDataImpl(url),
+                                    // TODO: Check for null
                                     new SubtitleFormatRegistry()
                                             .getFromSuffix(
                                                     url.substring(url.lastIndexOf(".") + 1)),
+                                    new SimpleProgressiveHTTPDeliveryDataImpl(url),
                                     false,
                                     JsonUtils.getString(caption, "language.id")
                             );
@@ -468,9 +469,9 @@ private void extractLiveVideoStreams() throws ParsingException {
                     .filter(JsonObject.class::isInstance)
                     .map(JsonObject.class::cast)
                     // TODO Check! This is the master playlist!
-                    .map(stream -> new SimpleVideoAudioStreamImpl(
-                            new SimpleHLSDeliveryDataImpl(stream.getString(PLAYLIST_URL, "")),
-                            VideoAudioFormatRegistry.MPEG_4)
+                    .map(s -> new SimpleVideoAudioStreamImpl(
+                            VideoAudioFormatRegistry.MPEG_4,
+                            new SimpleHLSDeliveryDataImpl(s.getString(PLAYLIST_URL, "")))
                     )
                     // Don't use the containsSimilarStream method because it will always
                     // return
@@ -502,9 +503,9 @@ private void addStreamsFromArray(
                                 stream,
                                 playlistUrl,
                                 (s, dd) -> new SimpleAudioStreamImpl(
-                                        dd,
                                         new AudioFormatRegistry()
-                                                .getFromSuffix(getExtensionFromStream(s))
+                                                .getFromSuffix(getExtensionFromStream(s)),
+                                        dd
                                 )
                         );
 
@@ -515,9 +516,9 @@ private void addStreamsFromArray(
                                 stream,
                                 playlistUrl,
                                 (s, dd) -> new SimpleVideoAudioStreamImpl(
-                                        dd,
                                         new VideoAudioFormatRegistry()
                                                 .getFromSuffix(getExtensionFromStream(s)),
+                                        dd,
                                         VideoQualityData.fromHeightFps(
                                                 resJson.getInt("id", VideoQualityData.UNKNOWN),
                                                 stream.getInt("fps", VideoQualityData.UNKNOWN))

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java

@@ -15,6 +15,7 @@
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
+import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException;
@@ -268,8 +269,8 @@ private List<AudioStream> extractAudioStreams() {
 
                     final String preset = transcoding.getString("preset", "");
 
-                    AudioMediaFormat mediaFormat = null;
-                    int averageBitrate = AudioStream.UNKNOWN_BITRATE;
+                    final AudioMediaFormat mediaFormat;
+                    final int averageBitrate;
                     if (preset.contains("mp3")) {
                         // Don't add the MP3 HLS stream if there is a progressive stream present
                         // because the two have the same bitrate
@@ -281,13 +282,15 @@ private List<AudioStream> extractAudioStreams() {
                     } else if (preset.contains("opus")) {
                         mediaFormat = AudioFormatRegistry.OPUS;
                         averageBitrate = 64;
+                    } else {
+                        return null;
                     }
 
                     return (AudioStream) new SimpleAudioStreamImpl(
+                            mediaFormat,
                             protocol.equals("hls")
                                     ? new SimpleHLSDeliveryDataImpl(mediaUrl)
                                     : new SimpleProgressiveHTTPDeliveryDataImpl(mediaUrl),
-                            mediaFormat,
                             averageBitrate
                     );
                 })
@@ -321,7 +324,17 @@ private Optional<AudioStream> extractDownloadableFileIfAvailable() {
             if (isNullOrEmpty(downloadUrl)) {
                 return Optional.empty();
             }
+
+            // Find out what type of file is served
+            final String fileType = determineFileTypeFromDownloadUrl(downloadUrl);
+
+            // No fileType found -> ignore it
+            if (isNullOrEmpty(fileType)) {
+                return Optional.empty();
+            }
+
             return Optional.of(new SimpleAudioStreamImpl(
+                    new AudioFormatRegistry().getFromSuffix(fileType),
                     new SimpleProgressiveHTTPDeliveryDataImpl(downloadUrl)
             ));
         } catch (final Exception ignored) {
@@ -331,6 +344,37 @@ private Optional<AudioStream> extractDownloadableFileIfAvailable() {
         }
     }
 
+    /**
+     * Determines the file type/extension of the download url.
+     * <p>
+     * Note: Uses HTTP FETCH for inspection.
+     * </p>
+     */
+    @Nullable
+    private String determineFileTypeFromDownloadUrl(final String downloadUrl)
+            throws IOException, ReCaptchaException {
+
+        final Response response = NewPipe.getDownloader().head(downloadUrl);
+
+        // As of 2022-06 Soundcloud uses AWS S3
+        // Use the AWS header to identify the filetype first because it's simpler
+        final String amzMetaFileType = response.getHeader("x-amz-meta-file-type");
+        if (!isNullOrEmpty(amzMetaFileType)) {
+            return amzMetaFileType;
+        }
+
+        // If the AWS header was not present try extract the filetype
+        // by inspecting the download file name
+        // Example-Value:
+        // attachment;filename="SoundCloud%20Download"; filename*=utf-8''song.mp3
+        final String contentDisp = response.getHeader("Content-Disposition");
+        if (!isNullOrEmpty(contentDisp) && contentDisp.contains(".")) {
+            return contentDisp.substring(contentDisp.lastIndexOf(".") + 1);
+        }
+
+        return null;
+    }
+
     /**
      * Parses a SoundCloud HLS manifest to get a single URL of HLS streams.
      *

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/AudioItagFormat.java

@@ -0,0 +1,7 @@
+package org.schabi.newpipe.extractor.services.youtube.itag.format;
+
+import org.schabi.newpipe.extractor.streamdata.format.AudioMediaFormat;
+
+public interface AudioItagFormat extends ItagFormat<AudioMediaFormat>, BaseAudioItagFormat {
+    // Nothing additional to implement
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/ItagFormat.java

@@ -0,0 +1,8 @@
+package org.schabi.newpipe.extractor.services.youtube.itag.format;
+
+public interface BaseAudioItagFormat {
+    /**
+     * Average audio bitrate in KBit/s
+     */
+    int averageBitrate();
+}