[SoundCloud] Use the HLS delivery method for all streams and extract only a single stream URL from HLS manifest for MP3 streams

SoundCloud broke the workaround used to get a single file from HLS manifests for Opus manifests, but it still works for MP3 ones.

The code has been adapted to prevent an unneeded request (the one to the Opus HLS manifest) and the HLS delivery method is now used for SoundCloud MP3 and Opus streams, plus the progressive one (for tracks which have a progressive stream (MP3) and for the ones which doesn't have one, it is still used by trying to get a progressive stream, using the workaround).

Streams extraction has been also moved to Java 8 Stream's API and the relevant test has been also updated.

Author: AudricV

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java

@@ -28,6 +28,7 @@
 import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
 import org.schabi.newpipe.extractor.stream.AudioStream;
+import org.schabi.newpipe.extractor.stream.DeliveryMethod;
 import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.Stream;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
@@ -169,7 +170,6 @@ public List<AudioStream> getAudioStreams() throws ExtractionException {
         // Streams can be streamable and downloadable - or explicitly not.
         // For playing the track, it is only necessary to have a streamable track.
         // If this is not the case, this track might not be published yet.
-        // If audio streams were calculated, return the calculated result
         if (!track.getBoolean("streamable") || !isAvailable) {
             return audioStreams;
         }
@@ -181,53 +181,37 @@ public List<AudioStream> getAudioStreams() throws ExtractionException {
                 extractAudioStreams(transcodings, checkMp3ProgressivePresence(transcodings),
                         audioStreams);
             }
+
             extractDownloadableFileIfAvailable(audioStreams);
         } catch (final NullPointerException e) {
-            throw new ExtractionException("Could not get SoundCloud's tracks audio URL", e);
+            throw new ExtractionException("Could not get audio streams", e);
         }
 
         return audioStreams;
     }
 
     private static boolean checkMp3ProgressivePresence(@Nonnull final JsonArray transcodings) {
-        boolean presence = false;
-        for (final Object transcoding : transcodings) {
-            final JsonObject transcodingJsonObject = (JsonObject) transcoding;
-            if (transcodingJsonObject.getString("preset").contains("mp3")
-                    && transcodingJsonObject.getObject("format").getString("protocol")
-                            .equals("progressive")) {
-                presence = true;
-                break;
-            }
-        }
-        return presence;
+        return transcodings.stream()
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                .anyMatch(transcodingJsonObject -> transcodingJsonObject.getString("preset")
+                        .contains("mp3") && transcodingJsonObject.getObject("format")
+                        .getString("protocol").equals("progressive"));
     }
 
     @Nonnull
-    private String getTranscodingUrl(final String endpointUrl,
-                                     final String protocol)
+    private String getTranscodingUrl(final String endpointUrl)
             throws IOException, ExtractionException {
-        final Downloader downloader = NewPipe.getDownloader();
-        final String apiStreamUrl = endpointUrl + "?client_id="
-                + clientId();
-        final String response = downloader.get(apiStreamUrl).responseBody();
+        final String apiStreamUrl = endpointUrl + "?client_id=" + clientId();
+        final String response = NewPipe.getDownloader().get(apiStreamUrl).responseBody();
         final JsonObject urlObject;
         try {
             urlObject = JsonParser.object().from(response);
         } catch (final JsonParserException e) {
             throw new ParsingException("Could not parse streamable URL", e);
         }
 
-        final String urlString = urlObject.getString("url");
-
-        if (protocol.equals("progressive")) {
-            return urlString;
-        } else if (protocol.equals("hls")) {
-            return getSingleUrlFromHlsManifest(urlString);
-        }
-
-        // else, unknown protocol
-        return EMPTY_STRING;
+        return urlObject.getString("url");
     }
 
     @Nullable
@@ -252,50 +236,87 @@ private String getDownloadUrl(@Nonnull final String trackId)
     private void extractAudioStreams(@Nonnull final JsonArray transcodings,
                                      final boolean mp3ProgressiveInStreams,
                                      final List<AudioStream> audioStreams) {
-        for (final Object transcoding : transcodings) {
-            final JsonObject transcodingJsonObject = (JsonObject) transcoding;
-            final String url = transcodingJsonObject.getString("url");
-            if (isNullOrEmpty(url)) {
-                continue;
-            }
-
-            final String mediaUrl;
-            final String preset = transcodingJsonObject.getString("preset", ID_UNKNOWN);
-            final String protocol = transcodingJsonObject.getObject("format")
-                    .getString("protocol");
-            MediaFormat mediaFormat = null;
-            int averageBitrate = UNKNOWN_BITRATE;
-            if (preset.contains("mp3")) {
-                // Don't add the MP3 HLS stream if there is a progressive stream present
-                // because the two have the same bitrate
-                if (mp3ProgressiveInStreams && protocol.equals("hls")) {
-                    continue;
-                }
-                mediaFormat = MediaFormat.MP3;
-                averageBitrate = 128;
-            } else if (preset.contains("opus")) {
-                mediaFormat = MediaFormat.OPUS;
-                averageBitrate = 64;
-            }
+        transcodings.stream()
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                .forEachOrdered(transcoding -> {
+                    final String url = transcoding.getString("url");
+                    if (isNullOrEmpty(url)) {
+                        return;
+                    }
 
-            try {
-                mediaUrl = getTranscodingUrl(url, protocol);
-                if (!mediaUrl.isEmpty()) {
-                    final AudioStream audioStream = new AudioStream.Builder()
-                            .setId(preset)
-                            .setContent(mediaUrl, true)
-                            .setMediaFormat(mediaFormat)
-                            .setAverageBitrate(averageBitrate)
-                            .build();
-                    if (!Stream.containSimilarStream(audioStream, audioStreams)) {
-                        audioStreams.add(audioStream);
+                    final String preset = transcoding.getString("preset", ID_UNKNOWN);
+                    final String protocol = transcoding.getObject("format").getString("protocol");
+                    final AudioStream.Builder builder = new AudioStream.Builder()
+                            .setId(preset);
+
+                    try {
+                        // streamUrl can be either the MP3 progressive stream URL or the
+                        // manifest URL of the HLS MP3 stream (if there is no MP3 progressive
+                        // stream, see above)
+                        final String streamUrl = getTranscodingUrl(url);
+
+                        if (preset.contains("mp3")) {
+                            // Don't add the MP3 HLS stream if there is a progressive stream
+                            // present because the two have the same bitrate
+                            final boolean isHls = protocol.equals("hls");
+                            if (mp3ProgressiveInStreams && isHls) {
+                                return;
+                            }
+
+                            builder.setMediaFormat(MediaFormat.MP3);
+                            builder.setAverageBitrate(128);
+
+                            if (isHls) {
+                                builder.setDeliveryMethod(DeliveryMethod.HLS);
+                                builder.setContent(streamUrl, true);
+
+                                final AudioStream hlsStream = builder.build();
+                                if (!Stream.containSimilarStream(hlsStream, audioStreams)) {
+                                    audioStreams.add(hlsStream);
+                                }
+
+                                final String progressiveHlsUrl =
+                                        getSingleUrlFromHlsManifest(streamUrl);
+                                builder.setDeliveryMethod(DeliveryMethod.PROGRESSIVE_HTTP);
+                                builder.setContent(progressiveHlsUrl, true);
+
+                                final AudioStream progressiveHlsStream = builder.build();
+                                if (!Stream.containSimilarStream(
+                                        progressiveHlsStream, audioStreams)) {
+                                    audioStreams.add(progressiveHlsStream);
+                                }
+
+                                // The MP3 HLS stream has been added in both versions (HLS and
+                                // progressive with the manifest parsing trick), so we need to
+                                // continue (otherwise the code would try to add again the stream,
+                                // which would be not added because the containsSimilarStream
+                                // method would return false and an audio stream object would be
+                                // created for nothing)
+                                return;
+                            } else {
+                                builder.setContent(streamUrl, true);
+                            }
+                        } else if (preset.contains("opus")) {
+                            // The HLS manifest trick doesn't work for opus streams
+                            builder.setContent(streamUrl, true);
+                            builder.setMediaFormat(MediaFormat.OPUS);
+                            builder.setAverageBitrate(64);
+                            builder.setDeliveryMethod(DeliveryMethod.HLS);
+                        } else {
+                            // Unknown format, skip to the next audio stream
+                            return;
+                        }
+
+                        final AudioStream audioStream = builder.build();
+                        if (!Stream.containSimilarStream(audioStream, audioStreams)) {
+                            audioStreams.add(audioStream);
+                        }
+                    } catch (final ExtractionException | IOException ignored) {
+                        // Something went wrong when trying to get and add this audio stream,
+                        // skip to the next one
                     }
-                }
-            } catch (final Exception ignored) {
-                // Something went wrong when parsing this transcoding URL, so don't add it to the
-                // audioStreams
-            }
-        }
+                });
     }
 
     /**
@@ -332,25 +353,28 @@ public void extractDownloadableFileIfAvailable(final List<AudioStream> audioStre
     }
 
     /**
-     * Parses a SoundCloud HLS manifest to get a single URL of HLS streams.
+     * Parses a SoundCloud HLS MP3 manifest to get a single URL of HLS streams.
      *
      * <p>
      * This method downloads the provided manifest URL, finds all web occurrences in the manifest,
      * gets the last segment URL, changes its segment range to {@code 0/track-length}, and return
      * this as a string.
      * </p>
      *
+     * <p>
+     * This was working before for Opus streams, but has been broken by SoundCloud.
+     * </p>
+     *
      * @param  hlsManifestUrl the URL of the manifest to be parsed
      * @return a single URL that contains a range equal to the length of the track
      */
     @Nonnull
     private static String getSingleUrlFromHlsManifest(@Nonnull final String hlsManifestUrl)
             throws ParsingException {
-        final Downloader dl = NewPipe.getDownloader();
         final String hlsManifestResponse;
 
         try {
-            hlsManifestResponse = dl.get(hlsManifestUrl).responseBody();
+            hlsManifestResponse = NewPipe.getDownloader().get(hlsManifestUrl).responseBody();
         } catch (final IOException | ReCaptchaException e) {
             throw new ParsingException("Could not get SoundCloud HLS manifest");
         }
@@ -359,12 +383,13 @@ private static String getSingleUrlFromHlsManifest(@Nonnull final String hlsManif
         for (int l = lines.length - 1; l >= 0; l--) {
             final String line = lines[l];
             // Get the last URL from manifest, because it contains the range of the stream
-            if (line.trim().length() != 0 && !line.startsWith("#") && line.startsWith("https")) {
+            if (line.trim().length() != 0 && !line.startsWith("#") && line.startsWith(HTTPS)) {
                 final String[] hlsLastRangeUrlArray = line.split("/");
                 return HTTPS + hlsLastRangeUrlArray[2] + "/media/0/" + hlsLastRangeUrlArray[5]
                         + "/" + hlsLastRangeUrlArray[6];
             }
         }
+
         throw new ParsingException("Could not get any URL from HLS manifest");
     }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java

@@ -188,25 +188,27 @@ public void testAudioStreams() throws Exception {
             super.testAudioStreams();
             final List<AudioStream> audioStreams = extractor.getAudioStreams();
             assertEquals(2, audioStreams.size());
-            for (final AudioStream audioStream : audioStreams) {
+            audioStreams.forEach(audioStream -> {
                 final DeliveryMethod deliveryMethod = audioStream.getDeliveryMethod();
-                assertSame(DeliveryMethod.PROGRESSIVE_HTTP, deliveryMethod,
-                        "Wrong delivery method for stream " + audioStream.getId() + ": "
-                                + deliveryMethod);
                 final String mediaUrl = audioStream.getContent();
                 if (audioStream.getFormat() == MediaFormat.OPUS) {
                     // Assert that it's an OPUS 64 kbps media URL with a single range which comes
                     // from an HLS SoundCloud CDN
                     ExtractorAsserts.assertContains("-hls-opus-media.sndcdn.com", mediaUrl);
                     ExtractorAsserts.assertContains(".64.opus", mediaUrl);
-                }
-                if (audioStream.getFormat() == MediaFormat.MP3) {
+                    assertSame(DeliveryMethod.HLS, deliveryMethod,
+                            "Wrong delivery method for stream " + audioStream.getId() + ": "
+                                    + deliveryMethod);
+                } else if (audioStream.getFormat() == MediaFormat.MP3) {
                     // Assert that it's a MP3 128 kbps media URL which comes from a progressive
                     // SoundCloud CDN
                     ExtractorAsserts.assertContains("-media.sndcdn.com/bKOA7Pwbut93.128.mp3",
                             mediaUrl);
+                    assertSame(DeliveryMethod.PROGRESSIVE_HTTP, deliveryMethod,
+                            "Wrong delivery method for stream " + audioStream.getId() + ": "
+                                    + deliveryMethod);
                 }
-            }
+            });
         }
     }
 }