Skip to content

Commit 072e977

Browse files
absurdlylongusernameStypox
absurdlylongusername
authored and
Stypox
committed
[SoundCloud] Refactor Soundcloud audio stream extraction code to separate building Hls and Progressive streams
Add HlsAudioStream to facilitate refreshing expired Hls playlists
1 parent 95ea906 commit 072e977

File tree

5 files changed

+284
-51
lines changed

5 files changed

+284
-51
lines changed

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java‎

Lines changed: 117 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,9 @@
1111

1212
import com.grack.nanojson.JsonArray;
1313
import com.grack.nanojson.JsonObject;
14-
import com.grack.nanojson.JsonParser;
15-
import com.grack.nanojson.JsonParserException;
1614

1715
import org.schabi.newpipe.extractor.Image;
1816
import org.schabi.newpipe.extractor.MediaFormat;
19-
import org.schabi.newpipe.extractor.NewPipe;
2017
import org.schabi.newpipe.extractor.StreamingService;
2118
import org.schabi.newpipe.extractor.downloader.Downloader;
2219
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
@@ -30,6 +27,8 @@
3027
import org.schabi.newpipe.extractor.stream.AudioStream;
3128
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
3229
import org.schabi.newpipe.extractor.stream.Description;
30+
import org.schabi.newpipe.extractor.stream.HlsAudioStream;
31+
import org.schabi.newpipe.extractor.stream.SoundcloudHlsUtils;
3332
import org.schabi.newpipe.extractor.stream.Stream;
3433
import org.schabi.newpipe.extractor.stream.StreamExtractor;
3534
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
@@ -190,25 +189,106 @@ public List<AudioStream> getAudioStreams() throws ExtractionException {
190189
return audioStreams;
191190
}
192191

193-
@Nonnull
194-
private String getTranscodingUrl(final String endpointUrl)
195-
throws IOException, ExtractionException {
196-
String apiStreamUrl = endpointUrl + "?client_id=" + clientId();
192+
// TODO: put this somewhere better
193+
/**
194+
* Constructs the API endpoint url for this track that will be called to get the url
195+
* for retrieving the actual byte data for playback
196+
* (e.g. the actual url could be an m3u8 playlist)
197+
* @param baseUrl The baseUrl needed to construct the full url
198+
* @return The full API endpoint url to call to get the actual playback url
199+
* @throws IOException If there is a problem getting clientId
200+
* @throws ExtractionException For the same reason
201+
*/
202+
private String getApiStreamUrl(final String baseUrl) throws ExtractionException, IOException {
203+
String apiStreamUrl = baseUrl + "?client_id=" + clientId();
197204

198205
final String trackAuthorization = track.getString("track_authorization");
199206
if (!isNullOrEmpty(trackAuthorization)) {
200207
apiStreamUrl += "&track_authorization=" + trackAuthorization;
201208
}
209+
return apiStreamUrl;
210+
}
202211

203-
final String response = NewPipe.getDownloader().get(apiStreamUrl).responseBody();
204-
final JsonObject urlObject;
205-
try {
206-
urlObject = JsonParser.object().from(response);
207-
} catch (final JsonParserException e) {
208-
throw new ParsingException("Could not parse streamable URL", e);
212+
public static final class StreamBuildResult {
213+
public final String contentUrl;
214+
public final MediaFormat mediaFormat;
215+
216+
public StreamBuildResult(final String contentUrl, final MediaFormat mediaFormat) {
217+
this.contentUrl = contentUrl;
218+
this.mediaFormat = mediaFormat;
219+
}
220+
221+
@Override
222+
public String toString() {
223+
return "StreamBuildResult{"
224+
+ "contentUrl='" + contentUrl + '\''
225+
+ ", mediaFormat=" + mediaFormat
226+
+ '}';
227+
}
228+
}
229+
230+
/**
231+
* Builds the common audio stream components for all SoundCloud audio streams<p>
232+
* Returns the stream content url if we support this type of transcoding, {@code null} otherwise
233+
* @param transcoding The SoundCloud JSON transcoding object for this stream
234+
* @param builder AudioStream builder to set the common values
235+
* @return the stream content url if this transcoding is supported and common values were built
236+
* {@code null} otherwise
237+
*/
238+
@Nullable
239+
private StreamBuildResult buildBaseAudioStream(final JsonObject transcoding,
240+
final AudioStream.Builder builder)
241+
throws ExtractionException, IOException {
242+
ExtractorLogger.d(TAG, getName() + " Building base audio stream info");
243+
final var preset = transcoding.getString("preset", ID_UNKNOWN);
244+
final MediaFormat mediaFormat;
245+
if (preset.contains("mp3")) {
246+
mediaFormat = MediaFormat.MP3;
247+
builder.setAverageBitrate(128);
248+
} else if (preset.contains("opus")) {
249+
mediaFormat = MediaFormat.OPUS;
250+
builder.setAverageBitrate(64);
251+
builder.setDeliveryMethod(DeliveryMethod.HLS);
252+
} else if (preset.contains("aac_160k")) {
253+
mediaFormat = MediaFormat.M4A;
254+
builder.setAverageBitrate(160);
255+
} else {
256+
// Unknown format, return null to skip to the next audio stream
257+
return null;
209258
}
210259

211-
return urlObject.getString("url");
260+
builder.setMediaFormat(mediaFormat);
261+
262+
builder.setId(preset);
263+
final var url = transcoding.getString("url");
264+
final var hlsPlaylistUrl = SoundcloudHlsUtils.getStreamContentUrl(getApiStreamUrl(url));
265+
builder.setContent(hlsPlaylistUrl, true);
266+
return new StreamBuildResult(hlsPlaylistUrl, mediaFormat);
267+
}
268+
269+
private HlsAudioStream buildHlsAudioStream(final JsonObject transcoding)
270+
throws ExtractionException, IOException {
271+
ExtractorLogger.d(TAG, getName() + "Extracting hls audio stream");
272+
final var builder = new HlsAudioStream.Builder();
273+
final StreamBuildResult buildResult = buildBaseAudioStream(transcoding, builder);
274+
if (buildResult == null) {
275+
return null;
276+
}
277+
278+
builder.setApiStreamUrl(getApiStreamUrl(transcoding.getString("url")));
279+
builder.setPlaylistId(SoundcloudHlsUtils.extractHlsPlaylistId(buildResult.contentUrl,
280+
buildResult.mediaFormat));
281+
282+
return builder.build();
283+
}
284+
285+
private AudioStream buildProgressiveAudioStream(final JsonObject transcoding)
286+
throws ExtractionException, IOException {
287+
ExtractorLogger.d(TAG, getName() + "Extracting progressive audio stream");
288+
final var builder = new AudioStream.Builder();
289+
final StreamBuildResult buildResult = buildBaseAudioStream(transcoding, builder);
290+
return buildResult == null ? null : builder.build();
291+
// TODO: anything else?
212292
}
213293

214294
private void extractAudioStreams(@Nonnull final JsonArray transcodings,
@@ -222,47 +302,35 @@ private void extractAudioStreams(@Nonnull final JsonArray transcodings,
222302
return;
223303
}
224304

225-
try {
226-
final String preset = transcoding.getString("preset", ID_UNKNOWN);
227-
final String protocol = transcoding.getObject("format")
228-
.getString("protocol");
229-
230-
if (protocol.contains("encrypted")) {
231-
// Skip DRM-protected streams, which have encrypted in their protocol
232-
// name
233-
return;
234-
}
235-
236-
final AudioStream.Builder builder = new AudioStream.Builder()
237-
.setId(preset);
305+
final String protocol = transcoding.getObject("format")
306+
.getString("protocol");
238307

239-
if (protocol.equals("hls")) {
240-
builder.setDeliveryMethod(DeliveryMethod.HLS);
241-
}
242-
243-
builder.setContent(getTranscodingUrl(url), true);
244-
245-
if (preset.contains("mp3")) {
246-
builder.setMediaFormat(MediaFormat.MP3);
247-
builder.setAverageBitrate(128);
248-
} else if (preset.contains("opus")) {
249-
builder.setMediaFormat(MediaFormat.OPUS);
250-
builder.setAverageBitrate(64);
251-
} else if (preset.contains("aac_160k")) {
252-
builder.setMediaFormat(MediaFormat.M4A);
253-
builder.setAverageBitrate(160);
254-
} else {
255-
// Unknown format, skip to the next audio stream
256-
return;
257-
}
308+
if (protocol.contains("encrypted")) {
309+
// Skip DRM-protected streams, which have encrypted in their protocol
310+
// name
311+
return;
312+
}
258313

259-
final AudioStream audioStream = builder.build();
260-
if (!Stream.containSimilarStream(audioStream, audioStreams)) {
314+
final AudioStream audioStream;
315+
try {
316+
// SoundCloud only has one progressive stream, the rest are HLS
317+
audioStream = protocol.equals("hls")
318+
? buildHlsAudioStream(transcoding)
319+
: buildProgressiveAudioStream(transcoding);
320+
if (audioStream != null
321+
&& !Stream.containSimilarStream(audioStream, audioStreams)) {
322+
ExtractorLogger.d(TAG, audioStream.getFormat().getName() + " "
323+
+ getName() + " " + audioStream.getContent());
261324
audioStreams.add(audioStream);
262325
}
263-
} catch (final ExtractionException | IOException ignored) {
326+
} catch (final ExtractionException | IOException e) {
264327
// Something went wrong when trying to get and add this audio stream,
265328
// skip to the next one
329+
final var preset = transcoding.getString("preset", "unknown");
330+
ExtractorLogger.e(TAG,
331+
getName() + " Failed to extract audio stream for transcoding "
332+
+ '[' + protocol + '/' + preset + "] " + url,
333+
e);
266334
}
267335
});
268336
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
import java.util.Locale;
2929
import java.util.Objects;
3030

31-
public final class AudioStream extends Stream {
31+
public class AudioStream extends Stream {
3232
public static final int UNKNOWN_BITRATE = -1;
3333

3434
private final int averageBitrate;
@@ -60,7 +60,7 @@ public final class AudioStream extends Stream {
6060
* Class to build {@link AudioStream} objects.
6161
*/
6262
@SuppressWarnings("checkstyle:hiddenField")
63-
public static final class Builder {
63+
public static class Builder {
6464
private String id;
6565
private String content;
6666
private boolean isUrl;

‎extractor/src/main/java/org/schabi/newpipe/extractor/stream/HlsAudioStream.java‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package org.schabi.newpipe.extractor.stream;
2+
3+
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
4+
5+
import java.io.IOException;
6+
7+
import javax.annotation.Nonnull;
8+
9+
public class HlsAudioStream extends AudioStream implements RefreshableStream {
10+
private final String apiStreamUrl;
11+
private final String playlistId;
12+
13+
HlsAudioStream(final Builder builder) {
14+
super(builder);
15+
apiStreamUrl = builder.apiStreamUrl;
16+
playlistId = builder.playlistId;
17+
}
18+
19+
@Nonnull
20+
public String fetchLatestUrl() throws IOException, ExtractionException {
21+
return SoundcloudHlsUtils.getStreamContentUrl(apiStreamUrl);
22+
}
23+
24+
@Nonnull
25+
public String initialUrl() {
26+
return getContent();
27+
}
28+
29+
@Override
30+
public String playlistId() {
31+
return playlistId;
32+
}
33+
34+
@SuppressWarnings({"checkstyle:HiddenField", "UnusedReturnValue"})
35+
public static class Builder extends AudioStream.Builder {
36+
private String apiStreamUrl;
37+
private String playlistId;
38+
39+
public Builder() {
40+
setDeliveryMethod(DeliveryMethod.HLS);
41+
}
42+
43+
@Override
44+
@Nonnull
45+
public HlsAudioStream build() {
46+
validateBuild();
47+
return new HlsAudioStream(this);
48+
}
49+
50+
public Builder setApiStreamUrl(@Nonnull final String apiStreamUrl) {
51+
this.apiStreamUrl = apiStreamUrl;
52+
return this;
53+
}
54+
55+
public Builder setPlaylistId(@Nonnull final String playlistId) {
56+
this.playlistId = playlistId;
57+
return this;
58+
}
59+
}
60+
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/stream/RefreshableStream.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package org.schabi.newpipe.extractor.stream;
2+
3+
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
4+
5+
import javax.annotation.Nonnull;
6+
import java.io.IOException;
7+
8+
@SuppressWarnings("checkstyle:LeftCurly")
9+
public interface RefreshableStream {
10+
@Nonnull
11+
String fetchLatestUrl() throws IOException, ExtractionException;
12+
String initialUrl();
13+
14+
String playlistId();
15+
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/stream/SoundcloudHlsUtils.java‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
package org.schabi.newpipe.extractor.stream;
2+
3+
import com.grack.nanojson.JsonObject;
4+
import com.grack.nanojson.JsonParser;
5+
import com.grack.nanojson.JsonParserException;
6+
7+
import org.schabi.newpipe.extractor.MediaFormat;
8+
import org.schabi.newpipe.extractor.NewPipe;
9+
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
10+
import org.schabi.newpipe.extractor.exceptions.ParsingException;
11+
import org.schabi.newpipe.extractor.utils.ExtractorLogger;
12+
import org.schabi.newpipe.extractor.utils.Parser;
13+
14+
import java.io.IOException;
15+
import java.util.regex.Pattern;
16+
17+
import javax.annotation.Nonnull;
18+
19+
public final class SoundcloudHlsUtils {
20+
private static final String TAG = HlsAudioStream.class.getSimpleName();
21+
private static final Pattern MP3_HLS_PATTERN =
22+
Pattern.compile("https://cf-hls-media\\.sndcdn.com/playlist/"
23+
+ "([a-zA-Z0-9]+)\\.128\\.mp3/playlist\\.m3u8");
24+
private static final Pattern AAC_HLS_PATTERN =
25+
Pattern.compile("https://playback\\.media-streaming\\.soundcloud\\.cloud/"
26+
+ "([a-zA-Z0-9]+)/aac_160k/[a-f0-9\\-]+/playlist\\.m3u8");
27+
private static final Pattern OPUS_HLS_PATTERN =
28+
Pattern.compile("https://cf-hls-opus-media\\.sndcdn\\.com/"
29+
+ "playlist/([a-zA-Z0-9]+)\\.64\\.opus/playlist\\.m3u8");
30+
31+
private SoundcloudHlsUtils() { }
32+
33+
/**
34+
* Calls the API endpoint url for this stream to get the url for retrieving the
35+
* actual byte data for playback (returns the m3u8 playlist url for HLS streams,
36+
* and the url to get the full binary track for progressives streams)<p>
37+
*
38+
* NOTE: this returns a different url every time! (for SoundCloud)
39+
* @param apiStreamUrl The url to call to get the actual stream data url
40+
* @return The url for playing the audio (e.g. playlist.m3u8)
41+
* @throws IOException If there's a problem calling the endpoint
42+
* @throws ExtractionException for the same reason
43+
*/
44+
public static String getStreamContentUrl(final String apiStreamUrl)
45+
throws IOException, ExtractionException {
46+
ExtractorLogger.d(TAG, "Fetching content url for " + apiStreamUrl);
47+
final String response = NewPipe.getDownloader()
48+
.get(apiStreamUrl)
49+
.validateResponseCode()
50+
.responseBody();
51+
final JsonObject urlObject;
52+
try {
53+
urlObject = JsonParser.object().from(response);
54+
} catch (final JsonParserException e) {
55+
// TODO: Improve error message.
56+
throw new ParsingException("Could not parse stream content from URL ("
57+
+ response + ")", e);
58+
}
59+
60+
return urlObject.getString("url");
61+
}
62+
63+
@Nonnull
64+
public static String extractHlsPlaylistId(final String hlsPlaylistUrl,
65+
final MediaFormat mediaFormat)
66+
throws ExtractionException {
67+
switch (mediaFormat) {
68+
case MP3: return extractHlsMp3PlaylistId(hlsPlaylistUrl);
69+
case M4A: return extractHlsAacPlaylistId(hlsPlaylistUrl);
70+
case OPUS: return extractHlsOpusPlaylistId(hlsPlaylistUrl);
71+
default:
72+
throw new IllegalArgumentException("Unsupported media format: " + mediaFormat);
73+
}
74+
}
75+
76+
private static String extractHlsMp3PlaylistId(final String hlsPlaylistUrl)
77+
throws ExtractionException {
78+
return Parser.matchGroup1(MP3_HLS_PATTERN, hlsPlaylistUrl);
79+
}
80+
81+
private static String extractHlsAacPlaylistId(final String hlsPlaylistUrl)
82+
throws ExtractionException {
83+
return Parser.matchGroup1(AAC_HLS_PATTERN, hlsPlaylistUrl);
84+
}
85+
86+
private static String extractHlsOpusPlaylistId(final String hlsPlaylistUrl)
87+
throws ExtractionException {
88+
return Parser.matchGroup1(OPUS_HLS_PATTERN, hlsPlaylistUrl);
89+
}
90+
}

0 commit comments

Comments
 (0)