Skip to content

Commit 1f7d72a

Browse files
adamintadamint
committed
properly implement request chunking (bulk requesting)
1 parent de53e59 commit 1f7d72a

File tree

11 files changed

+239
-171
lines changed

11 files changed

+239
-171
lines changed

‎src/commonMain/kotlin/com.adamratzman.spotify/SpotifyApi.kt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ internal const val base = "https://api.spotify.com/v1"
5959
* @property json The Json serializer/deserializer instance
6060
* @property logger The Spotify event logger
6161
* @property requestTimeoutMillis The maximum time, in milliseconds, before terminating an http request
62+
* @property allowBulkRequests Allow splitting too-large requests into smaller, allowable api requests
6263
*
6364
*/
6465
sealed class SpotifyApi<T : SpotifyApi<T, B>, B : ISpotifyApiBuilder<T, B>>(

‎src/commonMain/kotlin/com.adamratzman.spotify/SpotifyRestAction.kt

Lines changed: 29 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,6 @@ import com.adamratzman.spotify.utils.TimeUnit
77
import com.adamratzman.spotify.utils.getCurrentTimeMs
88
import com.adamratzman.spotify.utils.runBlocking
99
import com.adamratzman.spotify.utils.schedule
10-
import kotlin.coroutines.CoroutineContext
11-
import kotlin.coroutines.resume
12-
import kotlin.coroutines.resumeWithException
13-
import kotlin.coroutines.suspendCoroutine
14-
import kotlin.jvm.JvmOverloads
1510
import kotlinx.coroutines.CancellationException
1611
import kotlinx.coroutines.CoroutineScope
1712
import kotlinx.coroutines.Dispatchers
@@ -26,6 +21,11 @@ import kotlinx.coroutines.flow.flow
2621
import kotlinx.coroutines.flow.flowOn
2722
import kotlinx.coroutines.launch
2823
import kotlinx.coroutines.withContext
24+
import kotlin.coroutines.CoroutineContext
25+
import kotlin.coroutines.resume
26+
import kotlin.coroutines.resumeWithException
27+
import kotlin.coroutines.suspendCoroutine
28+
import kotlin.jvm.JvmOverloads
2929

3030
/**
3131
* Provides a uniform interface to retrieve, whether synchronously or asynchronously, [T] from Spotify
@@ -114,11 +114,11 @@ open class SpotifyRestAction<T> internal constructor(protected val api: SpotifyA
114114
*/
115115
@JvmOverloads
116116
fun queueAfter(
117-
quantity: Int,
118-
timeUnit: TimeUnit = TimeUnit.SECONDS,
119-
scope: CoroutineScope = GlobalScope,
120-
failure: (Throwable) -> Unit = { throw it },
121-
consumer: (T) -> Unit
117+
quantity: Int,
118+
timeUnit: TimeUnit = TimeUnit.SECONDS,
119+
scope: CoroutineScope = GlobalScope,
120+
failure: (Throwable) -> Unit = { throw it },
121+
consumer: (T) -> Unit
122122
): SpotifyRestAction<T> {
123123
val runAt = getCurrentTimeMs() + timeUnit.toMillis(quantity.toLong())
124124
queue({ exception ->
@@ -133,7 +133,7 @@ open class SpotifyRestAction<T> internal constructor(protected val api: SpotifyA
133133
}
134134

135135
class SpotifyRestActionPaging<Z : Any, T : AbstractPagingObject<Z>>(api: SpotifyApi<*, *>, supplier: suspend () -> T) :
136-
SpotifyRestAction<T>(api, supplier) {
136+
SpotifyRestAction<T>(api, supplier) {
137137

138138
/**
139139
* Synchronously retrieve all [AbstractPagingObject] associated with this rest action
@@ -162,8 +162,10 @@ class SpotifyRestActionPaging<Z : Any, T : AbstractPagingObject<Z>>(api: Spotify
162162
* Synchronously retrieve all [Z] associated with this rest action
163163
*/
164164
fun getAllItems(context: CoroutineContext = Dispatchers.Default) =
165-
api.tracks.toAction { suspendComplete(context)
166-
.getAllImpl().toList().map { it.items }.flatten() }
165+
api.tracks.toAction {
166+
suspendComplete(context)
167+
.getAllImpl().toList().map { it.items }.flatten()
168+
}
167169

168170
/**
169171
* Consume each [Z] by [consumer] as it is retrieved
@@ -190,13 +192,13 @@ class SpotifyRestActionPaging<Z : Any, T : AbstractPagingObject<Z>>(api: Spotify
190192
@JvmOverloads
191193
@ExperimentalCoroutinesApi
192194
fun flowPagingObjectsOrdered(context: CoroutineContext = Dispatchers.Default): Flow<AbstractPagingObject<Z>> =
193-
flow {
194-
suspendComplete(context).also { master ->
195-
emitAll(master.flowStartOrdered())
196-
emit(master)
197-
emitAll(master.flowEndOrdered())
198-
}
199-
}.flowOn(context)
195+
flow {
196+
suspendComplete(context).also { master ->
197+
emitAll(master.flowStartOrdered())
198+
emit(master)
199+
emitAll(master.flowEndOrdered())
200+
}
201+
}.flowOn(context)
200202

201203
/**
202204
* Flow the Paging action.
@@ -214,11 +216,11 @@ class SpotifyRestActionPaging<Z : Any, T : AbstractPagingObject<Z>>(api: Spotify
214216
@JvmOverloads
215217
@ExperimentalCoroutinesApi
216218
fun flowPagingObjects(context: CoroutineContext = Dispatchers.Default): Flow<AbstractPagingObject<Z>> =
217-
flow {
218-
suspendComplete(context).also { master ->
219-
emitAll(master.flowBackward())
220-
emit(master)
221-
emitAll(master.flowForward())
222-
}
223-
}.flowOn(context)
219+
flow {
220+
suspendComplete(context).also { master ->
221+
emitAll(master.flowBackward())
222+
emit(master)
223+
emitAll(master.flowForward())
224+
}
225+
}.flowOn(context)
224226
}

‎src/commonMain/kotlin/com.adamratzman.spotify/endpoints/client/ClientFollowingApi.kt

Lines changed: 28 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -84,11 +84,14 @@ class ClientFollowingApi(api: SpotifyApi<*, *>) : FollowingApi(api) {
8484
* @return A list of booleans corresponding to [users] of whether the current user is following that user
8585
*/
8686
fun isFollowingUsers(vararg users: String): SpotifyRestAction<List<Boolean>> {
87+
checkBulkRequesting(50, users.size)
8788
return toAction {
88-
get(
89-
EndpointBuilder("/me/following/contains").with("type", "user")
90-
.with("ids", users.joinToString(",") { UserUri(it).id.encodeUrl() }).toString()
91-
).toList(Boolean.serializer().list, api, json)
89+
bulkRequest(50, users.toList()) { chunk ->
90+
get(
91+
EndpointBuilder("/me/following/contains").with("type", "user")
92+
.with("ids", chunk.joinToString(",") { UserUri(it).id.encodeUrl() }).toString()
93+
).toList(Boolean.serializer().list, api, json)
94+
}.flatten()
9295
}
9396
}
9497

@@ -123,11 +126,14 @@ class ClientFollowingApi(api: SpotifyApi<*, *>) : FollowingApi(api) {
123126
* @return A list of booleans corresponding to [artists] of whether the current user is following that artist
124127
*/
125128
fun isFollowingArtists(vararg artists: String): SpotifyRestAction<List<Boolean>> {
129+
checkBulkRequesting(50, artists.size)
126130
return toAction {
127-
get(
128-
EndpointBuilder("/me/following/contains").with("type", "artist")
129-
.with("ids", artists.joinToString(",") { ArtistUri(it).id.encodeUrl() }).toString()
130-
).toList(Boolean.serializer().list, api, json)
131+
bulkRequest(50, artists.toList()) { chunk ->
132+
get(
133+
EndpointBuilder("/me/following/contains").with("type", "artist")
134+
.with("ids", chunk.joinToString(",") { ArtistUri(it).id.encodeUrl() }).toString()
135+
).toList(Boolean.serializer().list, api, json)
136+
}.flatten()
131137
}
132138
}
133139

@@ -145,8 +151,8 @@ class ClientFollowingApi(api: SpotifyApi<*, *>) : FollowingApi(api) {
145151
* with full [Artist] objects
146152
*/
147153
fun getFollowedArtists(
148-
limit: Int? = api.defaultLimit,
149-
after: String? = null
154+
limit: Int? = api.defaultLimit,
155+
after: String? = null
150156
): SpotifyRestActionPaging<Artist, CursorBasedPagingObject<Artist>> {
151157
return toActionPaging {
152158
get(
@@ -185,14 +191,15 @@ class ClientFollowingApi(api: SpotifyApi<*, *>) : FollowingApi(api) {
185191
* @throws BadRequestException if an invalid id is provided
186192
*/
187193
fun followUsers(vararg users: String): SpotifyRestAction<Unit> {
188-
if (users.size > 50 && !api.allowBulkRequests) throw BadRequestException("Too many users (${users.size}) provided, only 50 allowed", IllegalArgumentException("Bulk requests are not turned on, and too many users were provided"))
194+
checkBulkRequesting(50, users.size)
189195
return toAction {
190-
users.toList().chunked(50).forEach { list ->
196+
bulkRequest(50, users.toList()) { chunk ->
191197
put(
192198
EndpointBuilder("/me/following").with("type", "user")
193-
.with("ids", list.joinToString(",") { UserUri(it).id.encodeUrl() }).toString()
199+
.with("ids", chunk.joinToString(",") { UserUri(it).id.encodeUrl() }).toString()
194200
)
195201
}
202+
196203
Unit
197204
}
198205
}
@@ -224,14 +231,15 @@ class ClientFollowingApi(api: SpotifyApi<*, *>) : FollowingApi(api) {
224231
* @throws BadRequestException if an invalid id is provided
225232
*/
226233
fun followArtists(vararg artists: String): SpotifyRestAction<Unit> {
227-
if (artists.size > 50 && !api.allowBulkRequests) throw BadRequestException("Too many artists (${artists.size}) provided, only 50 allowed", IllegalArgumentException("Bulk requests are not turned on, and too many artists were provided"))
234+
checkBulkRequesting(50, artists.size)
228235
return toAction {
229-
artists.toList().chunked(50).forEach { list ->
236+
bulkRequest(50, artists.toList()) { chunk ->
230237
put(
231238
EndpointBuilder("/me/following").with("type", "artist")
232-
.with("ids", list.joinToString(",") { ArtistUri(it).id.encodeUrl() }).toString()
239+
.with("ids", chunk.joinToString(",") { ArtistUri(it).id.encodeUrl() }).toString()
233240
)
234241
}
242+
235243
Unit
236244
}
237245
}
@@ -294,9 +302,9 @@ class ClientFollowingApi(api: SpotifyApi<*, *>) : FollowingApi(api) {
294302
* @throws BadRequestException if an invalid id is provided
295303
*/
296304
fun unfollowUsers(vararg users: String): SpotifyRestAction<Unit> {
297-
if (users.size > 50 && !api.allowBulkRequests) throw BadRequestException("Too many users (${users.size}) provided, only 50 allowed", IllegalArgumentException("Bulk requests are not turned on, and too many users were provided"))
305+
checkBulkRequesting(50, users.size)
298306
return toAction {
299-
users.toList().chunked(50).forEach { list ->
307+
bulkRequest(50, users.toList()) { list ->
300308
delete(
301309
EndpointBuilder("/me/following").with("type", "user")
302310
.with("ids", list.joinToString(",") { UserUri(it).id.encodeUrl() }).toString()
@@ -336,9 +344,9 @@ class ClientFollowingApi(api: SpotifyApi<*, *>) : FollowingApi(api) {
336344
* @throws BadRequestException if an invalid id is provided
337345
*/
338346
fun unfollowArtists(vararg artists: String): SpotifyRestAction<Unit> {
339-
if (artists.size > 50 && !api.allowBulkRequests) throw BadRequestException("Too many artists (${artists.size}) provided, only 50 allowed", IllegalArgumentException("Bulk requests are not turned on, and too many artists were provided"))
347+
checkBulkRequesting(50, artists.size)
340348
return toAction {
341-
artists.toList().chunked(50).forEach { list ->
349+
bulkRequest(50, artists.toList()) { list ->
342350
delete(
343351
EndpointBuilder("/me/following").with("type", "artist")
344352
.with("ids", list.joinToString(",") { ArtistUri(it).id.encodeUrl() }).toString()

‎src/commonMain/kotlin/com.adamratzman.spotify/endpoints/client/ClientPlaylistApi.kt

Lines changed: 29 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -113,16 +113,17 @@ class ClientPlaylistApi(api: SpotifyApi<*, *>) : PlaylistApi(api) {
113113
* **[Api Reference](https://developer.spotify.com/documentation/web-api/reference/playlists/add-tracks-to-playlist/)**
114114
*
115115
* @param playlist The spotify id or uri for the playlist.
116-
* @param tracks Spotify track ids.
116+
* @param tracks Spotify track ids. Maximum 100
117117
* @param position The position to insert the tracks, a zero-based index. For example, to insert the tracks in the
118118
* first position: position=0; to insert the tracks in the third position: position=2. If omitted, the tracks will
119119
* be appended to the playlist. Tracks are added in the order they are listed in the query string or request body.
120120
*
121121
* @throws BadRequestException if any invalid track ids is provided or the playlist is not found
122122
*/
123123
fun addTracksToClientPlaylist(playlist: String, vararg tracks: String, position: Int? = null): SpotifyRestAction<Unit> {
124+
checkBulkRequesting(100, tracks.size)
124125
return toAction {
125-
chunk(100, tracks.toList()) { chunk ->
126+
bulkRequest(100, tracks.toList()) { chunk ->
126127
val body = jsonMap()
127128
body += json { "uris" to JsonArray(chunk.map { TrackUri(TrackUri(it).id.encodeUrl()).uri }.map(::JsonPrimitive)) }
128129
if (position != null) body += json { "position" to position }
@@ -131,6 +132,8 @@ class ClientPlaylistApi(api: SpotifyApi<*, *>) : PlaylistApi(api) {
131132
body.toJson()
132133
)
133134
}
135+
136+
Unit
134137
}
135138

136139
}
@@ -429,7 +432,7 @@ class ClientPlaylistApi(api: SpotifyApi<*, *>) : PlaylistApi(api) {
429432
) = removePlaylistTracksImpl(playlist, tracks.map { it to null }.toTypedArray(), snapshotId)
430433

431434
/**
432-
* Remove tracks (each with their own positions) from the given playlist.
435+
* Remove tracks (each with their own positions) from the given playlist. **Bulk requesting is only available when [snapshotId] is null.**
433436
*
434437
* Removing tracks from a user’s public playlist requires authorization of the [SpotifyScope.PLAYLIST_MODIFY_PUBLIC] scope;
435438
* removing tracks from a private playlist requires the [SpotifyScope.PLAYLIST_MODIFY_PRIVATE] scope.
@@ -451,27 +454,30 @@ class ClientPlaylistApi(api: SpotifyApi<*, *>) : PlaylistApi(api) {
451454
tracks: Array<Pair<String, SpotifyTrackPositions?>>,
452455
snapshotId: String?
453456
): SpotifyRestAction<PlaylistSnapshot> {
454-
return toAction {
455-
require(tracks.isNotEmpty()) { "You need to provide at least one track to remove" }
457+
checkBulkRequesting(100,tracks.size)
458+
if (snapshotId != null && tracks.size > 100) throw BadRequestException("You cannot provide both the snapshot id and attempt bulk requesting")
456459

457-
val body = jsonMap()
458-
if (snapshotId != null) body += json { "snapshot_id" to snapshotId }
459-
body += json {
460-
"tracks" to JsonArray(
461-
tracks.map { (track, positions) ->
462-
val json = jsonMap()
463-
json += json { "uri" to TrackUri(track).uri }
464-
if (positions?.positions?.isNotEmpty() == true) json += json {
465-
"positions" to JsonArray(
466-
positions.positions.map(::JsonPrimitive)
467-
)
468-
}
469-
JsonObject(json)
470-
})
471-
}
472-
delete(
473-
EndpointBuilder("/playlists/${PlaylistUri(playlist).id}/tracks").toString(), body = body.toJson()
474-
).toObject(PlaylistSnapshot.serializer(), api, json)
460+
return toAction {
461+
bulkRequest(100, tracks.toList()) { chunk ->
462+
val body = jsonMap()
463+
if (snapshotId != null) body += json { "snapshot_id" to snapshotId }
464+
body += json {
465+
"tracks" to JsonArray(
466+
chunk.map { (track, positions) ->
467+
val json = jsonMap()
468+
json += json { "uri" to TrackUri(track).uri }
469+
if (positions?.positions?.isNotEmpty() == true) json += json {
470+
"positions" to JsonArray(
471+
positions.positions.map(::JsonPrimitive)
472+
)
473+
}
474+
JsonObject(json)
475+
})
476+
}
477+
delete(
478+
EndpointBuilder("/playlists/${PlaylistUri(playlist).id}/tracks").toString(), body = body.toJson()
479+
).toObject(PlaylistSnapshot.serializer(), api, json)
480+
}.last()
475481
}
476482
}
477483
}

‎src/commonMain/kotlin/com.adamratzman.spotify/endpoints/public/AlbumApi.kt

Lines changed: 20 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,10 @@ class AlbumApi(api: SpotifyApi<*, *>) : SpotifyEndpoint(api) {
4141
return toAction {
4242
catch {
4343
get(
44-
EndpointBuilder("/albums/${AlbumUri(album).id}").with(
45-
"market",
46-
market?.name
47-
).toString()
44+
EndpointBuilder("/albums/${AlbumUri(album).id}").with(
45+
"market",
46+
market?.name
47+
).toString()
4848
).toObject(Album.serializer(), api, json)
4949
}
5050
}
@@ -62,11 +62,14 @@ class AlbumApi(api: SpotifyApi<*, *>) : SpotifyEndpoint(api) {
6262
* @return List of [Album] objects or null if the album could not be found, in the order requested
6363
*/
6464
fun getAlbums(vararg albums: String, market: Market? = null): SpotifyRestAction<List<Album?>> {
65+
checkBulkRequesting(20, albums.size)
6566
return toAction {
66-
get(
67-
EndpointBuilder("/albums").with("ids", albums.joinToString(",") { AlbumUri(it).id.encodeUrl() })
68-
.with("market", market?.name).toString()
69-
).toObject(AlbumsResponse.serializer(), api, json).albums
67+
bulkRequest(20, albums.toList()) { chunk ->
68+
get(
69+
EndpointBuilder("/albums").with("ids", chunk.joinToString(",") { AlbumUri(it).id.encodeUrl() })
70+
.with("market", market?.name).toString()
71+
).toObject(AlbumsResponse.serializer(), api, json).albums
72+
}.flatten()
7073
}
7174
}
7275

@@ -84,18 +87,18 @@ class AlbumApi(api: SpotifyApi<*, *>) : SpotifyEndpoint(api) {
8487
* @return [PagingObject] of [SimpleTrack] objects
8588
*/
8689
fun getAlbumTracks(
87-
album: String,
88-
limit: Int? = api.defaultLimit,
89-
offset: Int? = null,
90-
market: Market? = null
90+
album: String,
91+
limit: Int? = api.defaultLimit,
92+
offset: Int? = null,
93+
market: Market? = null
9194
): SpotifyRestActionPaging<SimpleTrack, PagingObject<SimpleTrack>> {
9295
return toActionPaging {
9396
get(
94-
EndpointBuilder("/albums/${AlbumUri(album).id.encodeUrl()}/tracks").with("limit", limit).with(
95-
"offset",
96-
offset
97-
).with("market", market?.name)
98-
.toString()
97+
EndpointBuilder("/albums/${AlbumUri(album).id.encodeUrl()}/tracks").with("limit", limit).with(
98+
"offset",
99+
offset
100+
).with("market", market?.name)
101+
.toString()
99102
).toPagingObject(SimpleTrack.serializer(), endpoint = this, json = json)
100103
}
101104
}

‎src/commonMain/kotlin/com.adamratzman.spotify/endpoints/public/ArtistApi.kt

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -64,12 +64,16 @@ class ArtistApi(api: SpotifyApi<*, *>) : SpotifyEndpoint(api) {
6464
* @return List of [Artist] objects or null if the artist could not be found, in the order requested
6565
*/
6666
fun getArtists(vararg artists: String): SpotifyRestAction<List<Artist?>> {
67+
checkBulkRequesting(50, artists.size)
68+
6769
return toAction {
68-
get(
69-
EndpointBuilder("/artists").with(
70-
"ids",
71-
artists.joinToString(",") { ArtistUri(it).id.encodeUrl() }).toString()
72-
).toObject(ArtistList.serializer(), api, json).artists
70+
bulkRequest(50, artists.toList()) { chunk ->
71+
get(
72+
EndpointBuilder("/artists").with(
73+
"ids",
74+
artists.joinToString(",") { ArtistUri(it).id.encodeUrl() }).toString()
75+
).toObject(ArtistList.serializer(), api, json).artists
76+
}.flatten()
7377
}
7478
}
7579

0 commit comments

Comments
 (0)