Skip to content

Commit 0313c0a

Browse files
committed
perf(model-client): stricter limit of the body size in pushObjects
The body was previously limit by the number of entries, but some entries are bigger than others. In some cases the body was too big causing a `413 Request Entity Too Large`. Now that actual serialized size of the entries is used to split them into chunks.
1 parent 88bd1a8 commit 0313c0a

File tree

1 file changed

+19
-8
lines changed

1 file changed

+19
-8
lines changed

model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -322,22 +322,33 @@ class ModelClientV2(
322322

323323
override suspend fun pushObjects(repository: RepositoryId, objects: Sequence<ObjectHashAndSerializedObject>) {
324324
LOG.debug { "${clientId.toString(16)}.pushObjects($repository)" }
325-
objects.chunked(100_000).forEach { unsortedChunk ->
326-
// Entries are sorted to avoid deadlocks on the server side between transactions.
327-
// Since ignite locks individual entries, this is equivalent to a lock ordering.
328-
// This is also fixed on the server side, but there might an old version of the server running that doesn't
329-
// contain this fix. This client-side sorting could be removed in a future version when all servers
330-
// are upgraded.
331-
val chunk = unsortedChunk.sortedBy { it.first }
325+
val maxBodySize = 16 * 1024 * 1024
326+
val chunkContent = StringBuilder(1024 * 1024)
327+
328+
suspend fun sendChunk() {
332329
httpClient.put {
333330
url {
334331
takeFrom(baseUrl)
335332
appendPathSegmentsEncodingSlash("repositories", repository.id, "objects")
336333
}
337334
contentType(ContentType.Text.Plain)
338-
setBody(chunk.flatMap { it.toList() }.joinToString("\n"))
335+
setBody(chunkContent.toString())
336+
}
337+
chunkContent.clear()
338+
}
339+
340+
val itr = objects.iterator()
341+
while (itr.hasNext()) {
342+
val entry = itr.next()
343+
val entrySize = (if (chunkContent.isEmpty()) 0 else 1) + entry.first.length + 1 + entry.second.length
344+
if (chunkContent.length + entrySize > maxBodySize) {
345+
sendChunk()
346+
} else {
347+
if (chunkContent.isNotEmpty()) chunkContent.append('\n')
348+
chunkContent.append(entry.first).append('\n').append(entry.second)
339349
}
340350
}
351+
if (chunkContent.isNotEmpty()) sendChunk()
341352
}
342353

343354
override suspend fun pull(branch: BranchReference, lastKnownVersion: IVersion?): IVersion {

0 commit comments

Comments
 (0)