feat(model-client): method for efficiently querying coarse grained history intervals

slisson · slisson · commit 49de28f6d266 · 2025-10-02T14:06:31.000+02:00
diff --git a/model-client/src/commonMain/kotlin/org/modelix/model/client2/IModelClientV2.kt b/model-client/src/commonMain/kotlin/org/modelix/model/client2/IModelClientV2.kt
@@ -1,6 +1,7 @@
 package org.modelix.model.client2
 
 import io.ktor.http.Url
+import kotlinx.datetime.Instant
 import org.modelix.datastructures.objects.ObjectHash
 import org.modelix.kotlin.utils.DeprecationInfo
 import org.modelix.model.IVersion
@@ -109,20 +110,18 @@ interface IModelClientV2 {
     fun getFrontendUrl(branch: BranchReference): Url
 
     /**
-     * @param headVersion starting point for history computations. For a paginated view this value should be the same and
-     *        the value for [skip] should be incremented instead. Only then its guaranteed that the returned list is
-     *        complete.
-     * @param skip for a paginated view of the history
-     * @param limit maximum size of the returned list
-     * @param interval splits the timeline into equally sized intervals and returns only the last version of each interval
+     * @param headVersion starting point for history computations. For a paginated view this value should be the same
+     *        and the value for [skip] should be incremented instead. Only then it's guaranteed that the returned list
+     *        is complete.
+     * @param timeRange return only intervals in this time range
+     * @param interval splits the timeline into equally sized intervals and returns a summary of the contained versions
      */
-    suspend fun getHistory(
+    suspend fun getHistoryIntervals(
         repositoryId: RepositoryId,
         headVersion: ObjectHash,
-        skip: Int = 0,
-        limit: Int = 1000,
-        interval: Duration?,
-    ): HistoryResponse
+        timeRange: ClosedRange<Instant>?,
+        interval: Duration,
+    ): List<HistoryInterval>
 
     suspend fun getHistoryRange(
         repositoryId: RepositoryId,
@@ -133,15 +132,16 @@ interface IModelClientV2 {
 }
 
 data class HistoryResponse(
-    val entries: List<HistoryEntry>,
+    val entries: List<HistoryInterval>,
     val nextVersions: List<ObjectHash>,
 )
 
-data class HistoryEntry(
+data class HistoryInterval(
     val firstVersionHash: ObjectHash,
     val lastVersionHash: ObjectHash,
-    val minTime: Long?,
-    val maxTime: Long?,
+    val size: Long,
+    val minTime: Instant,
+    val maxTime: Instant,
     val authors: Set<String>,
     // val headOfBranch: Set<BranchReference>,
 )
diff --git a/model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt b/model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt
@@ -36,6 +36,7 @@ import kotlinx.coroutines.flow.emptyFlow
 import kotlinx.coroutines.flow.flow
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.launch
+import kotlinx.datetime.Instant
 import mu.KotlinLogging
 import org.modelix.datastructures.model.HistoryIndexNode
 import org.modelix.datastructures.objects.IObjectGraph
@@ -332,64 +333,43 @@ class ModelClientV2(
         }
     }
 
-    override suspend fun getHistory(
+    override suspend fun getHistoryIntervals(
         repositoryId: RepositoryId,
         headVersion: ObjectHash,
-        skip: Int,
-        limit: Int,
-        interval: Duration?,
-    ): HistoryResponse {
+        timeRange: ClosedRange<Instant>?,
+        interval: Duration,
+    ): List<HistoryInterval> {
         val index: Object<HistoryIndexNode> = getHistoryIndex(repositoryId, headVersion)
-        val entries = if (interval != null) {
-            val mergedEntries = ArrayList<HistoryEntry>()
-            var previousIntervalId: Long = Long.MAX_VALUE
-            try {
-                index.data.splitAtInterval(interval).iterateSuspending(index.graph) {
-                    val intervalId = it.maxTime.toEpochMilliseconds() / interval.inWholeMilliseconds
-                    require(intervalId <= previousIntervalId)
-                    if (intervalId == previousIntervalId) {
-                        val entry = mergedEntries[mergedEntries.lastIndex]
-                        mergedEntries[mergedEntries.lastIndex] = HistoryEntry(
-                            firstVersionHash = it.firstVersion.getHash(),
-                            lastVersionHash = entry.lastVersionHash,
-                            minTime = it.minTime.epochSeconds,
-                            maxTime = entry.maxTime,
-                            authors = entry.authors + it.authors,
-                        )
-                    } else {
-                        if (mergedEntries.size >= limit) throw LimitedReached()
-                        previousIntervalId = intervalId
-                        mergedEntries += HistoryEntry(
-                            firstVersionHash = it.firstVersion.getHash(),
-                            lastVersionHash = it.lastVersion.getHash(),
-                            minTime = it.minTime.epochSeconds,
-                            maxTime = it.maxTime.epochSeconds,
-                            authors = it.authors,
-                        )
-                    }
-                }
-            } catch (ex: LimitedReached) {
-                // Expected exception used for exiting the iterateSuspending call
+        val mergedEntries = ArrayList<HistoryInterval>()
+        var previousIntervalId: Long = Long.MAX_VALUE
+
+        index.data.splitAtInterval(interval, timeRange).iterateSuspending(index.graph) {
+            val intervalId = it.maxTime.epochSeconds / interval.inWholeSeconds
+            check(intervalId <= previousIntervalId)
+            if (intervalId == previousIntervalId) {
+                val entry = mergedEntries[mergedEntries.lastIndex]
+                mergedEntries[mergedEntries.lastIndex] = HistoryInterval(
+                    firstVersionHash = it.firstVersion.getHash(),
+                    lastVersionHash = entry.lastVersionHash,
+                    size = entry.size + it.size,
+                    minTime = minOf(entry.minTime, it.minTime),
+                    maxTime = maxOf(entry.maxTime, it.maxTime),
+                    authors = entry.authors + it.authors,
+                )
+            } else {
+                previousIntervalId = intervalId
+                mergedEntries += HistoryInterval(
+                    firstVersionHash = it.firstVersion.getHash(),
+                    lastVersionHash = it.lastVersion.getHash(),
+                    size = it.size,
+                    minTime = it.minTime,
+                    maxTime = it.maxTime,
+                    authors = it.authors,
+                )
             }
-            mergedEntries
-        } else {
-            index.data.getAllVersionsReversed().flatMapOrdered { it.resolve() }.map { CLVersion(it) }
-                .map {
-                    val hash = it.getObjectHash()
-                    val time = it.getTimestamp()?.epochSeconds
-                    HistoryEntry(
-                        firstVersionHash = hash,
-                        lastVersionHash = hash,
-                        minTime = time,
-                        maxTime = time,
-                        authors = setOfNotNull(it.author),
-                    )
-                }
-                .take(limit)
-                .toList()
-                .getSuspending(index.graph)
         }
-        return HistoryResponse(entries = entries, nextVersions = emptyList())
+
+        return mergedEntries
     }
 
     override suspend fun getHistoryRange(
@@ -1073,4 +1053,4 @@ fun IVersion.runWrite(idGenerator: IIdGenerator, author: String?, body: (IBranch
 
 private fun String.ensureSuffix(suffix: String) = if (endsWith(suffix)) this else this + suffix
 
-private class LimitedReached : RuntimeException("limited reached")
+private class LimitReached : RuntimeException("limit reached")
diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/model/HistoryIndexNode.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/model/HistoryIndexNode.kt
@@ -46,7 +46,7 @@ sealed class HistoryIndexNode : IObjectData {
      * For the same interval multiple nodes may be returned.
      * Latest entry is returned first.
      */
-    abstract fun splitAtInterval(interval: Duration): IStream.Many<HistoryIndexNode>
+    abstract fun splitAtInterval(interval: Duration, timeRangeFilter: ClosedRange<Instant>?): IStream.Many<HistoryIndexNode>
 
     fun concat(
         self: Object<HistoryIndexNode>,
@@ -213,8 +213,8 @@ data class HistoryIndexLeafNode(
         }
     }
 
-    override fun splitAtInterval(interval: Duration): IStream.Many<HistoryIndexNode> {
-        TODO("Not yet implemented")
+    override fun splitAtInterval(interval: Duration, timeRangeFilter: ClosedRange<Instant>?): IStream.Many<HistoryIndexNode> {
+        return if (timeRangeFilter == null || timeRangeFilter.contains(time)) IStream.of(this) else IStream.empty()
     }
 }
 
@@ -335,8 +335,18 @@ data class HistoryIndexRangeNode(
         }.flatten()
     }
 
-    override fun splitAtInterval(interval: Duration): IStream.Many<HistoryIndexNode> {
-        TODO("Not yet implemented")
+    override fun splitAtInterval(interval: Duration, timeRangeFilter: ClosedRange<Instant>?): IStream.Many<HistoryIndexNode> {
+        if (timeRangeFilter != null && !timeRangeFilter.intersects(timeRange)) return IStream.empty()
+
+        val intervalIndex1 = timeRange.start.epochSeconds / interval.inWholeSeconds
+        val intervalIndex2 = timeRange.endInclusive.epochSeconds / interval.inWholeSeconds
+        if (intervalIndex1 == intervalIndex2) {
+            return IStream.of(this)
+        } else {
+            return IStream.of(child2, child1)
+                .flatMapOrdered { it.resolve() }
+                .flatMapOrdered { it.data.splitAtInterval(interval, timeRangeFilter) }
+        }
     }
 
     /**
diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/IVersion.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/IVersion.kt
@@ -31,6 +31,7 @@ interface IVersion {
     fun tryGetParentVersions(): List<VersionAndHash>
 
     fun getTimestamp(): Instant?
+    fun getAuthor(): String?
 
     companion object {
         fun builder(): VersionBuilder = VersionBuilder()
diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/CLVersion.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/CLVersion.kt
@@ -61,6 +61,8 @@ class CLVersion(val obj: Object<CPVersion>) : IVersion {
 
     val data: CPVersion get() = resolvedData.data
 
+    @get:JvmName("get_author")
+    @Deprecated("Use getAuthor()", ReplaceWith("getAuthor()"))
     val author: String?
         get() = data.author
 
@@ -83,6 +85,8 @@ class CLVersion(val obj: Object<CPVersion>) : IVersion {
         return null
     }
 
+    override fun getAuthor(): String? = data.author
+
     @Deprecated("Use getObjectHash()", ReplaceWith("getObjectHash()"))
     val hash: String
         get() = getContentHash()
diff --git a/model-server/src/test/kotlin/org/modelix/model/server/HistoryIndexIntervalTest.kt b/model-server/src/test/kotlin/org/modelix/model/server/HistoryIndexIntervalTest.kt
@@ -0,0 +1,132 @@
+package org.modelix.model.server
+
+import io.ktor.server.testing.ApplicationTestBuilder
+import io.ktor.server.testing.testApplication
+import mu.KotlinLogging
+import org.modelix.model.IVersion
+import org.modelix.model.client2.HistoryInterval
+import org.modelix.model.client2.IModelClientV2
+import org.modelix.model.historyAsSequence
+import org.modelix.model.lazy.RepositoryId
+import org.modelix.model.server.api.RepositoryConfig
+import org.modelix.model.server.handlers.IdsApiImpl
+import org.modelix.model.server.handlers.ModelReplicationServer
+import org.modelix.model.server.handlers.RepositoriesManager
+import org.modelix.model.server.store.InMemoryStoreClient
+import kotlin.random.Random
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.time.Duration.Companion.seconds
+
+private val LOG = KotlinLogging.logger { }
+
+class HistoryIndexIntervalTest {
+
+    private lateinit var statistics: StoreClientWithStatistics
+    private fun runTest(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication {
+        application {
+            try {
+                installDefaultServerPlugins()
+                statistics = StoreClientWithStatistics(InMemoryStoreClient())
+                val repoManager = RepositoriesManager(statistics)
+                ModelReplicationServer(repoManager).init(this)
+                IdsApiImpl(repoManager).init(this)
+            } catch (ex: Throwable) {
+                LOG.error("", ex)
+            }
+        }
+        block()
+    }
+
+    @Test fun interval_0_201_1() = runIntervalTest(0, 201, 1)
+
+    @Test fun interval_0_201_10() = runIntervalTest(0, 201, 10)
+
+    @Test fun interval_0_201_20() = runIntervalTest(0, 201, 20)
+
+    @Test fun interval_0_201_50() = runIntervalTest(0, 201, 50)
+
+    @Test fun interval_0_201_100() = runIntervalTest(0, 201, 100)
+
+    @Test fun interval_0_201_150() = runIntervalTest(0, 201, 150)
+
+    @Test fun interval_0_201_200() = runIntervalTest(0, 201, 200)
+
+    @Test fun interval_0_201_500() = runIntervalTest(0, 201, 500)
+
+    @Test fun interval_0_1_20() = runIntervalTest(0, 1, 20)
+
+    @Test fun interval_0_2_20() = runIntervalTest(0, 2, 20)
+
+    @Test fun interval_0_3_20() = runIntervalTest(0, 3, 20)
+
+    @Test fun interval_0_4_20() = runIntervalTest(0, 4, 20)
+
+    @Test fun interval_1_4_20() = runIntervalTest(1, 4, 20)
+
+    @Test fun interval_2_4_20() = runIntervalTest(2, 4, 20)
+
+    @Test fun interval_3_4_20() = runIntervalTest(3, 4, 20)
+
+    @Test fun interval_4_4_20() = runIntervalTest(4, 4, 20)
+
+    @Test fun interval_5_4_20() = runIntervalTest(5, 4, 20)
+
+    private fun runIntervalTest(skip: Int, limit: Int, intervalSeconds: Int) = runTest {
+        val rand = Random(8923345)
+        val modelClient: IModelClientV2 = createModelClient()
+        val repositoryId = RepositoryId("repo1")
+        val branchRef = repositoryId.getBranchReference()
+        val initialVersion = modelClient.initRepository(RepositoryConfig(repositoryId = repositoryId.id, repositoryName = repositoryId.id, modelId = "61bd6cb0-33ff-45d8-9d1b-2149fdb01d16"))
+        var currentVersion = initialVersion
+
+        repeat(100) {
+            run {
+                val newVersion = IVersion.builder()
+                    .baseVersion(currentVersion)
+                    .tree(currentVersion.getModelTree())
+                    .author("user1")
+                    .time(currentVersion.getTimestamp()!! + rand.nextInt(0, 3).seconds)
+                    .build()
+                currentVersion = modelClient.push(branchRef, newVersion, currentVersion)
+            }
+            run {
+                val newVersion = IVersion.builder()
+                    .baseVersion(currentVersion)
+                    .tree(currentVersion.getModelTree())
+                    .author("user2")
+                    .time(currentVersion.getTimestamp()!! + rand.nextInt(0, 3).seconds)
+                    .build()
+                currentVersion = modelClient.push(branchRef, newVersion, currentVersion)
+            }
+        }
+
+        val expectedHistory = currentVersion.historyAsSequence().toList().reversed()
+
+        val expectedIntervals = expectedHistory
+            .groupBy { it.getTimestamp()!!.epochSeconds / intervalSeconds }
+            .map { entry ->
+                val versions = entry.value
+                HistoryInterval(
+                    firstVersionHash = versions.first().getObjectHash(),
+                    lastVersionHash = versions.last().getObjectHash(),
+                    size = versions.size.toLong(),
+                    minTime = versions.minOf { it.getTimestamp()!! },
+                    maxTime = versions.maxOf { it.getTimestamp()!! },
+                    authors = versions.mapNotNull { it.getAuthor() }.toSet(),
+                )
+            }
+            .reversed()
+            .drop(skip)
+            .take(limit)
+
+        val timeRange = (expectedIntervals.minOf { it.minTime })..(expectedIntervals.maxOf { it.maxTime })
+        val history = modelClient.getHistoryIntervals(
+            repositoryId = repositoryId,
+            headVersion = currentVersion.getObjectHash(),
+            timeRange = timeRange,
+            interval = intervalSeconds.seconds,
+        )
+        assertEquals(expectedIntervals, history)
+    }
+}
diff --git a/model-server/src/test/kotlin/org/modelix/model/server/HistoryIndexPaginationTest.kt b/model-server/src/test/kotlin/org/modelix/model/server/HistoryIndexPaginationTest.kt
@@ -19,7 +19,7 @@ import kotlin.time.Duration.Companion.seconds
 
 private val LOG = KotlinLogging.logger { }
 
-class HistoryIndexTest {
+class HistoryIndexPaginationTest {
 
     private lateinit var statistics: StoreClientWithStatistics
     private fun runTest(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication {

Original file line number	Diff line number	Diff line change
`@@ -46,7 +46,7 @@ sealed class HistoryIndexNode : IObjectData {`
`46`	`46`	`* For the same interval multiple nodes may be returned.`
`47`	`47`	`* Latest entry is returned first.`
`48`	`48`	`*/`
`49`		`- abstract fun splitAtInterval(interval: Duration): IStream.Many<HistoryIndexNode>`
	`49`	`+ abstract fun splitAtInterval(interval: Duration, timeRangeFilter: ClosedRange<Instant>?): IStream.Many<HistoryIndexNode>`
`50`	`50`
`51`	`51`	`fun concat(`
`52`	`52`	`self: Object<HistoryIndexNode>,`
`@@ -213,8 +213,8 @@ data class HistoryIndexLeafNode(`
`213`	`213`	`}`
`214`	`214`	`}`
`215`	`215`
`216`		`- override fun splitAtInterval(interval: Duration): IStream.Many<HistoryIndexNode> {`
`217`		`- TODO("Not yet implemented")`
	`216`	`+ override fun splitAtInterval(interval: Duration, timeRangeFilter: ClosedRange<Instant>?): IStream.Many<HistoryIndexNode> {`
	`217`	`+ return if (timeRangeFilter == null \|\| timeRangeFilter.contains(time)) IStream.of(this) else IStream.empty()`
`218`	`218`	`}`
`219`	`219`	`}`
`220`	`220`
`@@ -335,8 +335,18 @@ data class HistoryIndexRangeNode(`
`335`	`335`	`}.flatten()`
`336`	`336`	`}`
`337`	`337`
`338`		`- override fun splitAtInterval(interval: Duration): IStream.Many<HistoryIndexNode> {`
`339`		`- TODO("Not yet implemented")`
	`338`	`+ override fun splitAtInterval(interval: Duration, timeRangeFilter: ClosedRange<Instant>?): IStream.Many<HistoryIndexNode> {`
	`339`	`+ if (timeRangeFilter != null && !timeRangeFilter.intersects(timeRange)) return IStream.empty()`
	`340`	`+`
	`341`	`+ val intervalIndex1 = timeRange.start.epochSeconds / interval.inWholeSeconds`
	`342`	`+ val intervalIndex2 = timeRange.endInclusive.epochSeconds / interval.inWholeSeconds`
	`343`	`+ if (intervalIndex1 == intervalIndex2) {`
	`344`	`+ return IStream.of(this)`
	`345`	`+ } else {`
	`346`	`+ return IStream.of(child2, child1)`
	`347`	`+ .flatMapOrdered { it.resolve() }`
	`348`	`+ .flatMapOrdered { it.data.splitAtInterval(interval, timeRangeFilter) }`
	`349`	`+ }`
`340`	`350`	`}`
`341`	`351`
`342`	`352`	`/**`