Skip to content

Commit 41393ae

Browse files
slissonslisson
committed
fix(model-datastructure): StackOverflowError in ITree.addNewChildren with many children
1 parent f47954b commit 41393ae

File tree

6 files changed

+266
-41
lines changed

6 files changed

+266
-41
lines changed

‎model-datastructure/src/commonMain/kotlin/org/modelix/model/async/AsyncTree.kt‎

Lines changed: 44 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package org.modelix.model.async
22

33
import com.badoo.reaktive.completable.andThen
44
import com.badoo.reaktive.maybe.Maybe
5+
import com.badoo.reaktive.maybe.asSingle
56
import com.badoo.reaktive.maybe.asSingleOrError
67
import com.badoo.reaktive.maybe.defaultIfEmpty
78
import com.badoo.reaktive.maybe.map
@@ -21,6 +22,7 @@ import com.badoo.reaktive.single.filter
2122
import com.badoo.reaktive.single.flatMap
2223
import com.badoo.reaktive.single.flatMapIterable
2324
import com.badoo.reaktive.single.flatMapMaybe
25+
import com.badoo.reaktive.single.flatMapObservable
2426
import com.badoo.reaktive.single.flatten
2527
import com.badoo.reaktive.single.map
2628
import com.badoo.reaktive.single.notNull
@@ -57,6 +59,7 @@ import org.modelix.model.lazy.IBulkQuery
5759
import org.modelix.model.lazy.IPrefetchGoal
5860
import org.modelix.model.lazy.KVEntryReference
5961
import org.modelix.model.lazy.NodeNotFoundException
62+
import org.modelix.model.persistent.CPHamtInternal
6063
import org.modelix.model.persistent.CPHamtNode
6164
import org.modelix.model.persistent.CPNode
6265
import org.modelix.model.persistent.CPNodeRef
@@ -84,6 +87,21 @@ open class AsyncTree(val treeData: CPTree, val store: IAsyncObjectStore) : IAsyn
8487
.asSingleOrError { NodeNotFoundException(id) }
8588
.flatMap { it.query() }
8689

90+
fun getNodes(ids: LongArray): Observable<CPNode> {
91+
return nodesMap.query().flatMap {
92+
it.getAll(ids, 0, store).toList().map {
93+
val entries = it.associateBy { it.first }
94+
ids.map { id ->
95+
val value = entries[id]?.second
96+
if (value == null) throw NodeNotFoundException(id)
97+
value
98+
}
99+
}
100+
}.flatMapObservable {
101+
it.asObservable().flatMapSingle { it.query() }
102+
}
103+
}
104+
87105
private fun tryGetNodeRef(id: Long): Maybe<KVEntryReference<CPNode>> = nodesMap.query().flatMapMaybe { it.get(id, store) }
88106

89107
@Deprecated("Prefetching will be replaced by usages of IAsyncNode")
@@ -287,7 +305,11 @@ open class AsyncTree(val treeData: CPTree, val store: IAsyncObjectStore) : IAsyn
287305

288306
override fun getChildren(parentId: Long, role: IChildLinkReference): Observable<Long> {
289307
val roleString = role.key()
290-
return getAllChildren(parentId).flatMapSingle { getNode(it) }.filter { it.roleInParent == roleString }.map { it.id }.loadPrefetch()
308+
return getNode(parentId)
309+
.flatMapObservable { getNodes(it.childrenIdArray) }
310+
.filter { it.roleInParent == roleString }
311+
.map { it.id }
312+
.loadPrefetch()
291313
}
292314

293315
private fun IRoleReference.key() = getRoleKey(this)
@@ -348,29 +370,29 @@ open class AsyncTree(val treeData: CPTree, val store: IAsyncObjectStore) : IAsyn
348370
newIds: LongArray,
349371
concepts: Array<ConceptReference>,
350372
): Single<IAsyncMutableTree> {
351-
val mapIncludingNewNodes = newIds.zip(concepts).asObservable().fold(nodesMap.query()) { nodesMap, (childId, concept) ->
352-
val childData = CPNode.create(
353-
childId,
354-
concept.getUID().takeIf { it != NullConcept.getUID() },
355-
parentId,
356-
getRoleKey(role),
357-
LongArray(0),
358-
arrayOf(),
359-
arrayOf(),
360-
arrayOf(),
361-
arrayOf(),
373+
val newNodes = newIds.zip(concepts).map { (childId, concept) ->
374+
childId to KVEntryReference(
375+
CPNode.create(
376+
childId,
377+
concept.getUID().takeIf { it != NullConcept.getUID() },
378+
parentId,
379+
getRoleKey(role),
380+
LongArray(0),
381+
arrayOf(),
382+
arrayOf(),
383+
arrayOf(),
384+
arrayOf(),
385+
),
362386
)
363-
nodesMap.flatMap {
364-
it.get(childId, store)
365-
.assertEmpty { "Node with ID ${childId.toString(16)} already exists" }
366-
.andThen(it.put(childData, store))
367-
}
368-
}.flatten()
369-
370-
val newParentData = insertChildrenIntoParentData(parentId, index, newIds, role)
387+
}
371388

372-
return mapIncludingNewNodes.zipWith(newParentData) { map, parentData ->
373-
map.put(parentData, store)
389+
val newParentData: Single<CPNode> = insertChildrenIntoParentData(parentId, index, newIds, role)
390+
return nodesMap.query().zipWith(newParentData) { nodesMap, newParentData ->
391+
nodesMap
392+
.getAll(newIds, 0, store)
393+
.assertEmpty { "Node with ID ${it.first.toString(16)} already exists" }
394+
.andThen(nodesMap.putAll(newNodes + (parentId to KVEntryReference(newParentData)), 0, store))
395+
.asSingle { CPHamtInternal.createEmpty() }
374396
}.flatten().newTree()
375397
}
376398

‎model-datastructure/src/commonMain/kotlin/org/modelix/model/persistent/CPHamtInternal.kt‎

Lines changed: 101 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import com.badoo.reaktive.maybe.Maybe
44
import com.badoo.reaktive.maybe.asObservable
55
import com.badoo.reaktive.maybe.filter
66
import com.badoo.reaktive.maybe.flatMap
7+
import com.badoo.reaktive.maybe.flatMapObservable
78
import com.badoo.reaktive.maybe.map
89
import com.badoo.reaktive.maybe.maybeOf
910
import com.badoo.reaktive.maybe.maybeOfEmpty
@@ -12,16 +13,19 @@ import com.badoo.reaktive.observable.Observable
1213
import com.badoo.reaktive.observable.asObservable
1314
import com.badoo.reaktive.observable.filter
1415
import com.badoo.reaktive.observable.flatMap
15-
import com.badoo.reaktive.observable.flatMapMaybe
1616
import com.badoo.reaktive.observable.flatMapSingle
1717
import com.badoo.reaktive.observable.flatten
1818
import com.badoo.reaktive.observable.map
1919
import com.badoo.reaktive.observable.observableOf
2020
import com.badoo.reaktive.observable.observableOfEmpty
21+
import com.badoo.reaktive.observable.toList
2122
import com.badoo.reaktive.single.Single
2223
import com.badoo.reaktive.single.asMaybe
2324
import com.badoo.reaktive.single.flatMapMaybe
25+
import com.badoo.reaktive.single.flatMapObservable
2426
import com.badoo.reaktive.single.flatten
27+
import com.badoo.reaktive.single.singleOf
28+
import com.badoo.reaktive.single.toSingle
2529
import com.badoo.reaktive.single.zipWith
2630
import org.modelix.model.async.IAsyncObjectStore
2731
import org.modelix.model.bitCount
@@ -81,6 +85,39 @@ class CPHamtInternal(
8185
}
8286
}
8387

88+
override fun putAll(entries: List<Pair<Long, KVEntryReference<CPNode>?>>, shift: Int, store: IAsyncObjectStore): Maybe<CPHamtNode> {
89+
val groups = entries.groupBy { indexFromKey(it.first, shift) }
90+
val logicalIndices = groups.keys.toIntArray()
91+
val newChildrenLists = groups.values.toList()
92+
return getChildren(logicalIndices, store).flatMapObservable { children: List<CPHamtNode?> ->
93+
children.withIndex().asObservable().flatMapSingle { (i, oldChild) ->
94+
val newChildren = newChildrenLists[i]
95+
if (oldChild == null) {
96+
val nonNullChildren = newChildren.filter { it.second != null }
97+
when (nonNullChildren.size) {
98+
0 -> null.toSingle()
99+
1 -> {
100+
val singleChild = nonNullChildren.single()
101+
CPHamtLeaf.create(singleChild.first, singleChild.second).toSingle()
102+
}
103+
else -> {
104+
createEmpty().putAll(nonNullChildren, shift + BITS_PER_LEVEL, store).orNull()
105+
}
106+
}
107+
} else {
108+
oldChild.putAll(newChildren, shift + BITS_PER_LEVEL, store).orNull()
109+
}
110+
}
111+
}.toList().flatMapMaybe { updatedChildren ->
112+
setChildren(
113+
logicalIndices,
114+
updatedChildren.map { it?.let { KVEntryReference(it) } },
115+
shift,
116+
store,
117+
)
118+
}
119+
}
120+
84121
override fun remove(key: Long, shift: Int, store: IAsyncObjectStore): Maybe<CPHamtNode> {
85122
require(shift <= CPHamtNode.MAX_SHIFT) { "$shift > ${CPHamtNode.MAX_SHIFT}" }
86123
val childIndex = CPHamtNode.indexFromKey(key, shift)
@@ -103,6 +140,19 @@ class CPHamtInternal(
103140
}
104141
}
105142

143+
override fun getAll(
144+
keys: LongArray,
145+
shift: Int,
146+
store: IAsyncObjectStore,
147+
): Observable<Pair<Long, KVEntryReference<CPNode>?>> {
148+
val groups = keys.groupBy { indexFromKey(it, shift) }
149+
return groups.entries.asObservable().flatMap { group ->
150+
getChild(group.key, store).flatMapObservable { child ->
151+
child.getAll(group.value.toLongArray(), shift + BITS_PER_LEVEL, store)
152+
}
153+
}
154+
}
155+
106156
protected fun getChild(logicalIndex: Int, store: IAsyncObjectStore): Maybe<CPHamtNode> {
107157
if (isBitNotSet(data.bitmap, logicalIndex)) {
108158
return maybeOfEmpty()
@@ -113,10 +163,60 @@ class CPHamtInternal(
113163
return getChild(childHash, store).asMaybe()
114164
}
115165

166+
private fun getChildren(logicalIndices: IntArray, store: IAsyncObjectStore): Single<List<CPHamtNode?>> {
167+
val childHashes = logicalIndices.map { logicalIndex ->
168+
if (isBitNotSet(data.bitmap, logicalIndex)) {
169+
null
170+
} else {
171+
val physicalIndex = logicalToPhysicalIndex(data.bitmap, logicalIndex)
172+
data.children[physicalIndex]
173+
}
174+
}
175+
return childHashes.asObservable().flatMapSingle { it?.getValue(store) ?: singleOf(null) }.toList()
176+
}
177+
116178
protected fun getChild(childHash: KVEntryReference<CPHamtNode>, store: IAsyncObjectStore): Single<CPHamtNode> {
117179
return childHash.getValue(store)
118180
}
119181

182+
fun setChildren(logicalIndices: IntArray, children: List<KVEntryReference<CPHamtNode>?>, shift: Int, store: IAsyncObjectStore): Maybe<CPHamtNode> {
183+
var oldBitmap = data.bitmap
184+
var newBitmap = data.bitmap
185+
val oldChildren = data.children
186+
var newChildren = data.children
187+
for (i in logicalIndices.indices) {
188+
val logicalIndex = logicalIndices[i]
189+
val newChild = children[i]
190+
val oldChild = if (isBitNotSet(oldBitmap, logicalIndex)) {
191+
null
192+
} else {
193+
oldChildren[logicalToPhysicalIndex(oldBitmap, logicalIndex)]
194+
}
195+
if (newChild == null) {
196+
if (oldChild == null) {
197+
// nothing changed
198+
} else {
199+
newChildren = COWArrays.removeAt(newChildren, logicalToPhysicalIndex(newBitmap, logicalIndex))
200+
newBitmap = newBitmap and (1 shl logicalIndex).inv() // clear bit
201+
}
202+
} else {
203+
if (oldChild == null) {
204+
newChildren = COWArrays.insert(newChildren, logicalToPhysicalIndex(newBitmap, logicalIndex), newChild)
205+
newBitmap = newBitmap or (1 shl logicalIndex) // set bit
206+
} else {
207+
newChildren = COWArrays.set(newChildren, logicalToPhysicalIndex(newBitmap, logicalIndex), newChild)
208+
}
209+
}
210+
}
211+
212+
val newNode = create(newBitmap, newChildren)
213+
return if (shift < MAX_BITS - BITS_PER_LEVEL) {
214+
CPHamtSingle.replaceIfSingleChild(newNode, store).asMaybe()
215+
} else {
216+
newNode.toMaybe()
217+
}
218+
}
219+
120220
fun setChild(logicalIndex: Int, child: CPHamtNode?, shift: Int, store: IAsyncObjectStore): Maybe<CPHamtNode> {
121221
if (child == null) {
122222
return deleteChild(logicalIndex, store)

‎model-datastructure/src/commonMain/kotlin/org/modelix/model/persistent/CPHamtLeaf.kt‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ class CPHamtLeaf(
3232
override fun put(key: Long, value: KVEntryReference<CPNode>?, shift: Int, store: IAsyncObjectStore): Maybe<CPHamtNode> {
3333
require(shift <= CPHamtNode.MAX_SHIFT + CPHamtNode.BITS_PER_LEVEL) { "$shift > ${CPHamtNode.MAX_SHIFT + CPHamtNode.BITS_PER_LEVEL}" }
3434
return if (key == this.key) {
35-
if (value?.getHash() == this.value?.getHash()) {
35+
if (value?.getHash() == this.value.getHash()) {
3636
this.toMaybe()
3737
} else {
3838
create(key, value).toMaybeNotNull()
@@ -45,6 +45,16 @@ class CPHamtLeaf(
4545
}
4646
}
4747

48+
override fun putAll(entries: List<Pair<Long, KVEntryReference<CPNode>?>>, shift: Int, store: IAsyncObjectStore): Maybe<CPHamtNode> {
49+
return if (entries.size == 1) {
50+
val entry = entries.single()
51+
put(entry.first, entry.second, shift, store)
52+
} else {
53+
val newEntries = if (entries.any { it.first == this.key }) entries else entries + (this.key to this.value)
54+
createEmptyNode().putAll(newEntries, shift, store)
55+
}
56+
}
57+
4858
override fun remove(key: Long, shift: Int, store: IAsyncObjectStore): Maybe<CPHamtNode> {
4959
require(shift <= CPHamtNode.MAX_SHIFT + CPHamtNode.BITS_PER_LEVEL) { "$shift > ${CPHamtNode.MAX_SHIFT + CPHamtNode.BITS_PER_LEVEL}" }
5060
return if (key == this.key) {
@@ -59,6 +69,14 @@ class CPHamtLeaf(
5969
return if (key == this.key) maybeOf(value) else maybeOfEmpty()
6070
}
6171

72+
override fun getAll(
73+
keys: LongArray,
74+
shift: Int,
75+
store: IAsyncObjectStore,
76+
): Observable<Pair<Long, KVEntryReference<CPNode>?>> {
77+
return if (keys.contains(this.key)) observableOf(key to value) else observableOfEmpty()
78+
}
79+
6280
override fun getEntries(store: IAsyncObjectStore): Observable<Pair<Long, KVEntryReference<CPNode>>> {
6381
return observableOf(key to value)
6482
}

‎model-datastructure/src/commonMain/kotlin/org/modelix/model/persistent/CPHamtNode.kt‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,10 @@ package org.modelix.model.persistent
22

33
import com.badoo.reaktive.maybe.Maybe
44
import com.badoo.reaktive.maybe.asSingleOrError
5-
import com.badoo.reaktive.maybe.defaultIfEmpty
65
import com.badoo.reaktive.observable.Observable
7-
import com.badoo.reaktive.observable.asObservable
8-
import com.badoo.reaktive.observable.flatMapSingle
96
import com.badoo.reaktive.observable.toList
107
import com.badoo.reaktive.single.Single
8+
import com.badoo.reaktive.single.map
119
import org.modelix.model.async.IAsyncObjectStore
1210
import org.modelix.model.lazy.KVEntryReference
1311
import org.modelix.model.persistent.SerializationUtil.intFromHex
@@ -28,8 +26,11 @@ abstract class CPHamtNode : IKVValue {
2826
return CPHamtInternal(0, arrayOf())
2927
}
3028

31-
fun getAll(keys: Iterable<Long>, store: IAsyncObjectStore): Single<List<KVEntryReference<CPNode>?>> {
32-
return keys.asObservable().flatMapSingle { get(it, 0, store).defaultIfEmpty(null) }.toList()
29+
fun getAll(keys: LongArray, store: IAsyncObjectStore): Single<List<KVEntryReference<CPNode>?>> {
30+
return getAll(keys, 0, store).toList().map {
31+
val entries = it.associateBy { it.first }
32+
keys.map { entries[it]?.second }
33+
}
3334
}
3435

3536
fun put(key: Long, value: KVEntryReference<CPNode>?, store: IAsyncObjectStore): Maybe<CPHamtNode> {
@@ -53,6 +54,8 @@ abstract class CPHamtNode : IKVValue {
5354

5455
abstract fun get(key: Long, shift: Int, store: IAsyncObjectStore): Maybe<KVEntryReference<CPNode>>
5556
abstract fun put(key: Long, value: KVEntryReference<CPNode>?, shift: Int, store: IAsyncObjectStore): Maybe<CPHamtNode>
57+
abstract fun putAll(entries: List<Pair<Long, KVEntryReference<CPNode>?>>, shift: Int, store: IAsyncObjectStore): Maybe<CPHamtNode>
58+
abstract fun getAll(keys: LongArray, shift: Int, store: IAsyncObjectStore): Observable<Pair<Long, KVEntryReference<CPNode>?>>
5659
abstract fun remove(key: Long, shift: Int, store: IAsyncObjectStore): Maybe<CPHamtNode>
5760
abstract fun getEntries(store: IAsyncObjectStore): Observable<Pair<Long, KVEntryReference<CPNode>>>
5861
abstract fun getChanges(oldNode: CPHamtNode?, shift: Int, store: IAsyncObjectStore, changesOnly: Boolean): Observable<MapChangeEvent>

‎model-datastructure/src/commonMain/kotlin/org/modelix/model/persistent/CPHamtSingle.kt‎

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -52,22 +52,38 @@ class CPHamtSingle(
5252
}
5353
}
5454

55+
override fun getAll(
56+
keys: LongArray,
57+
shift: Int,
58+
store: IAsyncObjectStore,
59+
): Observable<Pair<Long, KVEntryReference<CPNode>?>> {
60+
if (keys.any { maskBits(it, shift) == bits }) {
61+
return child.getValue(store).flatMapObservable {
62+
it.getAll(keys, shift + numLevels * BITS_PER_LEVEL, store)
63+
}
64+
} else {
65+
return observableOfEmpty()
66+
}
67+
}
68+
5569
override fun put(key: Long, value: KVEntryReference<CPNode>?, shift: Int, store: IAsyncObjectStore): Maybe<CPHamtNode> {
56-
require(shift <= CPHamtNode.MAX_SHIFT) { "$shift > ${CPHamtNode.MAX_SHIFT}" }
57-
if (maskBits(key, shift) == bits) {
58-
return getChild(store).flatMapMaybe { it.put(key, value, shift + CPHamtNode.BITS_PER_LEVEL * numLevels, store) }.map { withNewChild(it) }
70+
return putAll(listOf(key to value), shift, store)
71+
}
72+
73+
override fun putAll(
74+
entries: List<Pair<Long, KVEntryReference<CPNode>?>>,
75+
shift: Int,
76+
store: IAsyncObjectStore,
77+
): Maybe<CPHamtNode> {
78+
if (entries.all { maskBits(it.first, shift) == bits }) {
79+
return getChild(store)
80+
.flatMapMaybe { it.putAll(entries, shift + BITS_PER_LEVEL * numLevels, store) }
81+
.map { withNewChild(it) }
5982
} else {
6083
if (numLevels > 1) {
61-
return splitOneLevel().put(key, value, shift, store)
62-
// val nextLevel = CPHamtSingle(CPHamtSingle(numLevels - 1, bits and maskForLevels(numLevels - 1), child), store)
63-
// if (nextLevel.maskBits(key, shift + BITS_PER_LEVEL) == nextLevel.bits) {
64-
// val newNextLevel = nextLevel.put(key, value, shift + BITS_PER_LEVEL)
65-
// if (newNextLevel == null) return null
66-
// return CPHamtSingle(CPHamtSingle(1, bits ushr (BITS_PER_LEVEL * (numLevels - 1)), KVEntryReference(newNextLevel.getData())), store)
67-
// } else {
68-
// }
84+
return splitOneLevel().putAll(entries, shift, store)
6985
} else {
70-
return CPHamtInternal.replace(this).put(key, value, shift, store)
86+
return CPHamtInternal.replace(this).putAll(entries, shift, store)
7187
}
7288
}
7389
}

0 commit comments

Comments
 (0)