Skip to content

Commit 0a27082

Browse files
nikolay-egorovnikolay-egorov
committed
Store all meta-data related to a top-level variable name
1 parent 2b19e48 commit 0a27082

File tree

4 files changed

+64
-53
lines changed

4 files changed

+64
-53
lines changed

‎src/main/kotlin/org/jetbrains/kotlinx/jupyter/protocol.kt

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -347,9 +347,7 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup
347347
sendWrapped(msg, makeReplyMessage(msg, MessageType.SERIALIZATION_REPLY, content = result))
348348
}
349349
} else {
350-
repl.serializeVariables(content.cellId, content.descriptorsState) { result ->
351-
sendWrapped(msg, makeReplyMessage(msg, MessageType.SERIALIZATION_REPLY, content = result))
352-
}
350+
sendWrapped(msg, makeReplyMessage(msg, MessageType.SERIALIZATION_REPLY, content = null))
353351
}
354352
}
355353
}

‎src/main/kotlin/org/jetbrains/kotlinx/jupyter/repl.kt

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ interface ReplForJupyter {
137137

138138
suspend fun listErrors(code: Code, callback: (ListErrorsResult) -> Unit)
139139

140-
suspend fun serializeVariables(cellId: Int, descriptorsState: Map<String, SerializedVariablesState>, callback: (SerializationReply) -> Unit)
140+
suspend fun serializeVariables(cellId: Int, topLevelVarName: String, descriptorsState: Map<String, SerializedVariablesState>, callback: (SerializationReply) -> Unit)
141141

142142
suspend fun serializeVariables(topLevelVarName: String, descriptorsState: Map<String, SerializedVariablesState>, pathToDescriptor: List<String> = emptyList(),
143143
callback: (SerializationReply) -> Unit)
@@ -562,8 +562,8 @@ class ReplForJupyterImpl(
562562
}
563563

564564
private val serializationQueue = LockQueue<SerializationReply, SerializationArgs>()
565-
override suspend fun serializeVariables(cellId: Int, descriptorsState: Map<String, SerializedVariablesState>, callback: (SerializationReply) -> Unit) {
566-
doWithLock(SerializationArgs(descriptorsState, cellId = cellId, callback = callback), serializationQueue, SerializationReply(cellId, descriptorsState), ::doSerializeVariables)
565+
override suspend fun serializeVariables(cellId: Int, topLevelVarName: String, descriptorsState: Map<String, SerializedVariablesState>, callback: (SerializationReply) -> Unit) {
566+
doWithLock(SerializationArgs(descriptorsState, cellId = cellId, topLevelVarName = topLevelVarName, callback = callback), serializationQueue, SerializationReply(cellId, descriptorsState), ::doSerializeVariables)
567567
}
568568

569569
override suspend fun serializeVariables(topLevelVarName: String, descriptorsState: Map<String, SerializedVariablesState>, pathToDescriptor: List<String>,
@@ -579,7 +579,7 @@ class ReplForJupyterImpl(
579579
finalAns
580580
}
581581
args.descriptorsState.forEach { (name, state) ->
582-
resultMap[name] = variablesSerializer.doIncrementalSerialization(cellId - 1, name, state, args.pathToDescriptor)
582+
resultMap[name] = variablesSerializer.doIncrementalSerialization(cellId - 1, args.topLevelVarName ,name, state, args.pathToDescriptor)
583583
}
584584
log.debug("Serialization cellID: $cellId")
585585
log.debug("Serialization answer: ${resultMap.entries.first().value.fieldDescriptor}")

‎src/main/kotlin/org/jetbrains/kotlinx/jupyter/serializationUtils.kt

Lines changed: 29 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,8 @@ class ProcessedSerializedVarsState(
5656
data class ProcessedDescriptorsState(
5757
val processedSerializedVarsToJavaProperties: MutableMap<SerializedVariablesState, PropertiesData?> = mutableMapOf(),
5858
val processedSerializedVarsToKTProperties: MutableMap<SerializedVariablesState, KPropertiesData?> = mutableMapOf(),
59-
val instancesPerState: MutableMap<SerializedVariablesState, Any?> = mutableMapOf()
59+
val instancesPerState: MutableMap<SerializedVariablesState, Any?> = mutableMapOf(),
60+
val parent: ProcessedDescriptorsState? = null
6061
)
6162

6263
data class RuntimeObjectWrapper(
@@ -276,16 +277,6 @@ class VariablesSerializer(
276277
*/
277278
if (descriptors.size == 1 && descriptors.entries.first().key == "size") {
278279
descriptors.addDescriptor(value, "data")
279-
/*
280-
if (value is Collection<*>) {
281-
value.forEach {
282-
iterateThrough(descriptors, it)
283-
}
284-
} else if (value is Array<*>) {
285-
value.forEach {
286-
iterateThrough(descriptors, it)
287-
}
288-
}*/
289280
}
290281
}
291282

@@ -319,9 +310,9 @@ class VariablesSerializer(
319310
)
320311

321312
/**
322-
* Stores info computed descriptors in a cell
313+
* Stores info computed descriptors in a cell starting from the very variable as a root
323314
*/
324-
private val computedDescriptorsPerCell: MutableMap<Int, ProcessedDescriptorsState> = mutableMapOf()
315+
private val computedDescriptorsPerCell: MutableMap<Int, MutableMap<String, ProcessedDescriptorsState>> = mutableMapOf()
325316

326317
private val isSerializationActive: Boolean = System.getProperty(serializationSystemProperty)?.toBooleanStrictOrNull() ?: true
327318

@@ -409,7 +400,7 @@ class VariablesSerializer(
409400
log.debug("Unchanged variables: ${unchangedVariables - neededEntries.keys}")
410401

411402
// remove previous data
412-
computedDescriptorsPerCell[cellId]?.instancesPerState?.clear()
403+
// computedDescriptorsPerCell[cellId]?.instancesPerState?.clear()
413404
val serializedData = neededEntries.mapValues {
414405
val actualCell = variablesCells[it.key] ?: cellId
415406
serializeVariableState(actualCell, it.key, it.value)
@@ -424,14 +415,15 @@ class VariablesSerializer(
424415

425416
fun doIncrementalSerialization(
426417
cellId: Int,
418+
topLevelName: String,
427419
propertyName: String,
428420
serializedVariablesState: SerializedVariablesState,
429421
pathToDescriptor: List<String> = emptyList()
430422
): SerializedVariablesState {
431423
if (!isSerializationActive) return serializedVariablesState
432424

433425
val cellDescriptors = computedDescriptorsPerCell[cellId] ?: return serializedVariablesState
434-
return updateVariableState(cellId, propertyName, cellDescriptors, serializedVariablesState)
426+
return updateVariableState(cellId, propertyName, cellDescriptors[topLevelName]!!, serializedVariablesState)
435427
}
436428

437429
/**
@@ -456,38 +448,39 @@ class VariablesSerializer(
456448
return serializeVariableState(cellId, propertyName, property, value, isRecursive = false, false)
457449
}
458450

459-
private fun serializeVariableState(cellId: Int, name: String?, variableState: VariableState?, isOverride: Boolean = true): SerializedVariablesState {
460-
if (!isSerializationActive || variableState == null || name == null) return SerializedVariablesState()
451+
private fun serializeVariableState(cellId: Int, topLevelName: String?, variableState: VariableState?, isOverride: Boolean = true): SerializedVariablesState {
452+
if (!isSerializationActive || variableState == null || topLevelName == null) return SerializedVariablesState()
461453
// force recursive check
462454
variableState.stringValue
463-
return serializeVariableState(cellId, name, variableState.property, variableState.value.getOrNull(), variableState.isRecursive, isOverride)
455+
return serializeVariableState(cellId, topLevelName, variableState.property, variableState.value.getOrNull(), variableState.isRecursive, isOverride)
464456
}
465457

466-
private fun serializeVariableState(cellId: Int, name: String, property: Field?, value: Any?, isRecursive: Boolean, isOverride: Boolean = true): SerializedVariablesState {
458+
private fun serializeVariableState(cellId: Int, topLevelName: String, property: Field?, value: Any?, isRecursive: Boolean, isOverride: Boolean = true): SerializedVariablesState {
467459
val wrapper = value.toObjectWrapper(isRecursive)
468-
val processedData = createSerializeVariableState(name, getSimpleTypeNameFrom(property, value), wrapper)
469-
return doActualSerialization(cellId, processedData, wrapper, isRecursive, isOverride)
460+
val processedData = createSerializeVariableState(topLevelName, getSimpleTypeNameFrom(property, value), wrapper)
461+
return doActualSerialization(cellId, topLevelName, processedData, wrapper, isRecursive, isOverride)
470462
}
471463

472-
private fun serializeVariableState(cellId: Int, name: String, property: KProperty<*>, value: Any?, isRecursive: Boolean, isOverride: Boolean = true): SerializedVariablesState {
464+
private fun serializeVariableState(cellId: Int, topLevelName: String, property: KProperty<*>, value: Any?, isRecursive: Boolean, isOverride: Boolean = true): SerializedVariablesState {
473465
val wrapper = value.toObjectWrapper(isRecursive)
474-
val processedData = createSerializeVariableState(name, getSimpleTypeNameFrom(property, value), wrapper)
475-
return doActualSerialization(cellId, processedData, wrapper, isRecursive, isOverride)
466+
val processedData = createSerializeVariableState(topLevelName, getSimpleTypeNameFrom(property, value), wrapper)
467+
return doActualSerialization(cellId, topLevelName, processedData, wrapper, isRecursive, isOverride)
476468
}
477469

478-
private fun doActualSerialization(cellId: Int, processedData: ProcessedSerializedVarsState, value: RuntimeObjectWrapper, isRecursive: Boolean, isOverride: Boolean = true): SerializedVariablesState {
470+
private fun doActualSerialization(cellId: Int, topLevelName:String, processedData: ProcessedSerializedVarsState, value: RuntimeObjectWrapper, isRecursive: Boolean, isOverride: Boolean = true): SerializedVariablesState {
479471
val serializedVersion = processedData.serializedVariablesState
480472

481473
seenObjectsPerCell.putIfAbsent(cellId, mutableMapOf())
474+
computedDescriptorsPerCell.putIfAbsent(cellId, mutableMapOf())
482475

483476
if (isOverride) {
484-
val instances = computedDescriptorsPerCell[cellId]?.instancesPerState
485-
computedDescriptorsPerCell[cellId] = ProcessedDescriptorsState()
477+
val instances = computedDescriptorsPerCell[cellId]?.get(topLevelName)?.instancesPerState
478+
computedDescriptorsPerCell[cellId]!![topLevelName] = ProcessedDescriptorsState()
486479
if (instances != null) {
487-
computedDescriptorsPerCell[cellId]!!.instancesPerState += instances
480+
computedDescriptorsPerCell[cellId]!![topLevelName]!!.instancesPerState += instances
488481
}
489482
}
490-
val currentCellDescriptors = computedDescriptorsPerCell[cellId]
483+
val currentCellDescriptors = computedDescriptorsPerCell[cellId]?.get(topLevelName)
491484
// TODO should we stack?
492485
currentCellDescriptors!!.processedSerializedVarsToJavaProperties[serializedVersion] = processedData.propertiesData
493486
currentCellDescriptors.processedSerializedVarsToKTProperties[serializedVersion] = processedData.kPropertiesData
@@ -507,9 +500,9 @@ class VariablesSerializer(
507500
if (kProperties?.size == 1 && kProperties.first().name == "size") {
508501
serializedVersion.fieldDescriptor.addDescriptor(value.objectInstance, "data")
509502
}
510-
iterateThroughContainerMembers(cellId, value.objectInstance, serializedVersion.fieldDescriptor, isRecursive = isRecursive, kProperties = currentCellDescriptors.processedSerializedVarsToKTProperties[serializedVersion])
503+
iterateThroughContainerMembers(cellId, topLevelName, value.objectInstance, serializedVersion.fieldDescriptor, isRecursive = isRecursive, kProperties = currentCellDescriptors.processedSerializedVarsToKTProperties[serializedVersion])
511504
} else {
512-
iterateThroughContainerMembers(cellId, value.objectInstance, serializedVersion.fieldDescriptor, isRecursive = isRecursive, currentCellDescriptors.processedSerializedVarsToJavaProperties[serializedVersion])
505+
iterateThroughContainerMembers(cellId, topLevelName, value.objectInstance, serializedVersion.fieldDescriptor, isRecursive = isRecursive, currentCellDescriptors.processedSerializedVarsToJavaProperties[serializedVersion])
513506
}
514507
}
515508

@@ -518,6 +511,7 @@ class VariablesSerializer(
518511

519512
private fun iterateThroughContainerMembers(
520513
cellId: Int,
514+
topLevelName: String,
521515
callInstance: Any?,
522516
descriptor: MutableFieldDescriptor,
523517
isRecursive: Boolean = false,
@@ -543,7 +537,7 @@ class VariablesSerializer(
543537

544538
seenObjectsPerCell.putIfAbsent(cellId, mutableMapOf())
545539
val seenObjectsPerCell = seenObjectsPerCell[cellId]
546-
val currentCellDescriptors = computedDescriptorsPerCell[cellId]!!
540+
val currentCellDescriptors = computedDescriptorsPerCell[cellId]!![topLevelName]!!
547541
// ok, it's a copy on the left for some reason
548542
val instancesPerState = currentCellDescriptors.instancesPerState
549543

@@ -570,7 +564,7 @@ class VariablesSerializer(
570564
}
571565

572566
val isArrayType = checkForPossibleArray(callInstance)
573-
computedDescriptorsPerCell[cellId]!!.instancesPerState += instancesPerState
567+
computedDescriptorsPerCell[cellId]!![topLevelName]!!.instancesPerState += instancesPerState
574568

575569
if (descriptor.size == 2 && (descriptor.containsKey("data") || descriptor.containsKey("element"))) {
576570
val singleElemMode = descriptor.containsKey("element")
@@ -606,9 +600,10 @@ class VariablesSerializer(
606600
}
607601
}.toObjectWrapper(isRecursive)
608602

609-
computedDescriptorsPerCell[cellId]!!.instancesPerState += instancesPerState
603+
computedDescriptorsPerCell[cellId]!![topLevelName]!!.instancesPerState += instancesPerState
610604
iterateThroughContainerMembers(
611605
cellId,
606+
topLevelName,
612607
neededCallInstance.objectInstance,
613608
serializedVariablesState.fieldDescriptor,
614609
isRecursive = isRecursive,

‎src/test/kotlin/org/jetbrains/kotlinx/jupyter/test/repl/ReplTests.kt

Lines changed: 30 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -757,7 +757,7 @@ class ReplVarsTest : AbstractSingleReplTest() {
757757
val serializer = repl.variablesSerializer
758758
val descriptor = res.evaluatedVariablesState["l"]!!.fieldDescriptor
759759
val innerList = descriptor["elementData"]!!.fieldDescriptor["data"]
760-
val newData = serializer.doIncrementalSerialization(0, "data", innerList!!)
760+
val newData = serializer.doIncrementalSerialization(0, "l", "data", innerList!!)
761761
assertEquals(2, newData.fieldDescriptor.size)
762762
}
763763

@@ -798,7 +798,7 @@ class ReplVarsTest : AbstractSingleReplTest() {
798798
jupyterId = 2
799799
).metadata.evaluatedVariablesState
800800
val innerList = res["l"]!!.fieldDescriptor["elementData"]!!.fieldDescriptor["data"]
801-
val newData = serializer.doIncrementalSerialization(0, "data", innerList!!)
801+
val newData = serializer.doIncrementalSerialization(0, "l","data", innerList!!)
802802
assertTrue(newData.isContainer)
803803
assertTrue(newData.fieldDescriptor.size > 4)
804804
}
@@ -914,7 +914,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
914914
assertEquals(listOf(1, 2, 3, 4).toString().substring(1, actualContainer.value!!.length + 1), actualContainer.value)
915915

916916
val serializer = repl.variablesSerializer
917-
val newData = serializer.doIncrementalSerialization(0, "data", actualContainer)
917+
val newData = serializer.doIncrementalSerialization(0, "x","data", actualContainer)
918918
}
919919

920920
@Test
@@ -1003,7 +1003,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
10031003

10041004
val serializer = repl.variablesSerializer
10051005

1006-
val newData = serializer.doIncrementalSerialization(0, "i", descriptor["i"]!!)
1006+
val newData = serializer.doIncrementalSerialization(0, "c", "i", descriptor["i"]!!)
10071007
}
10081008

10091009
@Test
@@ -1023,7 +1023,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
10231023
val actualContainer = listData.fieldDescriptor.entries.first().value!!
10241024
val serializer = repl.variablesSerializer
10251025

1026-
val newData = serializer.doIncrementalSerialization(0, listData.fieldDescriptor.entries.first().key, actualContainer)
1026+
val newData = serializer.doIncrementalSerialization(0, "x", listData.fieldDescriptor.entries.first().key, actualContainer)
10271027
val receivedDescriptor = newData.fieldDescriptor
10281028
assertEquals(4, receivedDescriptor.size)
10291029

@@ -1036,7 +1036,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
10361036
}
10371037

10381038
val depthMostNode = actualContainer.fieldDescriptor.entries.first { it.value!!.isContainer }
1039-
val serializationAns = serializer.doIncrementalSerialization(0, depthMostNode.key, depthMostNode.value!!)
1039+
val serializationAns = serializer.doIncrementalSerialization(0, "x", depthMostNode.key, depthMostNode.value!!)
10401040
}
10411041

10421042
@Test
@@ -1054,7 +1054,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
10541054
val serializer = repl.variablesSerializer
10551055
val path = listOf("x", "a")
10561056

1057-
val newData = serializer.doIncrementalSerialization(0, listData.fieldDescriptor.entries.first().key, actualContainer, path)
1057+
val newData = serializer.doIncrementalSerialization(0, "x", listData.fieldDescriptor.entries.first().key, actualContainer, path)
10581058
val receivedDescriptor = newData.fieldDescriptor
10591059
assertEquals(4, receivedDescriptor.size)
10601060

@@ -1095,7 +1095,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
10951095

10961096
val serializer = repl.variablesSerializer
10971097

1098-
var newData = serializer.doIncrementalSerialization(0, "values", valuesDescriptor)
1098+
var newData = serializer.doIncrementalSerialization(0, "x", "values", valuesDescriptor)
10991099
var newDescriptor = newData.fieldDescriptor
11001100
assertEquals("4", newDescriptor["size"]!!.value)
11011101
assertEquals(3, newDescriptor["data"]!!.fieldDescriptor.size)
@@ -1110,7 +1110,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
11101110
val entriesDescriptor = listDescriptors["entries"]!!
11111111
assertEquals("4", valuesDescriptor.fieldDescriptor["size"]!!.value)
11121112
assertTrue(valuesDescriptor.fieldDescriptor["data"]!!.isContainer)
1113-
newData = serializer.doIncrementalSerialization(0, "entries", entriesDescriptor)
1113+
newData = serializer.doIncrementalSerialization(0, "x", "entries", entriesDescriptor)
11141114
newDescriptor = newData.fieldDescriptor
11151115
assertEquals("4", newDescriptor["size"]!!.value)
11161116
assertEquals(4, newDescriptor["data"]!!.fieldDescriptor.size)
@@ -1190,7 +1190,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
11901190
val propertyName = listData.fieldDescriptor.entries.first().key
11911191

11921192
runBlocking {
1193-
repl.serializeVariables(1, mapOf(propertyName to actualContainer)) { result ->
1193+
repl.serializeVariables(1, "x", mapOf(propertyName to actualContainer)) { result ->
11941194
val data = result.descriptorsState
11951195
assertTrue(data.isNotEmpty())
11961196

@@ -1251,7 +1251,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
12511251
val propertyName = listData.fieldDescriptor.entries.first().key
12521252

12531253
runBlocking {
1254-
repl.serializeVariables(1, mapOf(propertyName to actualContainer)) { result ->
1254+
repl.serializeVariables(1, "c", mapOf(propertyName to actualContainer)) { result ->
12551255
val data = result.descriptorsState
12561256
assertTrue(data.isNotEmpty())
12571257

@@ -1266,7 +1266,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
12661266

12671267
val anotherI = originalClass.fieldDescriptor["i"]!!
12681268
runBlocking {
1269-
repl.serializeVariables(1, mapOf(propertyName to anotherI)) { res ->
1269+
repl.serializeVariables(1, "c", mapOf(propertyName to anotherI)) { res ->
12701270
val data = res.descriptorsState
12711271
val innerList = data.entries.last().value
12721272
assertTrue(innerList.isContainer)
@@ -1358,4 +1358,22 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
13581358
state = repl.notebook.unchangedVariables()
13591359
assertTrue(state.isEmpty())
13601360
}
1361+
1362+
@Test
1363+
fun testSerializationClearInfo() {
1364+
var res = eval(
1365+
"""
1366+
val x = listOf(1, 2, 3, 4)
1367+
""".trimIndent(),
1368+
jupyterId = 1
1369+
).metadata.evaluatedVariablesState
1370+
var state = repl.notebook.unchangedVariables()
1371+
res = eval(
1372+
"""
1373+
val x = listOf(1, 2, 3, 4)
1374+
""".trimIndent(),
1375+
jupyterId = 2
1376+
).metadata.evaluatedVariablesState
1377+
val a = 1
1378+
}
13611379
}

0 commit comments

Comments
 (0)