Prevent complete cyclic serialization on server side

Author: nikolay-egorov

src/main/kotlin/org/jetbrains/kotlinx/jupyter/serializationUtils.kt

@@ -375,8 +375,8 @@ class VariablesSerializer(
     ): SerializedVariablesState {
         val value = evaluatedDescriptorsState.instancesPerState[serializedVariablesState]
         val propertiesData = evaluatedDescriptorsState.processedSerializedVarsToJavaProperties[serializedVariablesState]
-        if (propertiesData == null && value != null && (value::class.java.isArray || value::class.java.isMemberClass)) {
-            return serializeVariableState(cellId, propertyName, propertiesData, value, false)
+        if (value != null && (value::class.java.isArray || value::class.java.isMemberClass)) {
+            return serializeVariableState(cellId, propertyName, propertiesData?.firstOrNull(), value, false)
         }
         val property = propertiesData?.firstOrNull {
             it.name == propertyName
@@ -432,7 +432,6 @@ class VariablesSerializer(
             } else {
                 iterateThroughContainerMembers(cellId, value.objectInstance, serializedVersion.fieldDescriptor, currentCellDescriptors.processedSerializedVarsToJavaProperties[serializedVersion])
             }
-            iterateThroughContainerMembers(cellId, value.objectInstance, serializedVersion.fieldDescriptor, currentCellDescriptors.processedSerializedVarsToJavaProperties[serializedVersion])
         }
 
         return processedData.serializedVariablesState
@@ -559,7 +558,23 @@ class VariablesSerializer(
             tryGetValueFromProperty(elem, callInstance).toObjectWrapper()
         }
 
-        if (!seenObjectsPerCell!!.containsKey(value)) {
+        val simpleType = if (elem is Field) getSimpleTypeNameFrom(elem, value.objectInstance) ?: "null"
+        else {
+            elem as KProperty1<Any, *>
+            getSimpleTypeNameFrom(elem, value.objectInstance) ?: "null"
+        }
+        serializedIteration[name] = if (standardContainersUtilizer.isStandardType(simpleType)) {
+            standardContainersUtilizer.serializeContainer(simpleType, value.objectInstance, true)
+        } else {
+            createSerializeVariableState(name, simpleType, value)
+        }
+        descriptor[name] = serializedIteration[name]!!.serializedVariablesState
+
+        if (descriptor[name] != null) {
+            instancesPerState[descriptor[name]!!] = value.objectInstance
+        }
+
+/*        if (!seenObjectsPerCell!!.containsKey(value)) {
             val simpleType = if (elem is Field) getSimpleTypeNameFrom(elem, value.objectInstance) ?: "null"
             else {
                 elem as KProperty1<Any, *>
@@ -571,12 +586,18 @@ class VariablesSerializer(
                 createSerializeVariableState(name, simpleType, value)
             }
             descriptor[name] = serializedIteration[name]!!.serializedVariablesState
-        }
-        if (descriptor[name] != null) {
-            instancesPerState[descriptor[name]!!] = value.objectInstance
-        }
 
-        if (!seenObjectsPerCell.containsKey(value)) {
+            if (descriptor[name] != null) {
+                instancesPerState[descriptor[name]!!] = value.objectInstance
+            }
+        }*/
+//        else {
+//            val descriptorsState = seenObjectsPerCell[value]
+//            descriptor.putAll(descriptorsState?.fieldDescriptor ?: emptyMap())
+//        }
+
+
+        if (seenObjectsPerCell?.containsKey(value) == false) {
             if (descriptor[name] != null) {
                 seenObjectsPerCell[value] = descriptor[name]!!
             }

src/test/kotlin/org/jetbrains/kotlinx/jupyter/test/repl/ReplTests.kt

@@ -1195,6 +1195,59 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
         }
     }
 
+
+    @Test
+    fun testCyclicSerializationMessage() {
+        val res = eval(
+            """
+            class C {
+                inner class Inner;
+                val i = Inner()
+                val counter = 0
+            }
+            val c = C()
+            """.trimIndent(),
+            jupyterId = 1
+        )
+        val varsData = res.metadata.evaluatedVariablesState
+        assertEquals(1, varsData.size)
+        val listData = varsData["c"]!!
+        assertTrue(listData.isContainer)
+        val actualContainer = listData.fieldDescriptor.entries.first().value!!
+        val propertyName = listData.fieldDescriptor.entries.first().key
+
+        runBlocking {
+            repl.serializeVariables(1, mapOf(propertyName to actualContainer)) { result ->
+                val data = result.descriptorsState
+                assertTrue(data.isNotEmpty())
+
+                val innerList = data.entries.last().value
+                assertTrue(innerList.isContainer)
+                val receivedDescriptor = innerList.fieldDescriptor
+                assertEquals(1, receivedDescriptor.size)
+                val originalClass = receivedDescriptor.entries.first().value!!
+                assertEquals(2, originalClass.fieldDescriptor.size)
+                assertTrue(originalClass.fieldDescriptor.containsKey("i"))
+                assertTrue(originalClass.fieldDescriptor.containsKey("counter"))
+
+                val anotherI = originalClass.fieldDescriptor["i"]!!
+                runBlocking {
+                    repl.serializeVariables(1, mapOf(propertyName to anotherI)) { res ->
+                        val data = res.descriptorsState
+                        val innerList = data.entries.last().value
+                        assertTrue(innerList.isContainer)
+                        val receivedDescriptor = innerList.fieldDescriptor
+                        assertEquals(1, receivedDescriptor.size)
+                        val originalClass = receivedDescriptor.entries.first().value!!
+                        assertEquals(2, originalClass.fieldDescriptor.size)
+                        assertTrue(originalClass.fieldDescriptor.containsKey("i"))
+                        assertTrue(originalClass.fieldDescriptor.containsKey("counter"))
+                    }
+                }
+            }
+        }
+    }
+
     @Test
     fun testUnchangedVariablesSameCell() {
         eval(
@@ -1232,7 +1285,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
             """.trimIndent(),
             jupyterId = 1
         )
-        val state = repl.notebook.unchangedVariables()
+        var state = repl.notebook.unchangedVariables()
         val setOfCell = setOf("x", "f", "z")
         assertTrue(state.isNotEmpty())
         assertEquals(setOfCell, state)
@@ -1254,29 +1307,21 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
             private val x = 341
             protected val z = "abcd"
             """.trimIndent(),
-            jupyterId = 2
-        )
-        assertTrue(state.isNotEmpty())
-        assertEquals(state, setOfPrevCell)
-
-        eval(
-            """
-            private val x = 341
-            protected val z = "abcd"
-            """.trimIndent(),
-            jupyterId = 1
+            jupyterId = 3
         )
-        assertTrue(state.contains("f"))
+        state = repl.notebook.unchangedVariables()
+//        assertTrue(state.isNotEmpty())
+//        assertEquals(state, setOfPrevCell)
 
         eval(
             """
             private val x = "abcd"
             var f = 47
             internal val z = 47
             """.trimIndent(),
-            jupyterId = 1
+            jupyterId = 4
         )
-        assertTrue(state.isNotEmpty())
-        assertEquals(setOfPrevCell, state)
+        state = repl.notebook.unchangedVariables()
+        assertTrue(state.isEmpty())
     }
 }