Skip to content

Commit 694ba47

Browse files
Consider anonymous classes; still need a bit of to do
1 parent b7bc217 commit 694ba47

File tree

6 files changed

+187
-51
lines changed

6 files changed

+187
-51
lines changed

build.gradle.kts

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ plugins {
1212

1313
val deploy: Configuration by configurations.creating
1414

15+
val serializationFlagProperty = "jupyter.serialization.enabled"
16+
1517
deploy.apply {
1618
exclude("org.jetbrains.kotlinx", "kotlinx-serialization-json-jvm")
1719
exclude("org.jetbrains.kotlinx", "kotlinx-serialization-core-jvm")
@@ -112,7 +114,8 @@ tasks {
112114

113115
"junit.jupiter.execution.parallel.enabled" to doParallelTesting.toString() as Any,
114116
"junit.jupiter.execution.parallel.mode.default" to "concurrent",
115-
"junit.jupiter.execution.parallel.mode.classes.default" to "concurrent"
117+
"junit.jupiter.execution.parallel.mode.classes.default" to "concurrent",
118+
serializationFlagProperty to "true"
116119
)
117120
}
118121

jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/compiler/util/serializedCompiledScript.kt

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,13 @@ data class SerializedCompiledScriptsData(
2222

2323
@Serializable
2424
data class SerializedVariablesState(
25-
val name: String = "",
2625
val type: String = "",
2726
val value: String? = null,
2827
val isContainer: Boolean = false
2928
) {
3029
val fieldDescriptor: MutableMap<String, SerializedVariablesState?> = mutableMapOf()
3130
}
3231

33-
3432
@Serializable
3533
class EvaluatedSnippetMetadata(
3634
val newClasspath: Classpath = emptyList(),

src/main/kotlin/org/jetbrains/kotlinx/jupyter/repl.kt

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -421,8 +421,6 @@ class ReplForJupyterImpl(
421421
notebook.updateVariablesState(internalEvaluator)
422422
// printVars()
423423
// printUsagesInfo(jupyterId, cellVariables[jupyterId - 1])
424-
val entry = notebook.variablesState.entries.lastOrNull()
425-
val serializedVarsState = variablesSerializer.serializeVariableState(jupyterId - 1, entry?.key, entry?.value)
426424
val serializedData = variablesSerializer.serializeVariables(jupyterId - 1, notebook.variablesState)
427425

428426

src/main/kotlin/org/jetbrains/kotlinx/jupyter/serializationUtils.kt

Lines changed: 147 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,11 @@ package org.jetbrains.kotlinx.jupyter
22

33
import org.jetbrains.kotlinx.jupyter.api.VariableState
44
import org.jetbrains.kotlinx.jupyter.compiler.util.SerializedVariablesState
5+
import java.lang.reflect.Field
56
import kotlin.reflect.KClass
67
import kotlin.reflect.KProperty
78
import kotlin.reflect.KProperty1
9+
import kotlin.reflect.KTypeParameter
810
import kotlin.reflect.full.declaredMemberProperties
911
import kotlin.reflect.full.isSubclassOf
1012
import kotlin.reflect.jvm.isAccessible
@@ -13,14 +15,16 @@ typealias FieldDescriptor = Map<String, SerializedVariablesState?>
1315
typealias MutableFieldDescriptor = MutableMap<String, SerializedVariablesState?>
1416
typealias PropertiesData = Collection<KProperty1<out Any, *>>
1517

16-
data class ProcessedSerializedVarsState(
18+
class ProcessedSerializedVarsState(
1719
val serializedVariablesState: SerializedVariablesState,
18-
val propertiesData: PropertiesData?
20+
val propertiesData: PropertiesData?,
21+
val jvmOnlyFields: Array<Field>? = null
1922
)
2023

2124
data class ProcessedDescriptorsState(
22-
// perhaps, better tp make SerializedVariablesState -> PropertiesData?
23-
val processedSerializedVarsState: MutableMap<SerializedVariablesState, PropertiesData?> = mutableMapOf(),
25+
val processedSerializedVarsToKProperties: MutableMap<SerializedVariablesState, PropertiesData?> = mutableMapOf(),
26+
// do we need this? Probably, not
27+
// val processedSerializedVarsToJvmFields: MutableMap<SerializedVariablesState, Array<Field>?> = mutableMapOf(),
2428
val instancesPerState: MutableMap<SerializedVariablesState, Any?> = mutableMapOf()
2529
)
2630

@@ -41,8 +45,11 @@ class VariablesSerializer(private val serializationStep: Int = 2, private val se
4145
*/
4246
private val computedDescriptorsPerCell: MutableMap<Int, ProcessedDescriptorsState> = mutableMapOf()
4347

48+
private val isSerializationActive: Boolean = System.getProperty(serializationEnvProperty)?.toBooleanStrictOrNull() ?: true
4449

4550
fun serializeVariables(cellId: Int, variablesState: Map<String, VariableState>): Map<String, SerializedVariablesState> {
51+
if (!isSerializationActive) return emptyMap()
52+
4653
if (seenObjectsPerCell.containsKey(cellId)) {
4754
seenObjectsPerCell[cellId]!!.clear()
4855
}
@@ -54,6 +61,8 @@ class VariablesSerializer(private val serializationStep: Int = 2, private val se
5461
}
5562

5663
fun doIncrementalSerialization(cellId: Int, propertyName: String, serializedVariablesState: SerializedVariablesState): SerializedVariablesState {
64+
if (!isSerializationActive) return serializedVariablesState
65+
5766
val cellDescriptors = computedDescriptorsPerCell[cellId] ?: return serializedVariablesState
5867
return updateVariableState(cellId, propertyName, cellDescriptors, serializedVariablesState)
5968
}
@@ -62,12 +71,16 @@ class VariablesSerializer(private val serializationStep: Int = 2, private val se
6271
* @param evaluatedDescriptorsState - origin variable state to get value from
6372
* @param serializedVariablesState - current state of recursive state to go further
6473
*/
65-
private fun updateVariableState(cellId: Int, propertyName: String, evaluatedDescriptorsState: ProcessedDescriptorsState,
66-
serializedVariablesState: SerializedVariablesState): SerializedVariablesState {
74+
private fun updateVariableState(
75+
cellId: Int,
76+
propertyName: String,
77+
evaluatedDescriptorsState: ProcessedDescriptorsState,
78+
serializedVariablesState: SerializedVariablesState
79+
): SerializedVariablesState {
6780
val value = evaluatedDescriptorsState.instancesPerState[serializedVariablesState]
68-
val propertiesData = evaluatedDescriptorsState.processedSerializedVarsState[serializedVariablesState]
69-
if (propertiesData == null && value != null && value::class.java.isArray) {
70-
return serializeVariableState(cellId, propertyName, null, value, false)
81+
val propertiesData = evaluatedDescriptorsState.processedSerializedVarsToKProperties[serializedVariablesState]
82+
if (propertiesData == null && value != null && (value::class.java.isArray || value::class.java.isMemberClass)) {
83+
return serializeVariableState(cellId, propertyName, propertiesData, value, false)
7184
}
7285
val property = propertiesData?.firstOrNull {
7386
it.name == propertyName
@@ -76,14 +89,26 @@ class VariablesSerializer(private val serializationStep: Int = 2, private val se
7689
return serializeVariableState(cellId, propertyName, property, value, false)
7790
}
7891

79-
8092
fun serializeVariableState(cellId: Int, name: String?, variableState: VariableState?, isOverride: Boolean = true): SerializedVariablesState {
81-
if (variableState == null || name == null) return SerializedVariablesState()
93+
if (!isSerializationActive || variableState == null || name == null) return SerializedVariablesState()
8294
return serializeVariableState(cellId, name, variableState.property, variableState.value, isOverride)
8395
}
8496

85-
fun serializeVariableState(cellId: Int, name: String, property: KProperty<*>?, value: Any?, isOverride: Boolean = true): SerializedVariablesState {
86-
val processedData = createSerializeVariableState(name, property, value)
97+
private fun serializeVariableState(cellId: Int, name: String, property: Field?, value: Any?, isOverride: Boolean = true): SerializedVariablesState {
98+
val processedData = createSerializeVariableState(name, getSimpleTypeNameFrom(property, value), value)
99+
return doActualSerialization(cellId, processedData, value, isOverride)
100+
}
101+
102+
private fun serializeVariableState(cellId: Int, name: String, property: KProperty<*>, value: Any?, isOverride: Boolean = true): SerializedVariablesState {
103+
val processedData = createSerializeVariableState(name, getSimpleTypeNameFrom(property, value), value)
104+
return doActualSerialization(cellId, processedData, value, isOverride)
105+
}
106+
107+
private fun doActualSerialization(cellId: Int, processedData: ProcessedSerializedVarsState, value: Any?, isOverride: Boolean = true): SerializedVariablesState {
108+
fun isCanBeComputed(fieldDescriptors: MutableMap<String, SerializedVariablesState?>): Boolean {
109+
return (fieldDescriptors.isEmpty() || (fieldDescriptors.isNotEmpty() && fieldDescriptors.entries.first().value?.fieldDescriptor!!.isEmpty()))
110+
}
111+
87112
val serializedVersion = processedData.serializedVariablesState
88113

89114
seenObjectsPerCell.putIfAbsent(cellId, mutableMapOf())
@@ -92,26 +117,40 @@ class VariablesSerializer(private val serializationStep: Int = 2, private val se
92117
computedDescriptorsPerCell[cellId] = ProcessedDescriptorsState()
93118
}
94119
val currentCellDescriptors = computedDescriptorsPerCell[cellId]
95-
currentCellDescriptors!!.processedSerializedVarsState[serializedVersion] = processedData.propertiesData
120+
currentCellDescriptors!!.processedSerializedVarsToKProperties[serializedVersion] = processedData.propertiesData
121+
// currentCellDescriptors.processedSerializedVarsToJvmFields[serializedVersion] = processedData.jvmOnlyFields
96122

97123
if (value != null) {
98-
seenObjectsPerCell[cellId]!![value] = serializedVersion
124+
seenObjectsPerCell[cellId]!!.putIfAbsent(value, serializedVersion)
99125
}
100126
if (serializedVersion.isContainer) {
101-
iterateThroughContainerMembers(cellId, value, serializedVersion.fieldDescriptor, currentCellDescriptors.processedSerializedVarsState[serializedVersion])
127+
// check for seen
128+
if (seenObjectsPerCell[cellId]!!.containsKey(value)) {
129+
val previouslySerializedState = seenObjectsPerCell[cellId]!![value] ?: return processedData.serializedVariablesState
130+
serializedVersion.fieldDescriptor += previouslySerializedState.fieldDescriptor
131+
if (isCanBeComputed(serializedVersion.fieldDescriptor)) {
132+
iterateThroughContainerMembers(cellId, value, serializedVersion.fieldDescriptor, currentCellDescriptors.processedSerializedVarsToKProperties[serializedVersion])
133+
}
134+
} else {
135+
// add jvm descriptors
136+
processedData.jvmOnlyFields?.forEach {
137+
serializedVersion.fieldDescriptor[it.name] = serializeVariableState(cellId, it.name, it, value)
138+
}
139+
iterateThroughContainerMembers(cellId, value, serializedVersion.fieldDescriptor, currentCellDescriptors.processedSerializedVarsToKProperties[serializedVersion])
140+
}
102141
}
103142
return processedData.serializedVariablesState
104143
}
105144

106-
107-
private fun iterateThroughContainerMembers(cellId: Int, callInstance: Any?, descriptor: MutableFieldDescriptor, properties: PropertiesData?, currentDepth: Int = 0): Unit {
145+
private fun iterateThroughContainerMembers(cellId: Int, callInstance: Any?, descriptor: MutableFieldDescriptor, properties: PropertiesData?, currentDepth: Int = 0) {
108146
if (properties == null || callInstance == null || currentDepth >= serializationStep) return
109147

110148
val serializedIteration = mutableMapOf<String, ProcessedSerializedVarsState>()
111149

112150
seenObjectsPerCell.putIfAbsent(cellId, mutableMapOf())
113151
val seenObjectsPerCell = seenObjectsPerCell[cellId]
114152
val currentCellDescriptors = computedDescriptorsPerCell[cellId]!!
153+
// ok, it's a copy on the left for some reason
115154
val instancesPerState = currentCellDescriptors.instancesPerState
116155

117156
for (it in properties) {
@@ -123,7 +162,7 @@ class VariablesSerializer(private val serializationStep: Int = 2, private val se
123162
val value = tryGetValueFromProperty(it, callInstance)
124163

125164
if (!seenObjectsPerCell!!.containsKey(value)) {
126-
serializedIteration[name] = createSerializeVariableState(name, it, value)
165+
serializedIteration[name] = createSerializeVariableState(name, getSimpleTypeNameFrom(it, value), value)
127166
descriptor[name] = serializedIteration[name]!!.serializedVariablesState
128167
}
129168
if (descriptor[name] != null) {
@@ -143,6 +182,26 @@ class VariablesSerializer(private val serializationStep: Int = 2, private val se
143182
}
144183

145184
val isArrayType = checkCreateForPossibleArray(callInstance, descriptor, serializedIteration)
185+
computedDescriptorsPerCell[cellId]!!.instancesPerState += instancesPerState
186+
187+
// check for seen
188+
// for now it's O(c*n)
189+
if (serializedIteration.isEmpty()) {
190+
val processedVars = computedDescriptorsPerCell[cellId]!!.processedSerializedVarsToKProperties
191+
descriptor.forEach { (_, state) ->
192+
if (processedVars.containsKey(state)) {
193+
processedVars.entries.firstOrNull {
194+
val itValue = it.key
195+
if (itValue.value == state?.value && itValue.type == state?.value) {
196+
state?.fieldDescriptor?.put(itValue.type, itValue)
197+
true
198+
} else {
199+
false
200+
}
201+
}
202+
}
203+
}
204+
}
146205

147206
serializedIteration.forEach {
148207
val serializedVariablesState = it.value.serializedVariablesState
@@ -155,61 +214,108 @@ class VariablesSerializer(private val serializationStep: Int = 2, private val se
155214
isArrayType -> {
156215
callInstance
157216
}
158-
else -> { null }
217+
else -> {
218+
null
219+
}
159220
}
160221
if (isArrayType) {
161222
if (callInstance is List<*>) {
162223
callInstance.forEach { arrayElem ->
163-
iterateThroughContainerMembers(cellId, arrayElem, serializedVariablesState.fieldDescriptor,
164-
it.value.propertiesData, currentDepth + 1)
224+
iterateThroughContainerMembers(
225+
cellId,
226+
arrayElem,
227+
serializedVariablesState.fieldDescriptor,
228+
it.value.propertiesData,
229+
currentDepth + 1
230+
)
165231
}
166232
} else {
167233
callInstance as Array<*>
168234
callInstance.forEach { arrayElem ->
169-
iterateThroughContainerMembers(cellId, arrayElem, serializedVariablesState.fieldDescriptor,
170-
it.value.propertiesData, currentDepth + 1)
235+
iterateThroughContainerMembers(
236+
cellId,
237+
arrayElem,
238+
serializedVariablesState.fieldDescriptor,
239+
it.value.propertiesData,
240+
currentDepth + 1
241+
)
171242
}
172243
}
173244

174245
return@forEach
175246
}
176-
iterateThroughContainerMembers(cellId, neededCallInstance, serializedVariablesState.fieldDescriptor,
177-
it.value.propertiesData, currentDepth + 1)
247+
248+
// update state with JVMFields
249+
it.value.jvmOnlyFields?.forEach { field ->
250+
serializedVariablesState.fieldDescriptor[field.name] = serializeVariableState(cellId, field.name, field, neededCallInstance)
251+
val properInstance = serializedVariablesState.fieldDescriptor[field.name]
252+
instancesPerState[properInstance!!] = neededCallInstance
253+
seenObjectsPerCell?.set(neededCallInstance!!, serializedVariablesState)
254+
}
255+
computedDescriptorsPerCell[cellId]!!.instancesPerState += instancesPerState
256+
// computedDescriptorsPerCell[cellId]!!.processedSerializedVarsToJvmFields[serializedVariablesState] = it.value.jvmOnlyFields
257+
iterateThroughContainerMembers(
258+
cellId,
259+
neededCallInstance,
260+
serializedVariablesState.fieldDescriptor,
261+
it.value.propertiesData,
262+
currentDepth + 1
263+
)
178264
}
179265
}
180266
}
181267

268+
private fun getSimpleTypeNameFrom(property: Field?, value: Any?): String? {
269+
return if (property != null) {
270+
val returnType = property.type
271+
returnType.simpleName
272+
} else {
273+
value?.toString()
274+
}
275+
}
182276

183-
private fun createSerializeVariableState(name: String, property: KProperty<*>?, value: Any?): ProcessedSerializedVarsState {
184-
val simpleName = if (property != null) {
277+
private fun getSimpleTypeNameFrom(property: KProperty<*>?, value: Any?): String? {
278+
return if (property != null) {
185279
val returnType = property.returnType
186-
val classifier = returnType.classifier as KClass<*>
187-
classifier.simpleName
280+
val classifier = returnType.classifier
281+
if (classifier is KTypeParameter) {
282+
classifier.name
283+
} else {
284+
(classifier as KClass<*>).simpleName
285+
}
188286
} else {
189287
value?.toString()
190288
}
289+
}
191290

291+
private fun createSerializeVariableState(name: String, simpleTypeName: String?, value: Any?): ProcessedSerializedVarsState {
192292
// make it exception-safe
193293
val membersProperties = try {
194294
if (value != null) value::class.declaredMemberProperties else null
195295
} catch (e: Throwable) {
196296
null
197297
}
198-
val isContainer = if (membersProperties != null) (membersProperties.size > 1 || value!!::class.java.isArray) else false
199-
val type = if (value!= null && value::class.java.isArray) {
298+
val javaClass = value?.javaClass
299+
val jvmFields = if (javaClass != null && javaClass.isMemberClass) {
300+
javaClass.declaredFields
301+
} else { null }
302+
303+
val isContainer = if (membersProperties != null) (
304+
membersProperties.isNotEmpty() || value!!::class.java.isArray || (javaClass != null && javaClass.isMemberClass)
305+
) else false
306+
val type = if (value != null && value::class.java.isArray) {
200307
"Array"
201308
} else if (isContainer && value is List<*>) {
202309
"SingletonList"
203310
} else {
204-
simpleName.toString()
311+
simpleTypeName.toString()
205312
}
206313

207-
val serializedVariablesState = SerializedVariablesState(name, type, getProperString(value), isContainer)
314+
val serializedVariablesState = SerializedVariablesState(type, getProperString(value), isContainer)
208315

209-
return ProcessedSerializedVarsState(serializedVariablesState, membersProperties)
316+
return ProcessedSerializedVarsState(serializedVariablesState, membersProperties, jvmFields)
210317
}
211318

212-
213319
private fun tryGetValueFromProperty(property: KProperty1<Any, *>, callInstance: Any): Any? {
214320
// some fields may be optimized out like array size. Thus, calling it.isAccessible would return error
215321
val canAccess = try {
@@ -244,12 +350,13 @@ class VariablesSerializer(private val serializationStep: Int = 2, private val se
244350
}
245351
}
246352

353+
companion object {
354+
const val serializationEnvProperty = "jupyter.serialization.enabled"
355+
}
247356
}
248357

249-
// TODO: maybe think of considering the depth?
250358
fun getProperString(value: Any?): String {
251-
252-
fun print(builder: StringBuilder, containerSize:Int, index: Int, value: Any?): Unit {
359+
fun print(builder: StringBuilder, containerSize: Int, index: Int, value: Any?) {
253360
if (index != containerSize - 1) {
254361
builder.append(value, ", ")
255362
} else {
@@ -296,4 +403,4 @@ fun getProperString(value: Any?): String {
296403
}
297404

298405
return value.toString()
299-
}
406+
}

0 commit comments

Comments
 (0)