temp: PoC for merged document serialization/deserialization

Author: b41ex

package-lock.json

@@ -29,13 +29,14 @@
   "dependencies": {
     "@netcracker/qubership-apihub-api-unifier": "2.0.0",
     "@netcracker/qubership-apihub-json-crawl": "1.0.4",
-    "fast-equals": "4.0.3"
+    "fast-equals": "4.0.3",
+    "flatted": "^3.2.9"    
   },
   "devDependencies": {
     "@netcracker/qubership-apihub-compatibility-suites": "2.0.3",
     "@netcracker/qubership-apihub-graphapi": "1.0.8",
     "@netcracker/qubership-apihub-npm-gitflow": "3.1.0",
-    "@types/jest": "29.5.11",
+    "@types/jest": "29.5.11",    
     "@types/node": "20.11.6",
     "@typescript-eslint/eslint-plugin": "6.13.2",
     "@typescript-eslint/parser": "6.13.2",

package.json

@@ -16,6 +16,7 @@ import {
   SpecType,
 } from '@netcracker/qubership-apihub-api-unifier'
 import { DEFAULT_NORMALIZED_RESULT, DEFAULT_OPTION_DEFAULTS_META_KEY, DEFAULT_OPTION_ORIGINS_META_KEY, DIFF_META_KEY } from './core'
+import { serialize, deserialize } from './utils'
 
 export const COMPARE_ENGINES_MAP: Record<SpecType, CompareEngine> = {
   [SPEC_TYPE_JSON_SCHEMA_04]: compareJsonSchema(SPEC_TYPE_JSON_SCHEMA_04),
@@ -35,10 +36,16 @@ export function apiDiff(before: unknown, after: unknown, options: CompareOptions
     throw new Error(`Specification cannot be different. Got ${beforeSpec.type} and ${afterSpec.type}`)
   }
   const engine = COMPARE_ENGINES_MAP[beforeSpec.type]
-  return engine(before, after, {
+  
+  // Determine the metaKey to use (from options or default)
+  const metaKey = options.metaKey || DIFF_META_KEY
+  
+  // Time diff building
+  const diffStartTime = performance.now()
+  const result = engine(before, after, {
     mode: COMPARE_MODE_DEFAULT,
     normalizedResult: DEFAULT_NORMALIZED_RESULT,
-    metaKey: DIFF_META_KEY,
+    metaKey: metaKey,
     defaultsFlag: DEFAULT_OPTION_DEFAULTS_META_KEY,
     originsFlag: DEFAULT_OPTION_ORIGINS_META_KEY,
     compareScope: COMPARE_SCOPE_ROOT,
@@ -48,4 +55,34 @@ export function apiDiff(before: unknown, after: unknown, options: CompareOptions
     createdMergedJso: new Set(),
     ...options,
   })
+  const diffEndTime = performance.now()
+
+  // Serialize and deserialize the merged document, mapping metaKey to '__diff'
+  const symbolToStringMapping = new Map<symbol, string>()
+  symbolToStringMapping.set(metaKey, '__diff')
+  
+  const stringToSymbolMapping = new Map<string, symbol>()
+  stringToSymbolMapping.set('__diff', metaKey)
+  
+  // Time serialization
+  const serializeStartTime = performance.now()
+  const serialized = serialize(result.merged, symbolToStringMapping)
+  const serializeEndTime = performance.now()
+
+  // Time deserialization
+  const deserializeStartTime = performance.now()
+  const deserializedMerged = deserialize(serialized, stringToSymbolMapping)
+  const deserializeEndTime = performance.now()
+
+  // Log all timings in one line
+  const diffTime = diffEndTime - diffStartTime
+  const serializeTime = serializeEndTime - serializeStartTime
+  const deserializeTime = deserializeEndTime - deserializeStartTime
+  const totalTime = deserializeEndTime - diffStartTime
+  console.log(`ApiDiff timing - Diff: ${diffTime.toFixed(2)}ms, Serialize: ${serializeTime.toFixed(2)}ms, Deserialize: ${deserializeTime.toFixed(2)}ms, Total: ${totalTime.toFixed(2)}ms`)
+
+  return {
+    ...result,
+    merged: deserializedMerged,
+  }
 }

src/api.ts

@@ -23,5 +23,7 @@ export {
   isDiffRemove,
   isDiffRename,
   isDiffReplace,
+  serialize,
+  deserialize,
 } from './utils'
 export { onlyExistedArrayIndexes } from './utils'

src/index.ts

@@ -23,6 +23,7 @@ import {
   JSON_SCHEMA_NODE_TYPE_STRING,
   JsonSchemaNodesNormalizedType,
 } from '@netcracker/qubership-apihub-api-unifier'
+import { stringify, parse } from 'flatted'
 
 export const isObject = (value: unknown): value is Record<string | symbol, unknown> => {
   return typeof value === 'object' && value !== null
@@ -221,3 +222,169 @@ export const checkPrimitiveType = (value: unknown): PrimitiveType | undefined =>
   }
   return undefined
 }
+
+// This is just a PoC to serialize/deserialize the merged document.
+// Merged document is a JSON object with cycles and symbol keys.
+// Symbol key could be both in object and array.
+// flatted.stringify doesn't serialize custom properties on arrays
+// flatted.parse doesn't deserialize custom properties on arrays
+// lodash cloneDeepWith does not copy custom properties on arrays
+// Approximate times:
+// - OAS large x6: Diff: 57914.54ms, Serialize: 488.82ms, Deserialize: 1057.61ms
+// - GQL1: Diff: 25495.87ms, Serialize: 255.23ms, Deserialize: 477.04ms
+// - GQL2: Diff: 16599.96ms, Serialize: 341.62ms, Deserialize: 527.20ms
+// - Shopify (GQL): Diff: 21453.64ms, Serialize: 186.46ms, Deserialize: 371.04ms
+/**
+ * Serializes an object with cycles and symbol substitution to a string
+ * @param obj - The object to serialize (can contain cycles and Symbol keys)
+ * @param symbolToStringMapping - Mapping from Symbol keys to string keys
+ * @returns Serialized string representation
+ */
+export const serialize = (obj: unknown, symbolToStringMapping: Map<symbol, string>): string => {
+  
+  // Walk the object and replace symbol keys, handling cycles
+  const visited = new WeakSet()
+  
+  const replaceSymbolKeys = (object: any): any => {
+    // Handle cycles by tracking visited objects
+    if (isObject(object) && visited.has(object)) {
+      return object
+    }
+    
+    if (isObject(object)) {
+      visited.add(object)
+      
+      // Process symbol keys for both arrays and objects
+      const symbolKeys = Object.getOwnPropertySymbols(object)
+      let hasSymbolKeys = false
+      for (const symbolKey of symbolKeys) {
+        const stringKey = symbolToStringMapping.get(symbolKey)
+        if (stringKey) {
+          // Move value from symbol key to string key
+          object[stringKey] = replaceSymbolKeys(object[symbolKey])
+          delete object[symbolKey]
+          hasSymbolKeys = true
+        }
+      }
+      
+      if (isArray(object)) {
+        // If array has symbol keys converted to string keys, we need to convert it to a plain object
+        // because flatted.stringify doesn't serialize custom properties on arrays
+        if (hasSymbolKeys) {
+          const arrayAsObject: any = { __isArray: true }
+          // Copy array elements
+          for (let i = 0; i < object.length; i++) {
+            arrayAsObject[i] = replaceSymbolKeys(object[i])
+          }
+          // Copy any additional string properties (converted from symbols)
+          for (const [key, value] of Object.entries(object)) {
+            if (!(/^\d+$/.test(key))) { // Skip numeric indices
+              arrayAsObject[key] = replaceSymbolKeys(value)
+            }
+          }
+          arrayAsObject.length = object.length
+          return arrayAsObject
+        } else {
+          // Process array elements normally
+          for (let i = 0; i < object.length; i++) {
+            object[i] = replaceSymbolKeys(object[i])
+          }
+        }
+      } else {
+        // Process regular properties for objects
+        for (const [key, objValue] of Object.entries(object)) {
+          object[key] = replaceSymbolKeys(objValue)
+        }
+      }
+    }
+    
+    return object
+  }
+  
+  const processedObj = replaceSymbolKeys(obj)
+  return stringify(processedObj)
+}
+
+/**
+ * Deserializes a string back to an object with symbol key restoration
+ * @param str - The serialized string
+ * @param stringToSymbolMapping - Mapping from string keys to Symbol keys
+ * @returns Deserialized object with Symbol keys restored
+ */
+export const deserialize = (str: string, stringToSymbolMapping: Map<string, symbol>): unknown => {
+  // First, parse the string using flatted
+  const parsedObj = parse(str)
+  
+  // Then walk the parsed object and replace string keys with symbol keys, handling cycles
+  const visited = new WeakSet()
+  
+  const replaceStringKeys = (value: any): any => {
+    // Handle cycles by tracking visited objects
+    if (isObject(value) && visited.has(value)) {
+      return value
+    }
+    
+    if (isObject(value)) {
+      visited.add(value)
+      
+      // Check if this is a serialized array (converted to object during serialization)
+      if (value.__isArray === true) {
+        const arr: any[] = new Array(value.length || 0)
+        
+        // Restore array elements
+        for (let i = 0; i < arr.length; i++) {
+          if (i in value) {
+            arr[i] = replaceStringKeys(value[i])
+          }
+        }
+        
+        // Restore additional properties (including converted symbol keys)
+        for (const [key, objValue] of Object.entries(value)) {
+          if (key !== '__isArray' && key !== 'length' && !(/^\d+$/.test(key))) {
+            const symbolKey = stringToSymbolMapping.get(key)
+            if (symbolKey) {
+              (arr as any)[symbolKey] = replaceStringKeys(objValue)
+            } else {
+              (arr as any)[key] = replaceStringKeys(objValue)
+            }
+          }
+        }
+        
+        return arr
+      }
+      
+      // Process string keys that should be converted to symbol keys for both arrays and objects
+      const keysToReplace: Array<[string, symbol]> = []
+      
+      // First, identify which keys need to be replaced
+      for (const key of Object.keys(value)) {
+        const symbolKey = stringToSymbolMapping.get(key)
+        if (symbolKey) {
+          keysToReplace.push([key, symbolKey])
+        }
+      }
+      
+      // Replace string keys with symbol keys
+      for (const [stringKey, symbolKey] of keysToReplace) {
+        value[symbolKey] = replaceStringKeys(value[stringKey])
+        delete value[stringKey]
+      }
+      
+      if (isArray(value)) {
+        // Process array elements
+        for (let i = 0; i < value.length; i++) {
+          value[i] = replaceStringKeys(value[i])
+        }
+      } else {
+        // Process remaining properties for objects
+        for (const [key, objValue] of Object.entries(value)) {
+          value[key] = replaceStringKeys(objValue)
+        }
+      }
+    }
+    
+    return value
+  }
+  
+  return replaceStringKeys(parsedObj)
+}

src/utils.ts

@@ -208,6 +208,7 @@ describe('JSON schema changes', () => {
       expect(merged).not.toHaveProperty('writeOnly')
     })
 
+    // expected to fail in this PoC, we serialize/deserialize the merged document, but not the diffs
     it('diff share share instance', () => {
       const before = {
         type: 'object',