feat: Calculate hash and semantic hash in single pass for the document

package.json

@@ -41,24 +41,25 @@
   ],
   "dependencies": {
     "@netcracker/qubership-apihub-json-crawl": "1.0.4",
-    "object-hash": "3.0.0",
-    "fast-equals": "4.0.3"
+    "fast-equals": "4.0.3",
+    "object-hash": "3.0.0"
   },
   "devDependencies": {
+    "@jest/globals": "29.5.0",
     "@netcracker/qubership-apihub-graphapi": "feature-performance-optimization",
     "@netcracker/qubership-apihub-npm-gitflow": "3.1.0",
-    "@types/object-hash": "3.0.6",
     "@types/jest": "29.5.2",
     "@types/js-yaml": "4.0.5",
+    "@types/object-hash": "3.0.6",
     "@typescript-eslint/eslint-plugin": "6.0.0",
     "@typescript-eslint/parser": "6.0.0",
     "eslint": "8.48.0",
     "graphql": "16.9.0",
     "jest": "29.5.0",
     "jest-extended": "4.0.2",
     "js-yaml": "4.1.0",
-    "openapi-types": "12.1.3",
     "json-schema-merge-allof": "0.8.1",
+    "openapi-types": "12.1.3",
     "rimraf": "5.0.5",
     "ts-jest": "29.1.0",
     "ts-loader": "9.4.3",

src/hash.ts

@@ -1,101 +1,60 @@
-import {
-  BEFORE_SECOND_DATA_LEVEL as SECOND_DATA_LEVELS,
-  CURRENT_DATA_LEVEL,
-  HashOptions,
-  InternalHashOptions,
-  NormalizationRule,
-} from './types'
+import { HashOptions, InternalHashOptions, NormalizationRule } from './types'
 import { resolveSpec, SPEC_TYPE_GRAPH_API } from './spec-type'
-import { CrawlRules, isObject, syncClone, SyncCloneHook, syncCrawl, SyncCrawlHook } from '@netcracker/qubership-apihub-json-crawl'
+import { isObject, syncCrawl, SyncCrawlHook } from '@netcracker/qubership-apihub-json-crawl'
 import { RULES } from './rules'
-import objectHash, { NotUndefined } from 'object-hash'
+import { cryptoMd5, objectToString } from './utils'
 
-const calculateHash: (object: NotUndefined) => string = (object) => {
-  const res = objectHash(object, {
-    unorderedArrays: true,
-    unorderedObjects: true,
-    algorithm: 'md5',
-  })
-  return res
-}
+export type ObjPath = (string | number)[]
 
-const createHashObject: (object: NotUndefined, rules: CrawlRules<NormalizationRule>) => NotUndefined = (object, rules) => {
-  const creatorHook = createHashObjectCreatorHook()
-  return syncClone(object, creatorHook, { state: { dataLevel: 0 }, rules }) as NotUndefined
-}
+//todo only for tests
+const optionalFieldFlag = Symbol('optional-fields')
+
+const createHashObjectCreatorHook: (options: HashOptions) => HashScannerCrawlHook = (options) => {
+  const { semanticHashProperty, hashProperty } = options
 
-const createHashObjectCreatorHook: () => HashObjectCreatorCrawlHook = () => {
   const cycleGuard: Set<unknown> = new Set()
-  const creatorHook: HashObjectCreatorCrawlHook = ({ key, value, rules, state }) => {
+  const optionalFieldStates = new Map<any, ObjPath>()
+
+  const generateHash = (value: any, flag: symbol, optionalFields?: ObjPath): string => {
+    return cryptoMd5(objectToString(value, flag, optionalFields))
+  }
+
+  const hashHook: HashScannerCrawlHook = ({ key, value, rules, state }) => {
     if (typeof key === 'symbol') {
-      return { done: true }
+      return { done: true, state: { ...state, fieldsForOptionalHash: [] } }
     }
     if (!rules) {
+      state.fieldsForOptionalHash.push(key)
       return { done: true }
     }
-    let ignoreKey = true
-    switch (rules.hashStrategy) {
-      case CURRENT_DATA_LEVEL: {
-        ignoreKey = state.dataLevel > 0
-        break
-      }
-      case SECOND_DATA_LEVELS: {
-        ignoreKey = state.dataLevel > 1
-        break
-      }
-    }
+    const ignoreKey = rules.excludeFromSemanticHash
+    ignoreKey && state.fieldsForOptionalHash.push(key)
     if (!isObject(value)) {
-      return { done: ignoreKey, value }
+      return { done: true, value }
     }
-    if (cycleGuard.has(value)) {
-      return {
-        done: ignoreKey,
-        value: rules.newDataLayer ? value : undefined,
-        state: {
-          ...state,
-          dataLevel: key !== undefined && rules.newDataLayer ? state.dataLevel + 1 : state.dataLevel,
-        },
-      }
+    const nestedFieldsForOptionalHash = optionalFieldStates.get(value) || []
+    if (!optionalFieldStates.has(value)) {
+      optionalFieldStates.set(value, nestedFieldsForOptionalHash)
     }
-    cycleGuard.add(value)
+
+    const done = !(!cycleGuard.has(value) && cycleGuard.add(value))
     return {
-      done: ignoreKey,
+      done,
       value,
-      state: {
-        ...state,
-        dataLevel: key !== undefined && rules.newDataLayer ? state.dataLevel + 1 : state.dataLevel,
-      },
-    }
-  }
-  return creatorHook
-}
-
-const createHashScannerHook: (options: HashOptions) => HashScannerCrawlHook = (options) => {
-  const flag = options.hashFlag ?? Symbol('should-never-happen')
-  const cycleGuard: Set<unknown> = new Set()
-  const hashHook: HashScannerCrawlHook = ({ key, value, rules }) => {
-    if (typeof key === 'symbol') {
-      return { done: true }
-    }
-
-    if (!isObject(value)) {
-      return { done: true }
-    }
+      state: { ...state, fieldsForOptionalHash: nestedFieldsForOptionalHash },
+      exitHook: () => {
+        if (hashProperty) {
+          value[hashProperty] = generateHash(value, hashProperty)
+        }
 
-    if (cycleGuard.has(value)) {
-      return { done: true }
-    }
-    cycleGuard.add(value)
-    return {
-      value, exitHook: rules?.hashOwner ? () => {
-        let hash: string | undefined = undefined
-        value[flag] = () => {
-          if (!hash) {
-            hash = calculateHash(createHashObject(value, rules))
-          }
-          return hash
+        // even if the optionalFields are empty, it is necessary to calculate the hash again,
+        // since its children may have optional fields and their hash will be different.
+        if (semanticHashProperty) {
+          value[semanticHashProperty] = generateHash(value, semanticHashProperty, nestedFieldsForOptionalHash)
         }
-      } : undefined,
+
+        value[optionalFieldFlag] = nestedFieldsForOptionalHash
+      },
     }
   }
 
@@ -104,39 +63,33 @@ const createHashScannerHook: (options: HashOptions) => HashScannerCrawlHook = (o
 
 type HashScannerCrawlHook = SyncCrawlHook<HashScannerCrawlState, NormalizationRule>
 
-export interface HashScannerCrawlState {}
-
-type HashObjectCreatorCrawlHook = SyncCloneHook<HashObjectCreatorState, NormalizationRule>
-
-export interface HashObjectCreatorState {
-  dataLevel: number
+export interface HashScannerCrawlState {
+  fieldsForOptionalHash: ObjPath
 }
 
-
 export const hash = (value: unknown, options?: HashOptions) => {
   const internalOptions = {
     ...options,
   } satisfies InternalHashOptions
-  const flag = options?.hashFlag
-  if (!flag) {
+  const semanticHashProperty = options?.semanticHashProperty
+  const hashProperty = options?.hashProperty
+  if (!semanticHashProperty && !hashProperty) {
     return value
   }
   const spec = resolveSpec(value)
-  if (spec.type === SPEC_TYPE_GRAPH_API){
-    return value //cause not implemented
-  }
   syncCrawl<HashScannerCrawlState, NormalizationRule>(
     value,
-    [createHashScannerHook(internalOptions)],
-    { rules: RULES[spec.type] },
+    [createHashObjectCreatorHook(internalOptions)],
+    { state: { fieldsForOptionalHash: [] }, rules: RULES[spec.type] },
   )
 
   return value
 }
 
 export const deHash = (value: unknown, options?: HashOptions) => {
-  const flag = options?.hashFlag
-  if (!flag) {
+  const semanticHashProperty = options?.semanticHashProperty
+  const hashProperty = options?.hashProperty
+  if (!semanticHashProperty && !hashProperty) {
     return value
   }
   const cycleGuard: Set<unknown> = new Set()
@@ -148,7 +101,10 @@ export const deHash = (value: unknown, options?: HashOptions) => {
       return { done: true }
     }
     cycleGuard.add(value)
-    flag in value && delete value[flag]
+    if (semanticHashProperty && semanticHashProperty in value) {delete value[semanticHashProperty]}
+    if (hashProperty && hashProperty in value) { delete value[hashProperty] }
+    //todo del after tests
+    if (optionalFieldFlag in value) { delete value[optionalFieldFlag] }
     return { value }
   })
   return value

src/normalize.ts

@@ -26,14 +26,14 @@ export const normalize = (value: unknown, options: NormalizeOptions = {}) => {
   if (optionsWithDefaults.unify) { spec = unify(spec, optionsWithDefaults) }
   if (optionsWithDefaults.mergeAllOf && !optionsWithDefaults.unify && !optionsWithDefaults.allowNotValidSyntheticChanges) { spec = cleanUpSynthetic(spec, optionsWithDefaults) }
   if (optionsWithDefaults.removeOasExtensions) { spec = removeOasExtensions(spec, optionsWithDefaults) }
-  if (optionsWithDefaults.hashFlag) { spec = hash(spec, optionsWithDefaults) }
+  if (optionsWithDefaults.semanticHashProperty || optionsWithDefaults.hashProperty) { spec = hash(spec, optionsWithDefaults) }
   return spec
 }
 
 export const denormalize = (value: unknown, options: DenormalizeOptions = {}) => {
   const optionsWithDefaults = createOptionsWithDefaults(options)
   let spec = value
-  if (optionsWithDefaults.hashFlag) { spec = deHash(spec, optionsWithDefaults) }
+  if (optionsWithDefaults.semanticHashProperty || optionsWithDefaults.hashProperty) { spec = deHash(spec, optionsWithDefaults) }
   if (optionsWithDefaults.mergeAllOf && !optionsWithDefaults.unify && !optionsWithDefaults.allowNotValidSyntheticChanges) { spec = deCleanUpSynthetic(spec, optionsWithDefaults) }
   if (optionsWithDefaults.unify) { spec = deUnify(spec, optionsWithDefaults) }
   //if in future we found way to denormalize following operation it should be in this order