refactor: compare documents instead of operations

Author: CountRedClaw

src/apitypes/graphql/graphql.changes.ts

@@ -15,11 +15,116 @@
  */
 
 import { VersionGraphQLOperation } from './graphql.types'
-import { removeComponents, takeIf } from '../../utils'
-import { apiDiff, COMPARE_MODE_OPERATION, Diff } from '@netcracker/qubership-apihub-api-diff'
-import { NORMALIZE_OPTIONS } from '../../consts'
-import { GraphApiSchema } from '@netcracker/qubership-apihub-graphapi'
+import { isEmpty, removeComponents, removeFirstSlash, slugify, takeIf } from '../../utils'
+import {
+  apiDiff,
+  COMPARE_MODE_OPERATION,
+  DEFAULT_DIFFS_AGGREGATED_META_KEY,
+  Diff,
+  DIFF_META_KEY,
+} from '@netcracker/qubership-apihub-api-diff'
+import { NORMALIZE_OPTIONS, ORIGINS_SYMBOL } from '../../consts'
+import { GraphApiOperation, GraphApiSchema } from '@netcracker/qubership-apihub-graphapi'
+import { buildSchema } from 'graphql/utilities'
+import { buildGraphQLDocument } from './graphql.document'
+import {
+  CompareContext,
+  FILE_KIND,
+  NormalizedOperationId,
+  OperationChanges,
+  OperationsApiType,
+  ResolvedOperation,
+  ResolvedVersionDocument,
+  WithAggregatedDiffs,
+  WithDiffMetaRecord,
+} from '../../types'
+import { GRAPHQL_TYPE, GRAPHQL_TYPE_KEYS } from './graphql.consts'
+import { createOperationChange, getOperationTags } from '../../components'
 
+export const compareDocuments = async (apiType: OperationsApiType, operationsMap: Record<NormalizedOperationId, {
+  previous?: ResolvedOperation
+  current?: ResolvedOperation
+}>, prevFile: File, currFile: File, currDoc: ResolvedVersionDocument, prevDoc: ResolvedVersionDocument, ctx: CompareContext): Promise<{
+  operationChanges: OperationChanges[]
+  tags: string[]
+}> => {
+  const prevDocSchema = buildSchema(await prevFile.text(), { noLocation: true })
+  const currDocSchema = buildSchema(await currFile.text(), { noLocation: true })
+
+  const prevDocData = (await buildGraphQLDocument({
+    ...prevDoc,
+    source: prevFile,
+    kind: FILE_KIND.TEXT,
+    data: prevDocSchema,
+  }, prevDoc)).data
+  const currDocData = (await buildGraphQLDocument({
+    ...currDoc,
+    source: currFile,
+    kind: FILE_KIND.TEXT,
+    data: currDocSchema,
+  }, currDoc)).data
+
+  const { merged, diffs } = apiDiff(
+    prevDocData,
+    currDocData,
+    {
+      ...NORMALIZE_OPTIONS,
+      metaKey: DIFF_META_KEY,
+      originsFlag: ORIGINS_SYMBOL,
+      diffsAggregatedFlag: DEFAULT_DIFFS_AGGREGATED_META_KEY,
+      // mode: COMPARE_MODE_OPERATION,
+      normalizedResult: true,
+    },
+  ) as {merged: GraphApiSchema; diffs: Diff[]}
+
+  if (isEmpty(diffs)) {
+    return { operationChanges: [], tags: [] }
+  }
+
+  let operationDiffs: Diff[] = []
+
+  const { currentGroup, previousGroup } = ctx.config
+  const currGroupSlug = slugify(removeFirstSlash(currentGroup || ''))
+  const prevGroupSlug = slugify(removeFirstSlash(previousGroup || ''))
+
+  const tags = new Set<string>()
+  const changedOperations: OperationChanges[] = []
+
+  for (const type of GRAPHQL_TYPE_KEYS) {
+    const operationsByType = merged[type]
+    if (!operationsByType) { continue }
+
+    for (const operationKey of Object.keys(operationsByType)) {
+      const operationId = slugify(`${GRAPHQL_TYPE[type]}-${operationKey}`)
+      const methodData = operationsByType[operationKey]
+
+      const { current, previous } = operationsMap[operationId] ?? {}
+      if (current && previous) {
+        operationDiffs = [...(methodData as WithAggregatedDiffs<GraphApiOperation>)[DEFAULT_DIFFS_AGGREGATED_META_KEY]]
+      } else if (current || previous) {
+        for (const type of GRAPHQL_TYPE_KEYS) {
+          const operationsByType = (merged[type] as WithDiffMetaRecord<Record<string, GraphApiOperation>>)?.[DIFF_META_KEY]
+          if (!operationsByType) { continue }
+          operationDiffs.push(...Object.values(operationsByType))
+        }
+        if (isEmpty(operationDiffs)) {
+          throw new Error('should not happen')
+        }
+      }
+
+      if (isEmpty(operationDiffs)) {
+        continue
+      }
+
+      changedOperations.push(createOperationChange(apiType, operationDiffs, previous, current, currGroupSlug, prevGroupSlug, currentGroup, previousGroup))
+      getOperationTags(current ?? previous).forEach(tag => tags.add(tag))
+    }
+  }
+
+  return { operationChanges: changedOperations, tags: [...tags.values()] }
+}
+
+/** @deprecated */
 export const graphqlOperationsCompare = async (current: VersionGraphQLOperation | undefined, previous: VersionGraphQLOperation | undefined): Promise<Diff[]> => {
   let previousOperation = removeComponents(previous?.data)
   let currentOperation = removeComponents(current?.data)

src/apitypes/graphql/graphql.consts.ts

@@ -13,6 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import { ResolvedVersionDocument, ZippableDocument } from '../../types'
+import { GraphQLDocumentType } from './graphql.types'
 
 export const GRAPHQL_API_TYPE = 'graphql' as const
 
@@ -43,3 +45,7 @@ export const GRAPHQL_TYPE = {
   'mutations': 'mutation',
   'subscriptions': 'subscription',
 } as const
+
+export function isGraphqlDocument(document: ZippableDocument | ResolvedVersionDocument): boolean {
+  return Object.values(GRAPHQL_DOCUMENT_TYPE).includes(document.type as GraphQLDocumentType)
+}

src/apitypes/graphql/graphql.document.ts

@@ -14,13 +14,18 @@
  * limitations under the License.
  */
 
-import { buildFromIntrospection, buildFromSchema, GraphApiSchema, printGraphApi } from '@netcracker/qubership-apihub-graphapi'
+import {
+  buildFromIntrospection,
+  buildFromSchema,
+  GraphApiSchema,
+  printGraphApi,
+} from '@netcracker/qubership-apihub-graphapi'
 import type { GraphQLSchema, IntrospectionQuery } from 'graphql'
 
-import type { DocumentBuilder, DocumentDumper } from '../../types'
+import { BuildConfigFile, DocumentDumper, TextFile, VersionDocument } from '../../types'
 import { GRAPHQL_DOCUMENT_TYPE } from './graphql.consts'
 
-export const buildGraphQLDocument: DocumentBuilder<GraphApiSchema> = async (parsedFile, file) => {
+export const buildGraphQLDocument: (parsedFile: TextFile, file: BuildConfigFile) => Promise<VersionDocument<GraphApiSchema>> = async (parsedFile, file) => {
   let graphapi: GraphApiSchema
   if (parsedFile.type === GRAPHQL_DOCUMENT_TYPE.INTROSPECTION) {
     const introspection = (parsedFile?.data && '__schema' in parsedFile.data ? parsedFile?.data : parsedFile.data?.data) as IntrospectionQuery

src/apitypes/graphql/graphql.operation.ts

@@ -88,18 +88,13 @@ export const buildGraphQLOperation = (
 
   const apiKind = document.apiKind || API_KIND.BWC
 
-
-  const dataHash = syncDebugPerformance('[ModelsAndOperationHashing]', () => {
+  syncDebugPerformance('[ModelsAndOperationHashing]', () => {
     calculateSpecRefs(document.data, singleOperationRefsOnlySpec, singleOperationSpec)
-    const dataHash = calculateObjectHash(singleOperationSpec)
-    return dataHash
   }, debugCtx)
 
-
-
   return {
     operationId,
-    dataHash,
+    dataHash: 'dataHash is to be removed',
     apiType: GRAPHQL_API_TYPE,
     apiKind: rawToApiKind(apiKind),
     deprecated: !!singleOperationEffectiveSpec[type]?.[method]?.directives?.deprecated,

src/apitypes/graphql/index.ts

@@ -21,7 +21,7 @@ import { buildGraphQLOperations } from './graphql.operations'
 import { GRAPHQL_API_TYPE, GRAPHQL_DOCUMENT_TYPE } from './graphql.consts'
 import { parseGraphQLFile } from './graphql.parser'
 import { ApiBuilder } from '../../types'
-import { graphqlOperationsCompare } from './graphql.changes'
+import { compareDocuments, graphqlOperationsCompare } from './graphql.changes'
 
 export * from './graphql.consts'
 
@@ -33,4 +33,5 @@ export const graphqlApiBuilder: ApiBuilder<GraphApiSchema> = {
   buildOperations: buildGraphQLOperations,
   dumpDocument: dumpGraphQLDocument,
   compareOperationsData: graphqlOperationsCompare,
+  compareDocuments: compareDocuments,
 }

src/apitypes/rest/index.ts

@@ -18,7 +18,7 @@ import { OpenAPIV3 } from 'openapi-types'
 
 import { buildRestDocument, createRestExportDocument, dumpRestDocument } from './rest.document'
 import { REST_API_TYPE, REST_DOCUMENT_TYPE } from './rest.consts'
-import { compareRestOperationsData } from './rest.changes'
+import { compareDocuments, compareRestOperationsData } from './rest.changes'
 import { buildRestOperations, createNormalizedOperationId } from './rest.operations'
 import { parseRestFile } from './rest.parser'
 
@@ -34,6 +34,7 @@ export const restApiBuilder: ApiBuilder<OpenAPIV3.Document> = {
   buildOperations: buildRestOperations,
   dumpDocument: dumpRestDocument,
   compareOperationsData: compareRestOperationsData,
+  compareDocuments: compareDocuments,
   createNormalizedOperationId: createNormalizedOperationId,
   createExportDocument: createRestExportDocument,
 }

src/apitypes/rest/rest.changes.ts

@@ -15,28 +15,148 @@
  */
 
 import { RestOperationData, VersionRestOperation } from './rest.types'
-import { areDeprecatedOriginsNotEmpty, isOperationRemove, removeComponents } from '../../utils'
+import {
+  areDeprecatedOriginsNotEmpty,
+  IGNORE_PATH_PARAM_UNIFIED_PLACEHOLDER,
+  isEmpty,
+  isOperationRemove,
+  normalizePath,
+  removeComponents,
+  removeFirstSlash,
+  slugify,
+} from '../../utils'
 import {
   apiDiff,
   breaking,
   COMPARE_MODE_OPERATION,
+  DEFAULT_DIFFS_AGGREGATED_META_KEY,
   Diff,
+  DIFF_META_KEY,
   DiffAction,
   risky,
 } from '@netcracker/qubership-apihub-api-diff'
 import { MESSAGE_SEVERITY, NORMALIZE_OPTIONS, ORIGINS_SYMBOL } from '../../consts'
 import {
   BREAKING_CHANGE_TYPE,
+  CompareContext,
   CompareOperationsPairContext,
+  NormalizedOperationId,
+  OperationChanges,
+  OperationsApiType,
+  ResolvedOperation,
+  ResolvedVersionDocument,
   RISKY_CHANGE_TYPE,
+  WithAggregatedDiffs,
+  WithDiffMetaRecord,
 } from '../../types'
 import { isObject } from '@netcracker/qubership-apihub-json-crawl'
 import { areDeclarationPathsEqual } from '../../utils/path'
-import { JSON_SCHEMA_PROPERTY_DEPRECATED, pathItemToFullPath, resolveOrigins } from '@netcracker/qubership-apihub-api-unifier'
+import {
+  JSON_SCHEMA_PROPERTY_DEPRECATED,
+  pathItemToFullPath,
+  resolveOrigins,
+} from '@netcracker/qubership-apihub-api-unifier'
 import { findRequiredRemovedProperties } from './rest.required'
 import { calculateObjectHash } from '../../utils/hashes'
 import { REST_API_TYPE } from './rest.consts'
+import { OpenAPIV3 } from 'openapi-types'
+import { extractServersDiffs, getOperationBasePath } from './rest.utils'
+import { createOperationChange, getOperationTags, takeSubstringIf } from '../../components'
+
+export const compareDocuments = async (apiType: OperationsApiType, operationsMap: Record<NormalizedOperationId, {
+  previous?: ResolvedOperation
+  current?: ResolvedOperation
+}>, prevFile: File, currFile: File, currDoc: ResolvedVersionDocument, prevDoc: ResolvedVersionDocument, ctx: CompareContext): Promise<{
+  operationChanges: OperationChanges[]
+  tags: string[]
+}> => {
+  const prevDocData = JSON.parse(await prevFile.text())
+  const currDocData = JSON.parse(await currFile.text())
+
+  const { merged, diffs } = apiDiff(
+    prevDocData,
+    currDocData,
+    {
+      ...NORMALIZE_OPTIONS,
+      metaKey: DIFF_META_KEY,
+      originsFlag: ORIGINS_SYMBOL,
+      diffsAggregatedFlag: DEFAULT_DIFFS_AGGREGATED_META_KEY,
+      // mode: COMPARE_MODE_OPERATION,
+      normalizedResult: true,
+    },
+  ) as { merged: OpenAPIV3.Document; diffs: Diff[] }
+
+  if (isEmpty(diffs)) {
+    return { operationChanges: [], tags: [] }
+  }
+
+  // todo reclassify
+  // const olnyBreaking = diffs.filter((diff) => diff.type === breaking)
+  // if (olnyBreaking.length > 0 && previous?.operationId) {
+  //   await reclassifyBreakingChanges(previous.operationId, diffResult.merged, olnyBreaking, ctx)
+  // }
+
+  const { currentGroup, previousGroup } = ctx.config
+  const currGroupSlug = slugify(removeFirstSlash(currentGroup || ''))
+  const prevGroupSlug = slugify(removeFirstSlash(previousGroup || ''))
+
+  const tags = new Set<string>()
+  const changedOperations: OperationChanges[] = []
+
+  for (const path of Object.keys((merged as OpenAPIV3.Document).paths)) {
+    const pathData = (merged as OpenAPIV3.Document).paths[path]
+    if (typeof pathData !== 'object' || !pathData) { continue }
+
+    for (const key of Object.keys(pathData)) {
+      const inferredMethod = key as OpenAPIV3.HttpMethods
+
+      // check if field is a valid openapi http method defined in OpenAPIV3.HttpMethods
+      if (!Object.values(OpenAPIV3.HttpMethods).includes(inferredMethod)) {
+        continue
+      }
+
+      const methodData = pathData[inferredMethod]
+      const basePath = getOperationBasePath(methodData?.servers || pathData?.servers || merged.servers || [])
+      const operationPath = basePath + path
+      const operationId = slugify(`${removeFirstSlash(operationPath)}-${inferredMethod}`)
+      const normalizedOperationId = slugify(`${normalizePath(basePath + path)}-${inferredMethod}`, [], IGNORE_PATH_PARAM_UNIFIED_PLACEHOLDER)
+      // todo what's with prevslug? which tests affected? which slug to slice prev or curr?
+      const qwe = takeSubstringIf(!!currGroupSlug, normalizedOperationId, currGroupSlug.length)
+
+      const { current, previous } = operationsMap[qwe] ?? operationsMap[operationId] ?? {}
+
+      let operationDiffs: Diff[] = []
+      if (current && previous) {
+        operationDiffs = [
+          ...(methodData as WithAggregatedDiffs<OpenAPIV3.OperationObject>)[DEFAULT_DIFFS_AGGREGATED_META_KEY],
+          // todo what about security? add test
+          ...extractServersDiffs(merged),
+        ]
+
+        const pathParamRenameDiff = (merged.paths as WithDiffMetaRecord<OpenAPIV3.PathsObject>)[DIFF_META_KEY]?.[path]
+        pathParamRenameDiff && operationDiffs.push(pathParamRenameDiff)
+      } else if (current || previous) {
+        const operationDiff = (merged.paths as WithDiffMetaRecord<OpenAPIV3.PathsObject>)[DIFF_META_KEY]?.[path]
+        if (!operationDiff) {
+          throw new Error('should not happen')
+        }
+        operationDiffs.push(operationDiff)
+      }
+
+      if (isEmpty(operationDiffs)) {
+        continue
+      }
+
+      // todo operationDiffs can be [undefined] in 'type error must not appear during build'
+      changedOperations.push(createOperationChange(apiType, operationDiffs, previous, current, currGroupSlug, prevGroupSlug, currentGroup, previousGroup))
+      getOperationTags(current ?? previous).forEach(tag => tags.add(tag))
+    }
+  }
+
+  return { operationChanges: changedOperations, tags: [...tags.values()] }
+}
 
+/** @deprecated */
 export const compareRestOperationsData = async (current: VersionRestOperation | undefined, previous: VersionRestOperation | undefined, ctx: CompareOperationsPairContext): Promise<Diff[]> => {
 
   let previousOperation = removeComponents(previous?.data)

src/apitypes/rest/rest.operation.ts

@@ -133,7 +133,7 @@ export const buildRestOperation = (
 
   const models: Record<string, string> = {}
   const apiKind = effectiveOperationObject[REST_KIND_KEY] || document.apiKind || API_KIND.BWC
-  const [specWithSingleOperation, dataHash] = syncDebugPerformance('[ModelsAndOperationHashing]', () => {
+  const [specWithSingleOperation] = syncDebugPerformance('[ModelsAndOperationHashing]', () => {
     const specWithSingleOperation = createSingleOperationSpec(
       document.data,
       path,
@@ -144,8 +144,7 @@ export const buildRestOperation = (
       components?.securitySchemes,
     )
     calculateSpecRefs(document.data, refsOnlySingleOperationSpec, specWithSingleOperation, models, componentsHashMap)
-    const dataHash = calculateObjectHash(specWithSingleOperation)
-    return [specWithSingleOperation, dataHash]
+    return [specWithSingleOperation]
   }, debugCtx)
 
   const deprecatedOperationItem = deprecatedItems.find(isDeprecatedOperationItem)
@@ -156,7 +155,7 @@ export const buildRestOperation = (
 
   return {
     operationId,
-    dataHash,
+    dataHash: 'dataHash is to be removed',
     apiType: REST_API_TYPE,
     apiKind: rawToApiKind(apiKind),
     deprecated: !!effectiveOperationObject.deprecated,