feat: implement documents matching based on operations

Author: CountRedClaw

src/apitypes/graphql/graphql.changes.ts

@@ -19,16 +19,16 @@ import { isEmpty, removeComponents, removeFirstSlash, slugify, takeIf } from '..
 import {
   apiDiff,
   COMPARE_MODE_OPERATION,
-  DEFAULT_DIFFS_AGGREGATED_META_KEY,
   Diff,
   DIFF_META_KEY,
+  DIFFS_AGGREGATED_META_KEY,
 } from '@netcracker/qubership-apihub-api-diff'
 import { NORMALIZE_OPTIONS, ORIGINS_SYMBOL } from '../../consts'
 import { GraphApiOperation, GraphApiSchema } from '@netcracker/qubership-apihub-graphapi'
 import { buildSchema } from 'graphql/utilities'
 import { buildGraphQLDocument } from './graphql.document'
 import {
-  CompareContext,
+  CompareOperationsPairContext,
   FILE_KIND,
   NormalizedOperationId,
   OperationChanges,
@@ -44,34 +44,44 @@ import { createOperationChange, getOperationTags } from '../../components'
 export const compareDocuments = async (apiType: OperationsApiType, operationsMap: Record<NormalizedOperationId, {
   previous?: ResolvedOperation
   current?: ResolvedOperation
-}>, prevFile: File, currFile: File, currDoc: ResolvedVersionDocument, prevDoc: ResolvedVersionDocument, ctx: CompareContext): Promise<{
+}>, prevDoc: ResolvedVersionDocument | undefined, currDoc: ResolvedVersionDocument | undefined, ctx: CompareOperationsPairContext): Promise<{
   operationChanges: OperationChanges[]
   tags: string[]
 }> => {
-  const prevDocSchema = buildSchema(await prevFile.text(), { noLocation: true })
-  const currDocSchema = buildSchema(await currFile.text(), { noLocation: true })
+  const { rawDocumentResolver, previousVersion, currentVersion, previousPackageId, currentPackageId } = ctx
+  const prevFile = prevDoc && await rawDocumentResolver(previousVersion, previousPackageId, prevDoc.slug)
+  const currFile = currDoc && await rawDocumentResolver(currentVersion, currentPackageId, currDoc.slug)
+  const prevDocSchema = prevFile && buildSchema(await prevFile.text(), { noLocation: true })
+  const currDocSchema = currFile && buildSchema(await currFile.text(), { noLocation: true })
 
-  const prevDocData = (await buildGraphQLDocument({
+  let prevDocData = prevDocSchema && (await buildGraphQLDocument({
     ...prevDoc,
     source: prevFile,
     kind: FILE_KIND.TEXT,
     data: prevDocSchema,
   }, prevDoc)).data
-  const currDocData = (await buildGraphQLDocument({
+  let currDocData = currDocSchema && (await buildGraphQLDocument({
     ...currDoc,
     source: currFile,
     kind: FILE_KIND.TEXT,
     data: currDocSchema,
   }, currDoc)).data
 
+  if (!prevDocData && currDocData) {
+    prevDocData = getCopyWithEmptyOperations(currDocData)
+  }
+  if (prevDocData && !currDocData) {
+    currDocData = getCopyWithEmptyOperations(prevDocData)
+  }
+
   const { merged, diffs } = apiDiff(
     prevDocData,
     currDocData,
     {
       ...NORMALIZE_OPTIONS,
       metaKey: DIFF_META_KEY,
       originsFlag: ORIGINS_SYMBOL,
-      diffsAggregatedFlag: DEFAULT_DIFFS_AGGREGATED_META_KEY,
+      diffsAggregatedFlag: DIFFS_AGGREGATED_META_KEY,
       // mode: COMPARE_MODE_OPERATION,
       normalizedResult: true,
     },
@@ -83,7 +93,7 @@ export const compareDocuments = async (apiType: OperationsApiType, operationsMap
 
   let operationDiffs: Diff[] = []
 
-  const { currentGroup, previousGroup } = ctx.config
+  const { currentGroup, previousGroup } = ctx
   const currGroupSlug = slugify(removeFirstSlash(currentGroup || ''))
   const prevGroupSlug = slugify(removeFirstSlash(previousGroup || ''))
 
@@ -100,7 +110,7 @@ export const compareDocuments = async (apiType: OperationsApiType, operationsMap
 
       const { current, previous } = operationsMap[operationId] ?? {}
       if (current && previous) {
-        operationDiffs = [...(methodData as WithAggregatedDiffs<GraphApiOperation>)[DEFAULT_DIFFS_AGGREGATED_META_KEY]]
+        operationDiffs = [...(methodData as WithAggregatedDiffs<GraphApiOperation>)[DIFFS_AGGREGATED_META_KEY]]
       } else if (current || previous) {
         for (const type of GRAPHQL_TYPE_KEYS) {
           const operationsByType = (merged[type] as WithDiffMetaRecord<Record<string, GraphApiOperation>>)?.[DIFF_META_KEY]

src/apitypes/graphql/graphql.operation.ts

@@ -94,6 +94,7 @@ export const buildGraphQLOperation = (
 
   return {
     operationId,
+    documentId: document.slug,
     dataHash: 'dataHash is to be removed',
     apiType: GRAPHQL_API_TYPE,
     apiKind: rawToApiKind(apiKind),

src/apitypes/rest/rest.changes.ts

@@ -29,16 +29,15 @@ import {
   apiDiff,
   breaking,
   COMPARE_MODE_OPERATION,
-  DEFAULT_DIFFS_AGGREGATED_META_KEY,
   Diff,
   DIFF_META_KEY,
   DiffAction,
+  DIFFS_AGGREGATED_META_KEY,
   risky,
 } from '@netcracker/qubership-apihub-api-diff'
 import { MESSAGE_SEVERITY, NORMALIZE_OPTIONS, ORIGINS_SYMBOL } from '../../consts'
 import {
   BREAKING_CHANGE_TYPE,
-  CompareContext,
   CompareOperationsPairContext,
   NormalizedOperationId,
   OperationChanges,
@@ -66,12 +65,22 @@ import { createOperationChange, getOperationTags, takeSubstringIf } from '../../
 export const compareDocuments = async (apiType: OperationsApiType, operationsMap: Record<NormalizedOperationId, {
   previous?: ResolvedOperation
   current?: ResolvedOperation
-}>, prevFile: File, currFile: File, currDoc: ResolvedVersionDocument, prevDoc: ResolvedVersionDocument, ctx: CompareContext): Promise<{
+}>, prevDoc: ResolvedVersionDocument | undefined, currDoc: ResolvedVersionDocument | undefined, ctx: CompareOperationsPairContext): Promise<{
   operationChanges: OperationChanges[]
   tags: string[]
 }> => {
-  const prevDocData = JSON.parse(await prevFile.text())
-  const currDocData = JSON.parse(await currFile.text())
+  const { rawDocumentResolver, previousVersion, currentVersion, previousPackageId, currentPackageId } = ctx
+  const prevFile = prevDoc && await rawDocumentResolver(previousVersion, previousPackageId, prevDoc.slug)
+  const currFile = currDoc && await rawDocumentResolver(currentVersion, currentPackageId, currDoc.slug)
+  let prevDocData = prevFile && JSON.parse(await prevFile.text())
+  let currDocData = currFile && JSON.parse(await currFile.text())
+
+  if (!prevDocData && currDocData) {
+    prevDocData = createCopyWithEmptyPath(currDocData)
+  }
+  if (prevDocData && !currDocData) {
+    currDocData = createCopyWithEmptyPath(prevDocData)
+  }
 
   const { merged, diffs } = apiDiff(
     prevDocData,
@@ -80,7 +89,7 @@ export const compareDocuments = async (apiType: OperationsApiType, operationsMap
       ...NORMALIZE_OPTIONS,
       metaKey: DIFF_META_KEY,
       originsFlag: ORIGINS_SYMBOL,
-      diffsAggregatedFlag: DEFAULT_DIFFS_AGGREGATED_META_KEY,
+      diffsAggregatedFlag: DIFFS_AGGREGATED_META_KEY,
       // mode: COMPARE_MODE_OPERATION,
       normalizedResult: true,
     },
@@ -128,17 +137,22 @@ export const compareDocuments = async (apiType: OperationsApiType, operationsMap
       let operationDiffs: Diff[] = []
       if (current && previous) {
         operationDiffs = [
-          ...(methodData as WithAggregatedDiffs<OpenAPIV3.OperationObject>)[DEFAULT_DIFFS_AGGREGATED_META_KEY],
+          ...(methodData as WithAggregatedDiffs<OpenAPIV3.OperationObject>)[DIFFS_AGGREGATED_META_KEY],
           // todo what about security? add test
           ...extractServersDiffs(merged),
         ]
 
         const pathParamRenameDiff = (merged.paths as WithDiffMetaRecord<OpenAPIV3.PathsObject>)[DIFF_META_KEY]?.[path]
         pathParamRenameDiff && operationDiffs.push(pathParamRenameDiff)
       } else if (current || previous) {
-        const operationDiff = (merged.paths as WithDiffMetaRecord<OpenAPIV3.PathsObject>)[DIFF_META_KEY]?.[path]
+        const operationDiff = (merged.paths[path] as WithDiffMetaRecord<OpenAPIV3.PathsObject>)[DIFF_META_KEY]?.[inferredMethod]
         if (!operationDiff) {
-          throw new Error('should not happen')
+          // ignore removed and added operations, they'll be handled in a separate docs comparison
+          continue
+        }
+        const deprecatedInVersionsCount = previousVersionDeprecations?.operations.find((operation) => operation.operationId === operationId)?.deprecatedInPreviousVersions?.length ?? 0
+        if (isOperationRemove(operationDiff) && deprecatedInVersionsCount > 1) {
+          operationDiff.type = risky
         }
         operationDiffs.push(operationDiff)
       }
@@ -162,11 +176,11 @@ export const compareRestOperationsData = async (current: VersionRestOperation |
   let previousOperation = removeComponents(previous?.data)
   let currentOperation = removeComponents(current?.data)
   if (!previousOperation && currentOperation) {
-    previousOperation = getCopyWithEmptyPath(currentOperation as RestOperationData)
+    previousOperation = createCopyWithEmptyPath(currentOperation as RestOperationData)
   }
 
   if (previousOperation && !currentOperation) {
-    currentOperation = getCopyWithEmptyPath(previousOperation as RestOperationData)
+    currentOperation = createCopyWithEmptyPath(previousOperation as RestOperationData)
   }
 
   const diffResult = apiDiff(
@@ -201,7 +215,6 @@ async function reclassifyBreakingChanges(
   if (!previosVersionDeprecations) {
     return
   }
-  previosVersionDeprecations.operations[0]
 
   const previousOperation = previosVersionDeprecations.operations[0]
 
@@ -272,11 +285,14 @@ async function reclassifyBreakingChanges(
   }
 }
 
-function getCopyWithEmptyPath(template: RestOperationData): RestOperationData {
+export function createCopyWithEmptyPath(template: RestOperationData): RestOperationData {
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   const { paths, ...rest } = template
+
   return {
-    paths: {},
+    paths: {
+      ...Object.fromEntries(Object.keys(template.paths).map(key => [key, {}])),
+    },
     ...rest,
   }
 }

src/apitypes/rest/rest.operation.ts

@@ -155,6 +155,7 @@ export const buildRestOperation = (
 
   return {
     operationId,
+    documentId: document.slug,
     dataHash: 'dataHash is to be removed',
     apiType: REST_API_TYPE,
     apiKind: rawToApiKind(apiKind),

src/builder.ts

@@ -57,6 +57,8 @@ import {
   BUILD_TYPE,
   DEFAULT_BATCH_SIZE,
   DEFAULT_VALIDATION_RULES_SEVERITY_CONFIG,
+  EXPORT_BUILD_TYPES,
+  ExportBuildType,
   MESSAGE_SEVERITY,
   SUPPORTED_FILE_FORMATS,
   VERSION_STATUS,
@@ -136,13 +138,20 @@ export class PackageVersionBuilder implements IPackageVersionBuilder {
 
   // todo rename
   async createNodeVersionPackage(): Promise<{ packageVersion: any; exportFileName?: string }> {
-    return {packageVersion: await createVersionPackage(this.buildResult, new AdmZipTool(), this.builderContext(this.config)), exportFileName: this.buildResult.exportFileName}
+    return {
+      packageVersion: await createVersionPackage(this.buildResult, new AdmZipTool(), this.builderContext(this.config)),
+      exportFileName: this.buildResult.exportFileName,
+    }
   }
 
   get operationList(): ApiOperation[] {
     return getOperationsList(this.buildResult)
   }
 
+  get documentList(): VersionDocument[] {
+    return [...this.buildResult.documents.values()]
+  }
+
   get packageConfig(): PackageConfig {
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     const { files, ...config } = this.config
@@ -311,6 +320,19 @@ export class PackageVersionBuilder implements IPackageVersionBuilder {
     packageId: PackageId,
     slug: string,
   ): Promise<File> {
+    if (this.canBeResolvedLocally(version, packageId)) {
+      const document = this.documentList.find((document) => document.slug === slug)
+      if (!document) {
+        throw new Error(`Raw document ${slug} is missing in local cache`)
+      }
+      const apiBuilder = this.apiBuilders.find(apiBuilder => apiBuilder.types.includes(document?.type))
+      if (!apiBuilder) {
+        // todo
+        throw new Error(`Cannot find apiBuilder for type ${document?.type}`)
+      }
+      return new File([apiBuilder.dumpDocument(document)], document.filename)
+    }
+
     if (!this.params.resolvers.rawDocumentResolver) {
       throw new Error('rawDocumentResolver is not provided')
     }
@@ -453,6 +475,13 @@ export class PackageVersionBuilder implements IPackageVersionBuilder {
     apiType?: OperationsApiType,
   ): Promise<ResolvedVersionDocuments | null> {
     packageId = packageId ?? this.config.packageId
+    // this is a case when the version has been built just now, and there's nothing to fetch yet, so
+    // the only way possible is to get the docs from buildResult, but the referenced packages map will be empty
+    if (this.canBeResolvedLocally(version, packageId)) {
+      const apiBuilder = this.apiBuilders.find(apiBuilder => apiBuilder.apiType === apiType)
+      const currentApiTypeDocuments = this.documentList.filter(({ type }) => apiBuilder?.types.includes(type))
+      return { documents: currentApiTypeDocuments, packages: {} }
+    }
 
     const { versionDocumentsResolver } = this.params.resolvers
     if (!versionDocumentsResolver) {
@@ -509,6 +538,7 @@ export class PackageVersionBuilder implements IPackageVersionBuilder {
   private canBeResolvedLocally(version: string, packageId: string | undefined): boolean {
     return this.config.buildType !== BUILD_TYPE.CHANGELOG &&
       this.config.buildType !== BUILD_TYPE.PREFIX_GROUPS_CHANGELOG &&
+      !EXPORT_BUILD_TYPES.includes(this.config.buildType as ExportBuildType) &&
       version === this.config.version &&
       packageId === this.config.packageId
   }