feat: performance optimization for changelog build type

.github/super-linter.env

@@ -41,5 +41,8 @@ VALIDATE_TYPESCRIPT_STANDARD=false
 VALIDATE_CHECKOV=false
 VALIDATE_OPENAPI=false
 
+# Skip GraphQL prettier, since we need GraphQL specs as is for testing purposes
+VALIDATE_GRAPHQL_PRETTIER=false
+
 # Skip checking of specially broken specifications
 FILTER_REGEX_EXCLUDE=test/projects/broken/.*

package.json

@@ -31,9 +31,9 @@
     "update-lock-file": "update-lock-file @netcracker"
   },
   "dependencies": {
-    "@netcracker/qubership-apihub-api-diff": "2.3.0",
-    "@netcracker/qubership-apihub-api-unifier": "2.4.0",
-    "@netcracker/qubership-apihub-graphapi": "1.0.8",
+    "@netcracker/qubership-apihub-api-diff": "feature-move-prefix-from-server-to-path",
+    "@netcracker/qubership-apihub-api-unifier": "feature-performance-optimization",
+    "@netcracker/qubership-apihub-graphapi": "feature-performance-optimization",
     "@netcracker/qubership-apihub-json-crawl": "1.0.4",
     "adm-zip": "0.5.10",
     "ajv": "^8.12.0",

src/apitypes/graphql/graphql.changes.ts

@@ -14,36 +14,113 @@
  * limitations under the License.
  */
 
-import { VersionGraphQLOperation } from './graphql.types'
-import { removeComponents, takeIf } from '../../utils'
-import { apiDiff, COMPARE_MODE_OPERATION, Diff } from '@netcracker/qubership-apihub-api-diff'
-import { NORMALIZE_OPTIONS } from '../../consts'
-import { GraphApiSchema } from '@netcracker/qubership-apihub-graphapi'
-
-export const graphqlOperationsCompare = async (current: VersionGraphQLOperation | undefined, previous: VersionGraphQLOperation | undefined): Promise<Diff[]> => {
-  let previousOperation = removeComponents(previous?.data)
-  let currentOperation = removeComponents(current?.data)
-  if (!previousOperation && currentOperation) {
-    previousOperation = getCopyWithEmptyOperations(currentOperation as GraphApiSchema)
-  }
+import { isEmpty, slugify, takeIf } from '../../utils'
+import { aggregateDiffsWithRollup, apiDiff, Diff, DIFF_META_KEY, DIFFS_AGGREGATED_META_KEY } from '@netcracker/qubership-apihub-api-diff'
+import { NORMALIZE_OPTIONS, ORIGINS_SYMBOL } from '../../consts'
+import { GraphApiOperation, GraphApiSchema } from '@netcracker/qubership-apihub-graphapi'
+import { buildSchema } from 'graphql/utilities'
+import { buildGraphQLDocument } from './graphql.document'
+import {
+  CompareOperationsPairContext,
+  FILE_KIND,
+  OperationChanges,
+  ResolvedVersionDocument,
+  WithAggregatedDiffs,
+  WithDiffMetaRecord,
+} from '../../types'
+import { GRAPHQL_TYPE, GRAPHQL_TYPE_KEYS } from './graphql.consts'
+import { createOperationChange, getOperationTags, OperationsMap } from '../../components'
+
+export const compareDocuments = async (
+  operationsMap: OperationsMap,
+  prevDoc: ResolvedVersionDocument | undefined,
+  currDoc: ResolvedVersionDocument | undefined,
+  ctx: CompareOperationsPairContext): Promise<{
+    operationChanges: OperationChanges[]
+    tags: Set<string>
+  }> => {
+  const { apiType, rawDocumentResolver, previousVersion, currentVersion, previousPackageId, currentPackageId } = ctx
+  const prevFile = prevDoc && await rawDocumentResolver(previousVersion, previousPackageId, prevDoc.slug)
+  const currFile = currDoc && await rawDocumentResolver(currentVersion, currentPackageId, currDoc.slug)
+  const prevDocSchema = prevFile && buildSchema(await prevFile.text(), { noLocation: true })
+  const currDocSchema = currFile && buildSchema(await currFile.text(), { noLocation: true })
+
+  let prevDocData = prevDocSchema && (await buildGraphQLDocument({
+    ...prevDoc,
+    source: prevFile,
+    kind: FILE_KIND.TEXT,
+    data: prevDocSchema,
+  }, prevDoc)).data
+  let currDocData = currDocSchema && (await buildGraphQLDocument({
+    ...currDoc,
+    source: currFile,
+    kind: FILE_KIND.TEXT,
+    data: currDocSchema,
+  }, currDoc)).data
 
-  if (previousOperation && !currentOperation) {
-    currentOperation = getCopyWithEmptyOperations(previousOperation as GraphApiSchema)
+  if (!prevDocData && currDocData) {
+    prevDocData = getCopyWithEmptyOperations(currDocData)
+  }
+  if (prevDocData && !currDocData) {
+    currDocData = getCopyWithEmptyOperations(prevDocData)
   }
 
-  //todo think about normalize options
-  const { diffs } = apiDiff(
-    previousOperation,
-    currentOperation,
+  const { merged, diffs } = apiDiff(
+    prevDocData,
+    currDocData,
     {
       ...NORMALIZE_OPTIONS,
-      mode: COMPARE_MODE_OPERATION,
+      metaKey: DIFF_META_KEY,
+      originsFlag: ORIGINS_SYMBOL,
       normalizedResult: true,
-      beforeSource: previous?.data,
-      afterSource: current?.data,
     },
-  )
-  return diffs
+  ) as { merged: GraphApiSchema; diffs: Diff[] }
+
+  if (isEmpty(diffs)) {
+    return { operationChanges: [], tags: new Set() }
+  }
+
+  aggregateDiffsWithRollup(merged, DIFF_META_KEY, DIFFS_AGGREGATED_META_KEY)
+
+  const { currentGroup, previousGroup } = ctx
+
+  const tags = new Set<string>()
+  const operationChanges: OperationChanges[] = []
+
+  for (const type of GRAPHQL_TYPE_KEYS) {
+    const operationsByType = merged[type]
+    if (!operationsByType) { continue }
+
+    for (const operationKey of Object.keys(operationsByType)) {
+      const operationId = slugify(`${GRAPHQL_TYPE[type]}-${operationKey}`)
+      const methodData = operationsByType[operationKey]
+
+      const { current, previous } = operationsMap[operationId] ?? {}
+      if (!current && !previous) {
+        throw new Error(`Can't find the ${operationId} operation from documents pair ${prevDoc?.fileId} and ${currDoc?.fileId}`)
+      }
+      const operationChanged = Boolean(current && previous)
+      const operationAddedOrRemoved = !operationChanged
+
+      let operationDiffs: Diff[] = []
+      if (operationChanged) {
+        operationDiffs = [...(methodData as WithAggregatedDiffs<GraphApiOperation>)[DIFFS_AGGREGATED_META_KEY] ?? []]
+      }
+      if (operationAddedOrRemoved) {
+        const operationAddedOrRemovedDiff = (merged[type] as WithDiffMetaRecord<Record<string, GraphApiOperation>>)[DIFF_META_KEY]?.[operationKey]
+        operationAddedOrRemovedDiff && operationDiffs.push(operationAddedOrRemovedDiff)
+      }
+
+      if (isEmpty(operationDiffs)) {
+        continue
+      }
+
+      operationChanges.push(createOperationChange(apiType, operationDiffs, previous, current, currentGroup, previousGroup))
+      getOperationTags(current ?? previous).forEach(tag => tags.add(tag))
+    }
+  }
+
+  return { operationChanges, tags }
 }
 
 function getCopyWithEmptyOperations(template: GraphApiSchema): GraphApiSchema {

src/apitypes/graphql/graphql.consts.ts

@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import { ResolvedVersionDocument, ZippableDocument } from '../../types'
 
 export const GRAPHQL_API_TYPE = 'graphql' as const
 
@@ -43,3 +44,7 @@ export const GRAPHQL_TYPE = {
   'mutations': 'mutation',
   'subscriptions': 'subscription',
 } as const
+
+export function isGraphqlDocument(document: ZippableDocument | ResolvedVersionDocument): boolean {
+  return Object.values(GRAPHQL_DOCUMENT_TYPE).some(type => document.type === type)
+}

src/apitypes/graphql/graphql.document.ts

@@ -14,13 +14,18 @@
  * limitations under the License.
  */
 
-import { buildFromIntrospection, buildFromSchema, GraphApiSchema, printGraphApi } from '@netcracker/qubership-apihub-graphapi'
+import {
+  buildFromIntrospection,
+  buildFromSchema,
+  GraphApiSchema,
+  printGraphApi,
+} from '@netcracker/qubership-apihub-graphapi'
 import type { GraphQLSchema, IntrospectionQuery } from 'graphql'
 
-import type { DocumentBuilder, DocumentDumper } from '../../types'
+import { BuildConfigFile, DocumentDumper, TextFile, VersionDocument } from '../../types'
 import { GRAPHQL_DOCUMENT_TYPE } from './graphql.consts'
 
-export const buildGraphQLDocument: DocumentBuilder<GraphApiSchema> = async (parsedFile, file) => {
+export const buildGraphQLDocument: (parsedFile: TextFile, file: BuildConfigFile) => Promise<VersionDocument<GraphApiSchema>> = async (parsedFile, file) => {
   let graphapi: GraphApiSchema
   if (parsedFile.type === GRAPHQL_DOCUMENT_TYPE.INTROSPECTION) {
     const introspection = (parsedFile?.data && '__schema' in parsedFile.data ? parsedFile?.data : parsedFile.data?.data) as IntrospectionQuery

src/apitypes/graphql/graphql.operation.ts

@@ -88,18 +88,13 @@ export const buildGraphQLOperation = (
 
   const apiKind = document.apiKind || API_KIND.BWC
 
-
-  const dataHash = syncDebugPerformance('[ModelsAndOperationHashing]', () => {
+  syncDebugPerformance('[ModelsAndOperationHashing]', () => {
     calculateSpecRefs(document.data, singleOperationRefsOnlySpec, singleOperationSpec)
-    const dataHash = calculateObjectHash(singleOperationSpec)
-    return dataHash
   }, debugCtx)
 
-
-
   return {
     operationId,
-    dataHash,
+    documentId: document.slug,
     apiType: GRAPHQL_API_TYPE,
     apiKind: rawToApiKind(apiKind),
     deprecated: !!singleOperationEffectiveSpec[type]?.[method]?.directives?.deprecated,

src/apitypes/graphql/index.ts

@@ -21,7 +21,7 @@ import { buildGraphQLOperations } from './graphql.operations'
 import { GRAPHQL_API_TYPE, GRAPHQL_DOCUMENT_TYPE } from './graphql.consts'
 import { parseGraphQLFile } from './graphql.parser'
 import { ApiBuilder } from '../../types'
-import { graphqlOperationsCompare } from './graphql.changes'
+import { compareDocuments } from './graphql.changes'
 
 export * from './graphql.consts'
 
@@ -32,5 +32,5 @@ export const graphqlApiBuilder: ApiBuilder<GraphApiSchema> = {
   buildDocument: buildGraphQLDocument,
   buildOperations: buildGraphQLOperations,
   dumpDocument: dumpGraphQLDocument,
-  compareOperationsData: graphqlOperationsCompare,
+  compareDocuments: compareDocuments,
 }

src/apitypes/rest/index.ts

@@ -18,13 +18,15 @@ import { OpenAPIV3 } from 'openapi-types'
 
 import { buildRestDocument, createRestExportDocument, dumpRestDocument } from './rest.document'
 import { REST_API_TYPE, REST_DOCUMENT_TYPE } from './rest.consts'
-import { compareRestOperationsData } from './rest.changes'
+import { compareDocuments } from './rest.changes'
 import { buildRestOperations } from './rest.operations'
 import { parseRestFile } from './rest.parser'
 import { ApiBuilder } from '../../types'
 import { calculateNormalizedOperationId } from '../../utils'
 
 export * from './rest.consts'
+export type { RestOperationData } from './rest.types'
+export { createCopyWithPrefixGroupOperationsOnly } from './rest.changes'
 
 export const restApiBuilder: ApiBuilder<OpenAPIV3.Document> = {
   apiType: REST_API_TYPE,
@@ -33,7 +35,7 @@ export const restApiBuilder: ApiBuilder<OpenAPIV3.Document> = {
   buildDocument: buildRestDocument,
   buildOperations: buildRestOperations,
   dumpDocument: dumpRestDocument,
-  compareOperationsData: compareRestOperationsData,
+  compareDocuments: compareDocuments,
   createNormalizedOperationId: calculateNormalizedOperationId,
   createExportDocument: createRestExportDocument,
 }