feat: use mask (*) instead of parameter names while matching rest operations during comparison

Author: CountRedClaw

src/apitypes/rest/rest.types.ts

@@ -18,13 +18,14 @@ import type { OpenAPIV3 } from 'openapi-types'
 
 import type { ApiOperation, NotificationMessage, VersionDocument } from '../../types'
 import { REST_DOCUMENT_TYPE, REST_KIND_KEY, REST_SCOPES } from './rest.consts'
+import { NormalizedPath } from '../../utils'
 
 export type RestScopeType = keyof typeof REST_SCOPES
 export type RestDocumentType = (typeof REST_DOCUMENT_TYPE)[keyof typeof REST_DOCUMENT_TYPE]
 export type CustomTags = Record<string, unknown>
 
 export interface RestOperationMeta {
-  path: string                            // `/packages/*/version/*`
+  path: NormalizedPath                    // `/packages/*/version/*`
   originalPath: string                    // `/packages/{packageId}/version/{version}`
   method: OpenAPIV3.HttpMethods           // `get` | `post` | ...
   tags?: string[]                         // operations tags

src/builder.ts

@@ -27,8 +27,6 @@ import {
   ResolvedDeprecatedOperations,
   ResolvedDocuments,
   ResolvedOperations,
-  ResolvedVersionOperationsHashMap,
-  VALIDATION_RULES_SEVERITY_LEVEL_WARNING,
   VersionId,
 } from './types'
 import {
@@ -492,10 +490,9 @@ export class PackageVersionBuilder implements IPackageVersionBuilder {
     const operationsTypes: OperationTypes[] = []
 
     for (const apiType of this.existingOperationsApiTypes) {
-      const operationsHashMap = this.operationsHashMapByApiType(apiType)
       operationsTypes.push({
         apiType: apiType,
-        operations: operationsHashMap,
+        operationsCount: this.operations.size,
       })
     }
 
@@ -508,18 +505,6 @@ export class PackageVersionBuilder implements IPackageVersionBuilder {
     return new Set(apiTypes)
   }
 
-  private operationsHashMapByApiType(operationsApiType: OperationsApiType): ResolvedVersionOperationsHashMap {
-    const hashMap: ResolvedVersionOperationsHashMap = {}
-
-    for (const { apiType, operationId, dataHash } of this.operations.values()) {
-      if (apiType === operationsApiType) {
-        hashMap[operationId] = dataHash
-      }
-    }
-
-    return hashMap
-  }
-
   async parseFile(fileId: string, source: Blob): Promise<File | null> {
     if (this.parsedFiles.has(fileId)) {
       return this.parsedFiles.get(fileId) ?? null

src/components/compare/compare.operations.ts

@@ -35,7 +35,6 @@ import {
   getOperationTags,
   getOperationTypesFromTwoVersions,
   getUniqueApiTypesFromVersions,
-  OperationIdentityMap,
   takeSubstringIf,
   totalChanges,
 } from './compare.utils'
@@ -50,9 +49,9 @@ import {
   removeObjectDuplicates,
   slugify,
 } from '../../utils'
-import { asyncFunction } from '../../utils/async'
-import { asyncDebugPerformance, DebugPerformanceContext, logLongBuild, syncDebugPerformance } from '../../utils/logs'
+import { asyncDebugPerformance, DebugPerformanceContext } from '../../utils/logs'
 import { validateBwcBreakingChanges } from './bwc.validation'
+import { REST_API_TYPE } from '../../apitypes'
 
 export async function compareVersionsOperations(
   prev: VersionParams,
@@ -107,6 +106,18 @@ export async function compareVersionsOperations(
   }
 }
 
+const HANDLE_TYPE_FULLY_ADDED = 'added'
+const HANDLE_TYPE_FULLY_REMOVED = 'removed'
+const HANDLE_TYPE_FULLY_CHANGED = 'changed'
+
+type HandleType = typeof HANDLE_TYPE_FULLY_ADDED | typeof HANDLE_TYPE_FULLY_REMOVED | typeof HANDLE_TYPE_FULLY_CHANGED
+
+export interface MappingResult<T extends PropertyKey> {
+  [HANDLE_TYPE_FULLY_ADDED]: T[]
+  [HANDLE_TYPE_FULLY_REMOVED]: T[]
+  [HANDLE_TYPE_FULLY_CHANGED]: Record<T, T>
+}
+
 async function compareCurrentApiType(
   apiType: OperationsApiType,
   prev: VersionCache | null,
@@ -124,30 +135,37 @@ async function compareCurrentApiType(
 
   const [prevOperationTypesData, currOperationTypesData] = getOperationTypesFromTwoVersions(prev, curr)
 
-  const changedIdToOriginal: OperationIdentityMap = {}
-  const prevOperationHashMap = getOperationsHashMapByApiType(apiType, prevOperationTypesData, changedIdToOriginal, ctx)
-  const currOperationHashMap = getOperationsHashMapByApiType(apiType, currOperationTypesData, changedIdToOriginal, ctx, true)
+  const prevOperationsCount = prevOperationTypesData.find(data => apiType === data.apiType)?.operationsCount
+  const currOperationsCount = currOperationTypesData.find(data => apiType === data.apiType)?.operationsCount
 
-  const operationIds = new Set([...Object.keys(prevOperationHashMap), ...Object.keys(currOperationHashMap)])
-  const operationsMapping: Record<string, Array<string>> = { added: [], removed: [], changed: [] }
-  const apiAudienceTransitions: ApiAudienceTransition[] = []
-  const pairedOperationIds: Array<string> = []
+  const { operations: prevOperations = [] } = await versionOperationsResolver(apiType, prev?.version ?? '', prev?.packageId ?? '', undefined, false, prevOperationsCount) || {}
+  const { operations: currOperations = [] } = await versionOperationsResolver(apiType, curr?.version ?? '', curr?.packageId ?? '', undefined, false, currOperationsCount) || {}
 
-  for (const operationId of operationIds) {
-    const v1OperationHash = prevOperationHashMap[operationId]
-    const v2OperationHash = currOperationHashMap[operationId]
-    if (v1OperationHash && v2OperationHash) {
-      pairedOperationIds.push(changedIdToOriginal[operationId] || operationId)
+  const [prevReducedOperationIdToHashMap, prevReducedOperationIdToOriginal] = getOperationsHashMapByApiType(apiType, prevOperations, ctx)
+  const [currReducedOperationIdToHashMap, currReducedOperationIdToOriginal] = getOperationsHashMapByApiType(apiType, currOperations, ctx, true)
+
+  const reducedOperationIds = new Set([...Object.keys(prevReducedOperationIdToHashMap), ...Object.keys(currReducedOperationIdToHashMap)])
+  const operationsMapping: MappingResult<OperationId> = { [HANDLE_TYPE_FULLY_ADDED]: [], [HANDLE_TYPE_FULLY_REMOVED]: [], [HANDLE_TYPE_FULLY_CHANGED]: {} }
+  const apiAudienceTransitions: ApiAudienceTransition[] = []
+  const pairedOperationIds: Record<OperationId, OperationId> = {}
+
+  for (const reducedOperationId of reducedOperationIds) {
+    const prevOperationHash = prevReducedOperationIdToHashMap[reducedOperationId]
+    const currOperationHash = currReducedOperationIdToHashMap[reducedOperationId]
+    const prevOperationId = prevReducedOperationIdToOriginal[reducedOperationId]
+    const currOperationId = currReducedOperationIdToOriginal[reducedOperationId]
+    if (prevOperationHash && currOperationHash) {
+      pairedOperationIds[prevOperationId] = currOperationId
       // operation not changed
-      if (v1OperationHash === v2OperationHash) { continue }
+      if (prevOperationHash === currOperationHash) { continue }
       // operation changed
-      operationsMapping.changed.push(changedIdToOriginal[operationId] || operationId)
-    } else if (v1OperationHash) {
+      operationsMapping[HANDLE_TYPE_FULLY_CHANGED][prevOperationId] = currOperationId
+    } else if (prevOperationHash) {
       // operation removed
-      operationsMapping.removed.push(changedIdToOriginal[operationId] || operationId)
-    } else if (v2OperationHash) {
+      operationsMapping[HANDLE_TYPE_FULLY_REMOVED].push(prevOperationId)
+    } else if (currOperationHash) {
       // operation added
-      operationsMapping.added.push(changedIdToOriginal[operationId] || operationId)
+      operationsMapping[HANDLE_TYPE_FULLY_ADDED].push(currOperationId)
     }
   }
 
@@ -229,9 +247,11 @@ async function compareCurrentApiType(
   }
 
   // todo: convert from objects analysis to apihub-diff result analysis after the "info" section participates in the comparison of operations
-  await asyncDebugPerformance('[ApiAudience]', async () => await executeInBatches(pairedOperationIds, async (operationsBatch) => {
-    const { operations: prevOperationsWithoutData = [] } = await versionOperationsResolver(apiType, prevVersion, prevPackageId, operationsBatch, false) || {}
-    const { operations: currOperationsWithoutData = [] } = await versionOperationsResolver(apiType, currVersion, currPackageId, operationsBatch, false) || {}
+  await asyncDebugPerformance('[ApiAudience]', async () => await executeInBatches(Object.entries(pairedOperationIds), async (operationsBatch) => {
+    const previousBatch = operationsBatch.map(([prevOperationId]) => prevOperationId)
+    const currentBatch = operationsBatch.map(([, currOperationId]) => currOperationId)
+    const { operations: currOperationsWithoutData = [] } = await versionOperationsResolver(apiType, currVersion, currPackageId, previousBatch, false) || {}
+    const { operations: prevOperationsWithoutData = [] } = await versionOperationsResolver(apiType, prevVersion, prevPackageId, currentBatch, false) || {}
 
     const pairOperationsMap = createPairOperationsMap(currGroupSlug, prevGroupSlug, currOperationsWithoutData, prevOperationsWithoutData)
     Object.values(pairOperationsMap).forEach((pair) => {
@@ -262,12 +282,11 @@ async function compareCurrentApiType(
   if (!apiBuilder.compareOperationsData) { return null }
 
   await asyncDebugPerformance('[Changed]', async (innerDebugCtx) =>
-    await executeInBatches(operationsMapping.changed, async (operationsBatch) => {
-      const currentBatch = currentGroup ? operationsBatch.map(operationId => currGroupSlug + operationId.substring(currGroupSlug.length)) : operationsBatch
-      const previousBatch = previousGroup ? operationsBatch.map(operationId => prevGroupSlug + operationId.substring(prevGroupSlug.length)) : operationsBatch
-
-      const { operations: prevOperationsWithData = [] } = await versionOperationsResolver(apiType, prevVersion!, prevPackageId!, previousBatch) || {}
-      const { operations: currOperationsWithData = [] } = await versionOperationsResolver(apiType, currVersion!, currPackageId!, currentBatch) || {}
+    await executeInBatches(Object.entries(operationsMapping.changed), async (operationsBatch) => {
+      const previousBatch = operationsBatch.map(([prevOperationId]) => prevOperationId)
+      const currentBatch = operationsBatch.map(([, currOperationId]) => currOperationId)
+      const { operations: prevOperationsWithData = [] } = await versionOperationsResolver(apiType, prevVersion, prevPackageId, previousBatch) || {}
+      const { operations: currOperationsWithData = [] } = await versionOperationsResolver(apiType, currVersion, currPackageId, currentBatch) || {}
 
       const operationsMap = createPairOperationsMap(currGroupSlug, prevGroupSlug, currOperationsWithData, prevOperationsWithData)
 
@@ -343,21 +362,20 @@ async function compareCurrentApiType(
   ]
 }
 
-const HANDLE_TYPE_FULLY_ADDED = 'added'
-const HANDLE_TYPE_FULLY_REMOVED = 'removed'
-
-type HandleType = typeof HANDLE_TYPE_FULLY_ADDED | typeof HANDLE_TYPE_FULLY_REMOVED
-
 const createPairOperationsMap = (currGroupSlug: string, prevGroupSlug: string, currentOperations: ResolvedOperation[], previousOperations: ResolvedOperation[]): Record<string, { previous?: ResolvedOperation; current: ResolvedOperation }> => {
 
   const operationsMap: Record<string, { previous?: ResolvedOperation; current: ResolvedOperation }> = {}
 
   for (const currentOperation of currentOperations) {
-    operationsMap[takeSubstringIf(!!currGroupSlug, currentOperation.operationId, currGroupSlug.length)] = { current: currentOperation }
+    // todo
+    const normalizedOperationId = currentOperation.apiType === REST_API_TYPE ? slugify(`${removeFirstSlash(`${currentOperation.metadata.path}`)}-${currentOperation.metadata.method}`) : currentOperation.operationId
+    operationsMap[takeSubstringIf(!!currGroupSlug, normalizedOperationId, currGroupSlug.length)] = { current: currentOperation }
   }
 
   for (const previousOperation of previousOperations) {
-    const prevOperationId = takeSubstringIf(!!prevGroupSlug, previousOperation.operationId, prevGroupSlug.length)
+    // todo
+    const normalizedOperationId = previousOperation.apiType === REST_API_TYPE ? slugify(`${removeFirstSlash(`${previousOperation.metadata.path}`)}-${previousOperation.metadata.method}`) : previousOperation.operationId
+    const prevOperationId = takeSubstringIf(!!prevGroupSlug, normalizedOperationId, prevGroupSlug.length)
     const operationsMappingElement = operationsMap[prevOperationId]
     if (operationsMappingElement) {
       operationsMap[prevOperationId] = {

src/components/compare/compare.utils.ts

@@ -28,7 +28,10 @@ import {
   VersionCache,
 } from '../../types'
 import { BUILD_TYPE, EMPTY_CHANGE_SUMMARY, MESSAGE_SEVERITY } from '../../consts'
-import { convertToSlug, takeIfDefined } from '../../utils'
+import { convertToSlug, NormalizedPath, PATH_PARAM_UNIFIED_PLACEHOLDER, slugify, takeIfDefined } from '../../utils'
+import { REST_API_TYPE } from '../../apitypes'
+import { OpenAPIV3 } from 'openapi-types'
+import { CharMap } from 'slug'
 
 export const totalChanges = (changeSummary?: ChangeSummary): number => {
   return changeSummary
@@ -86,28 +89,42 @@ export function getOperationTypesFromTwoVersions(
 type OperationIdWithoutGroupPrefix = string
 export type OperationIdentityMap = Record<OperationIdWithoutGroupPrefix, OperationId>
 
+type NormalizedOperationId = `${NormalizedPath}-${OpenAPIV3.HttpMethods}`
+
+const IGNORE_PATH_PARAM_UNIFIED_PLACEHOLDER: CharMap = { [PATH_PARAM_UNIFIED_PLACEHOLDER]: PATH_PARAM_UNIFIED_PLACEHOLDER }
+
 export function getOperationsHashMapByApiType(
   currentApiType: OperationsApiType,
-  operationTypes: OperationTypes[],
-  operationIdentityMap: OperationIdentityMap,
+  operations: ResolvedOperation[],
   ctx: CompareContext,
   areOperationsFromCurrentVersion: boolean = false,
-): ResolvedVersionOperationsHashMap {
-  const resolvedHashMap = { ...operationTypes.find(({ apiType: type }) => type === currentApiType)?.operations || {} }
+): [ResolvedVersionOperationsHashMap, OperationIdentityMap] {
   const { buildType, currentGroup, previousGroup } = ctx.config
+  const currentApiTypeOperations = operations.filter(({ apiType }) => apiType === currentApiType)
+
+  const resolvedHashMap: ResolvedVersionOperationsHashMap = {}
+  const normalizedToOriginalOperationIdMap: Record<NormalizedOperationId | OperationId, OperationId> = {}
+
+  for (const { operationId, apiType, metadata, dataHash } of currentApiTypeOperations) {
+    const normalizedOperationId = apiType === REST_API_TYPE ? slugify(`${metadata.path}-${metadata.method}`, [], IGNORE_PATH_PARAM_UNIFIED_PLACEHOLDER) : operationId
+    resolvedHashMap[normalizedOperationId] = dataHash
+    normalizedToOriginalOperationIdMap[normalizedOperationId] = operationId
+  }
 
   if (buildType !== BUILD_TYPE.PREFIX_GROUPS_CHANGELOG) {
-    return resolvedHashMap
+    return [resolvedHashMap, normalizedToOriginalOperationIdMap]
   }
 
   if (!currentGroup || !previousGroup) {
     ctx.notifications.push({
       severity: MESSAGE_SEVERITY.Warning,
       message: `Build type is prefix group changelog, but one of the groups is not provided: currentGroup=${currentGroup}, previousGroup=${previousGroup}`,
     })
-    return resolvedHashMap
+    return [resolvedHashMap, normalizedToOriginalOperationIdMap]
   }
 
+  const changedIdToOriginal: OperationIdentityMap = {}
+
   for (const [operationId, dataHash] of Object.entries(resolvedHashMap)) {
     Reflect.deleteProperty(resolvedHashMap, operationId)
 
@@ -116,11 +133,11 @@ export function getOperationsHashMapByApiType(
     if (operationId.startsWith(groupSlug)) {
       const changedOperationId = operationId.substring(groupSlug.length)
       resolvedHashMap[changedOperationId] = dataHash
-      operationIdentityMap[changedOperationId] = operationId
+      changedIdToOriginal[changedOperationId] = normalizedToOriginalOperationIdMap[operationId]
     }
   }
 
-  return resolvedHashMap
+  return [resolvedHashMap, changedIdToOriginal]
 }
 
 export function getOperationMetadata(operation: ResolvedOperation): OperationChangesMetadata {

src/types/external/operations.ts

@@ -27,6 +27,7 @@ export type VersionOperationsResolver = (
   packageId: PackageId,
   operationsIds?: OperationId[],
   includeData?: boolean,
+  operationsCount?: number,
 ) => Promise<ResolvedOperations | null>
 
 export interface ResolvedOperations {
@@ -35,7 +36,7 @@ export interface ResolvedOperations {
 }
 
 export interface ResolvedOperation<M = any> {
-  operationId: string
+  operationId: OperationId
   title: string
   dataHash: string
   apiType: OperationsApiType

src/types/external/version.ts

@@ -21,6 +21,7 @@ export type VersionResolver = (
   packageId: PackageId,
   version: VersionId,
   includeOperations?: boolean,
+  includeSummary?: boolean,
 ) => Promise<ResolvedVersion | null>
 
 export type ResolvedVersion = {
@@ -43,7 +44,6 @@ export type OperationTypes = {
   changesSummary?: ChangeSummary
   operationsCount?: number
   deprecatedCount?: number
-  operations?: ResolvedVersionOperationsHashMap
 }
 
 export interface ResolvedVersionOperationsHashMap {

src/utils/builder.ts

@@ -15,13 +15,15 @@
  */
 
 import { ApiAudienceTransition } from './../types/external/comparison'
-import { ApiKind, ChangeSummary, ResolvedOperation } from '../types'
 import {
   ANNOTATION_CHANGE_TYPE,
+  ApiKind,
   BREAKING_CHANGE_TYPE,
+  ChangeSummary,
   DEPRECATED_CHANGE_TYPE,
   ImpactedOperationSummary,
   NON_BREAKING_CHANGE_TYPE,
+  ResolvedOperation,
   SEMI_BREAKING_CHANGE_TYPE,
   UNCLASSIFIED_CHANGE_TYPE,
 } from '../types'
@@ -41,10 +43,19 @@ export const removeFirstSlash = (input: string): string => {
   return input.startsWith('/') ? input.substring(1) : input
 }
 
-export const normalizePath = (path: string): string => {
-  return path.replace(new RegExp('{.*?}', 'g'), '*')
+export type NormalizedPath = string
+
+export const normalizePath = (path: string): NormalizedPath => {
+  return hidePathParamNames(path)
+}
+
+export function hidePathParamNames(path: string): string {
+  return path.replace(PATH_PARAMETER_REGEXP, PATH_PARAM_UNIFIED_PLACEHOLDER)
 }
 
+const PATH_PARAMETER_REGEXP = /\{.*?\}/g
+export const PATH_PARAM_UNIFIED_PLACEHOLDER = '*'
+
 export const filesDiff = (files1: { fileId: string }[], files2: { fileId: string }[]): { fileId: string }[] => {
   return files1.filter((f1) => !files2.find((f2) => f1.fileId === f2.fileId))
 }

src/utils/document.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import createSlug from 'slug'
+import createSlug, { CharMap } from 'slug'
 import {
   _ParsedFileResolver,
   ApiOperation,
@@ -87,6 +87,7 @@ createSlug.extend({ '_': '_' })
 createSlug.extend({ '.': '-' })
 createSlug.extend({ '(': '-' })
 createSlug.extend({ ')': '-' })
+// createSlug.extend({ '*': '*' })
 
 export const findSharedPath = (fileIds: string[]): string => {
   if (!fileIds.length) { return '' }
@@ -99,12 +100,12 @@ export const findSharedPath = (fileIds: string[]): string => {
   return first.slice(0, i).join('/') + (i ? '/' : '')
 }
 
-export const slugify = (text: string, slugs: string[] = []): string => {
+export const slugify = (text: string, slugs: string[] = [], charMapEntry?: CharMap): string => {
   if (!text) {
     return ''
   }
 
-  const slug = createSlug(text)
+  const slug = createSlug(text, { charmap: { ...createSlug.charmap, ...charMapEntry } })
   let suffix: string = ''
   // add suffix if not unique
   while (slugs.includes(slug + suffix)) { suffix = String(+suffix + 1) }