feat: hash operation helper (#6876)

Author: n1ru4l

.changeset/perfect-actors-develop.md

@@ -0,0 +1,6 @@
+---
+'@graphql-hive/core': minor
+---
+
+Add helper function `hashOperation` for generating consistent hashes used within the Hive Console
+platform.

packages/libraries/core/package.json

@@ -48,13 +48,15 @@
     "@graphql-tools/utils": "^10.0.0",
     "@whatwg-node/fetch": "^0.10.6",
     "async-retry": "^1.3.3",
+    "js-md5": "0.8.3",
     "lodash.sortby": "^4.7.0",
     "tiny-lru": "^8.0.2"
   },
   "devDependencies": {
     "@apollo/federation": "0.38.1",
     "@apollo/subgraph": "2.9.3",
     "@types/async-retry": "1.4.8",
+    "@types/js-md5": "0.8.0",
     "@types/lodash.sortby": "4.7.9",
     "graphql": "16.9.0",
     "nock": "14.0.0",

packages/libraries/core/src/index.ts

@@ -1,4 +1,5 @@
 export * from './normalize/operation.js';
+export { collectSchemaCoordinates } from './client/collect-schema-coordinates.js';
 export type {
   HivePluginOptions,
   HiveClient,

packages/libraries/core/src/normalize/operation.ts

@@ -3,17 +3,24 @@ import {
   DefinitionNode,
   DirectiveNode,
   DocumentNode,
+  GraphQLSchema,
   Kind,
   OperationDefinitionNode,
   print,
   SelectionNode,
   separateOperations,
   stripIgnoredCharacters,
+  TypeInfo,
   VariableDefinitionNode,
   visit,
 } from 'graphql';
+import { md5 } from 'js-md5';
 import sortBy from 'lodash.sortby';
+import { collectSchemaCoordinates } from '../client/collect-schema-coordinates.js';
 
+/**
+ * Normalize a operation document.
+ */
 export function normalizeOperation({
   document,
   operationName,
@@ -152,3 +159,93 @@ function dropUnusedDefinitions(doc: DocumentNode, operationName?: string) {
 
   return separateOperations(doc)[operationName] ?? doc;
 }
+
+function findOperationDefinition(doc: DocumentNode) {
+  return doc.definitions.find(isOperationDef);
+}
+
+/** normalize a graphql operation into a stable hash as used internally within our ClickHouse Database. */
+export function preprocessOperation(operation: {
+  document: DocumentNode;
+  schemaCoordinates: Iterable<string>;
+  operationName: string | null;
+}) {
+  const body = normalizeOperation({
+    document: operation.document,
+    hideLiterals: true,
+    removeAliases: true,
+  });
+
+  // Two operations with the same hash has to be equal:
+  // 1. body is the same
+  // 2. name is the same
+  // 3. used schema coordinates are equal - this is important to assign schema coordinate to an operation
+
+  const uniqueCoordinatesSet = new Set<string>();
+  for (const field of operation.schemaCoordinates) {
+    uniqueCoordinatesSet.add(field);
+    // Add types as well:
+    // `Query.foo` -> `Query`
+    const at = field.indexOf('.');
+    if (at > -1) {
+      uniqueCoordinatesSet.add(field.substring(0, at));
+    }
+  }
+
+  const sortedCoordinates = Array.from(uniqueCoordinatesSet).sort();
+
+  const operationDefinition = findOperationDefinition(operation.document);
+
+  if (!operationDefinition) {
+    return null;
+  }
+
+  const operationName = operation.operationName ?? operationDefinition.name?.value;
+
+  const hash = md5
+    .create()
+    .update(body)
+    .update(operationName ?? '')
+    .update(sortedCoordinates.join(';')) // we do not need to sort from A to Z, default lexicographic sorting is enough
+    .hex();
+
+  return {
+    type: operationDefinition.operation,
+    hash,
+    body,
+    coordinates: sortedCoordinates,
+    name: operationName || null,
+  };
+}
+
+/**
+ * Hash a executable GraphQL document according to Hive platforms algorithm
+ * for identification.
+ *
+ * Return null if no executable operation definition was found.
+ */
+export function hashOperation(args: {
+  documentNode: DocumentNode;
+  variables: null | {
+    [key: string]: unknown;
+  };
+  operationName: string | null;
+  schema: GraphQLSchema;
+  typeInfo?: TypeInfo;
+}) {
+  const schemaCoordinates = collectSchemaCoordinates({
+    documentNode: args.documentNode,
+    processVariables: args.variables !== null,
+    variables: args.variables ?? {},
+    schema: args.schema,
+    typeInfo: args.typeInfo ?? new TypeInfo(args.schema),
+  });
+
+  const result = preprocessOperation({
+    document: args.documentNode,
+    schemaCoordinates: schemaCoordinates,
+    operationName: args.operationName,
+  });
+
+  return result?.hash ?? null;
+}

packages/services/api/src/modules/app-deployments/providers/persisted-document-ingester.ts

@@ -1,10 +1,9 @@
 import { buildSchema, DocumentNode, GraphQLError, Kind, parse, TypeInfo, validate } from 'graphql';
 import PromiseQueue from 'p-queue';
 import { z } from 'zod';
-import { collectSchemaCoordinates } from '@graphql-hive/core/src/client/collect-schema-coordinates';
+import { collectSchemaCoordinates, preprocessOperation } from '@graphql-hive/core';
 import { buildOperationS3BucketKey } from '@hive/cdn-script/artifact-storage-reader';
 import { ServiceLogger } from '@hive/service-common';
-import { normalizeOperation } from '@hive/usage-ingestor/src/normalize-operation';
 import { sql as c_sql, ClickHouse } from '../../operations/providers/clickhouse-client';
 import { S3Config } from '../../shared/providers/s3-config';
 
@@ -186,18 +185,18 @@ export class PersistedDocumentIngester {
 
       const operationName = operationNames[0] ?? null;
 
-      const coordinates = collectSchemaCoordinates({
+      const schemaCoordinates = collectSchemaCoordinates({
         documentNode,
         processVariables: false,
         variables: null,
         schema,
         typeInfo,
       });
 
-      const normalizedOperation = normalizeOperation({
-        document: operation.body,
-        fields: coordinates,
+      const normalizedOperation = preprocessOperation({
+        document: documentNode,
         operationName,
+        schemaCoordinates,
       });
 
       documents.push({
@@ -206,7 +205,7 @@ export class PersistedDocumentIngester {
         internalHash: normalizedOperation?.hash ?? operation.hash,
         body: operation.body,
         operationName,
-        schemaCoordinates: Array.from(coordinates),
+        schemaCoordinates: Array.from(schemaCoordinates),
       });
 
       index++;

packages/services/usage-ingestor/src/normalize-operation.ts

@@ -1,5 +1,6 @@
-import type { OperationTypeNode } from 'graphql';
+import { parse, type DocumentNode, type OperationTypeNode } from 'graphql';
 import LRU from 'tiny-lru';
+import { preprocessOperation } from '@graphql-hive/core';
 import type { ServiceLogger } from '@hive/service-common';
 import type {
   ProcessedOperation,
@@ -18,7 +19,6 @@ import {
   schemaCoordinatesSize,
   totalOperations,
 } from './metrics';
-import { normalizeOperation } from './normalize-operation';
 import {
   stringifyAppDeploymentUsageRecord,
   stringifyQueryOrMutationOperation,
@@ -41,14 +41,28 @@ type NormalizeFunction = (arg: RawOperationMapRecord) => {
 const DAY_IN_MS = 86_400_000;
 const RETENTION_FALLBACK = 365;
 
+function parseSafe(operation: string): DocumentNode | null {
+  try {
+    return parse(operation);
+  } catch {
+    return null;
+  }
+}
+
 export function createProcessor(config: { logger: ServiceLogger }) {
   const { logger } = config;
   const normalize = cache(
     (operation: RawOperationMapRecord) => {
       normalizeCacheMisses.inc();
-      return normalizeOperation({
-        document: operation.operation,
-        fields: operation.fields,
+      const document = parseSafe(operation.operation);
+
+      if (!document) {
+        return null;
+      }
+
+      return preprocessOperation({
+        document,
+        schemaCoordinates: operation.fields,
         operationName: operation.operationName ?? null,
       });
     },