feat: add SchemaAccessor COMPASS-8799

Author: paula-stacho

package-lock.json

@@ -56,6 +56,7 @@
     "@types/mocha": "^10.0.1",
     "@types/node": "^18.11.18",
     "@types/reservoir": "^0.1.0",
+    "@types/json-schema": "^7.0.15",
     "@typescript-eslint/eslint-plugin": "^5.47.1",
     "@typescript-eslint/parser": "^5.47.1",
     "bson": "^6.7.0",

package.json

@@ -1,3 +1,4 @@
+import { InternalSchemaBasedAccessor, SchemaAccessor } from './schema-accessor';
 import { SchemaAnalyzer } from './schema-analyzer';
 import type {
   ArraySchemaType,
@@ -6,7 +7,7 @@ import type {
   DocumentSchemaType,
   PrimitiveSchemaType,
   SchemaType,
-  Schema,
+  Schema as InternalSchema,
   SchemaField,
   SchemaParseOptions,
   SimplifiedSchemaBaseType,
@@ -17,31 +18,18 @@ import type {
   SimplifiedSchema
 } from './schema-analyzer';
 import * as schemaStats from './stats';
+import { AnyIterable, StandardJSONSchema, MongodbJSONSchema, ExtendedJSONSchema } from './types';
+import { getCompletedSchemaAnalyzer } from './utils';
 
-type AnyIterable<T = any> = Iterable<T> | AsyncIterable<T>;
-
-function verifyStreamSource(
-  source: AnyIterable
-): AnyIterable {
-  if (!(Symbol.iterator in source) && !(Symbol.asyncIterator in source)) {
-    throw new Error(
-      'Unknown input type for `docs`. Must be an array, ' +
-        'stream or MongoDB Cursor.'
-    );
-  }
-
-  return source;
-}
-
-async function getCompletedSchemaAnalyzer(
+/**
+ * Analyze documents - schema can be retrieved in different formats.
+ */
+async function analyzeDocuments(
   source: AnyIterable,
   options?: SchemaParseOptions
-): Promise<SchemaAnalyzer> {
-  const analyzer = new SchemaAnalyzer(options);
-  for await (const doc of verifyStreamSource(source)) {
-    analyzer.analyzeDoc(doc);
-  }
-  return analyzer;
+): Promise<SchemaAccessor> {
+  const internalSchema = (await getCompletedSchemaAnalyzer(source, options)).getResult();
+  return new InternalSchemaBasedAccessor(internalSchema, options?.signal);
 }
 
 /**
@@ -51,7 +39,7 @@ async function getCompletedSchemaAnalyzer(
 async function parseSchema(
   source: AnyIterable,
   options?: SchemaParseOptions
-): Promise<Schema> {
+): Promise<InternalSchema> {
   return (await getCompletedSchemaAnalyzer(source, options)).getResult();
 }
 
@@ -78,19 +66,24 @@ export type {
   DocumentSchemaType,
   PrimitiveSchemaType,
   SchemaType,
-  Schema,
+  InternalSchema as Schema,
+  InternalSchema,
   SchemaField,
   SchemaParseOptions,
   SimplifiedSchemaBaseType,
   SimplifiedSchemaArrayType,
   SimplifiedSchemaDocumentType,
   SimplifiedSchemaType,
   SimplifiedSchemaField,
-  SimplifiedSchema
+  SimplifiedSchema,
+  StandardJSONSchema,
+  MongodbJSONSchema,
+  ExtendedJSONSchema
 };
 
 export {
   parseSchema,
+  analyzeDocuments,
   getSchemaPaths,
   getSimplifiedSchema,
   SchemaAnalyzer,

src/index.ts

@@ -0,0 +1,48 @@
+import { Schema as InternalSchema } from './schema-analyzer';
+import { internalSchemaToExtended, internalSchemaToMongodb, internalSchemaToStandard } from './schema-convertors';
+import { ExtendedJSONSchema, MongodbJSONSchema, StandardJSONSchema } from './types';
+
+export interface SchemaAccessor {
+  getStandardJsonSchema: () => Promise<StandardJSONSchema>;
+  getMongodbJsonSchema: () => Promise<MongodbJSONSchema>;
+  getExtendedJsonSchema: () => Promise<ExtendedJSONSchema>;
+  getInternalSchema: () => Promise<InternalSchema>;
+}
+
+/**
+ * Accessor for different schema formats.
+ * Internal schema is provided at initialization,
+ * the others are converted lazily and memoized.
+ * Conversion can be aborted.
+ */
+export class InternalSchemaBasedAccessor implements SchemaAccessor {
+  private internalSchema: InternalSchema;
+  private standardJSONSchema?: StandardJSONSchema;
+  private mongodbJSONSchema?: MongodbJSONSchema;
+  private extendedJSONSchema?: ExtendedJSONSchema;
+  private signal?: AbortSignal;
+
+  constructor(internalSchema: InternalSchema, signal?: AbortSignal) {
+    this.signal = signal;
+    this.internalSchema = internalSchema;
+  }
+
+  async getInternalSchema(): Promise<InternalSchema> {
+    return this.internalSchema;
+  }
+
+  async getStandardJsonSchema(): Promise<StandardJSONSchema> {
+    if (this.standardJSONSchema) return this.standardJSONSchema;
+    return this.standardJSONSchema = await internalSchemaToStandard(this.internalSchema, { signal: this.signal });
+  }
+
+  async getMongodbJsonSchema(): Promise<MongodbJSONSchema> {
+    if (this.mongodbJSONSchema) return this.mongodbJSONSchema;
+    return this.mongodbJSONSchema = await internalSchemaToMongodb(this.internalSchema, { signal: this.signal });
+  }
+
+  async getExtendedJsonSchema(): Promise<ExtendedJSONSchema> {
+    if (this.extendedJSONSchema) return this.extendedJSONSchema;
+    return this.extendedJSONSchema = await internalSchemaToExtended(this.internalSchema, { signal: this.signal });
+  }
+}

src/schema-accessor.ts

@@ -163,6 +163,7 @@ type SemanticTypeMap = {
 export type SchemaParseOptions = {
   semanticTypes?: boolean | SemanticTypeMap;
   storeValues?: boolean;
+  signal?: AbortSignal;
 };
 
 /**

src/schema-analyzer.ts

@@ -0,0 +1,29 @@
+import { Schema as InternalSchema } from './schema-analyzer';
+import { ExtendedJSONSchema, MongodbJSONSchema, StandardJSONSchema } from './types';
+
+export function internalSchemaToStandard(
+  internalSchema: InternalSchema,
+  options: {
+    signal?: AbortSignal
+}): StandardJSONSchema {
+  // TODO: COMPASS-8700
+  return {};
+}
+
+export function internalSchemaToMongodb(
+  internalSchema: InternalSchema,
+  options: {
+    signal?: AbortSignal
+}): MongodbJSONSchema {
+  // TODO: COMPASS-8701
+  return {} as MongodbJSONSchema;
+}
+
+export function internalSchemaToExtended(
+  internalSchema: InternalSchema,
+  options: {
+    signal?: AbortSignal
+}): ExtendedJSONSchema {
+  // TODO: COMPASS-8702
+  return {} as ExtendedJSONSchema;
+}

src/schema-convertors.ts

@@ -0,0 +1,22 @@
+import { JSONSchema4 } from 'json-schema';
+
+export type StandardJSONSchema = JSONSchema4;
+
+export type MongodbJSONSchema = Pick<StandardJSONSchema, 'title' | 'required' | 'description'> & {
+  bsonType: string;
+  properties?: Record<string, MongodbJSONSchema>;
+  items?: MongodbJSONSchema[];
+  anyOf?: MongodbJSONSchema[];
+}
+
+export type ExtendedJSONSchema = StandardJSONSchema & {
+  ['x-bsonType']: string;
+  ['x-metadata']: {
+    hasDuplicates: boolean;
+    probability: number;
+    count: number;
+  };
+  ['x-sampleValues']: any[];
+}
+
+export type AnyIterable<T = any> = Iterable<T> | AsyncIterable<T>;

src/types.ts

@@ -0,0 +1,27 @@
+import { SchemaAnalyzer, SchemaParseOptions } from './schema-analyzer';
+import { AnyIterable } from './types';
+
+export function verifyStreamSource(
+  source: AnyIterable
+): AnyIterable {
+  if (!(Symbol.iterator in source) && !(Symbol.asyncIterator in source)) {
+    throw new Error(
+      'Unknown input type for `docs`. Must be an array, ' +
+        'stream or MongoDB Cursor.'
+    );
+  }
+
+  return source;
+}
+
+export async function getCompletedSchemaAnalyzer(
+  source: AnyIterable,
+  options?: SchemaParseOptions
+): Promise<SchemaAnalyzer> {
+  const analyzer = new SchemaAnalyzer(options);
+  for await (const doc of verifyStreamSource(source)) {
+    if (options?.signal?.aborted) throw options.signal.aborted;
+    analyzer.analyzeDoc(doc);
+  }
+  return analyzer;
+}