CCM-12896: Preprocess schemas to remove allOf

gareth-allan · gareth-allan · commit d8d4a8fcb715 · 2025-12-15T12:48:54.000Z
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -61,6 +61,7 @@
     "utils/sender-management",
     "src/cloudevents",
     "src/digital-letters-events",
+    "src/python-schema-generator",
     "src/typescript-schema-generator",
     "tests/playwright"
   ]
diff --git a/project.code-workspace b/project.code-workspace
@@ -79,10 +79,10 @@
     },
     "terminal.integrated.scrollback": 10000,
     "jest.virtualFolders": [
-
       { "name": "key-generation", "rootPath": "lambdas/key-generation" },
       { "name": "mesh-poll", "rootPath": "lambdas/mesh-poll" },
       { "name": "refresh-apim-access-token", "rootPath": "lambdas/refresh-apim-access-token" },
+      { "name": "python-schema-generator", "rootPath": "src/python-schema-generator" },
       { "name": "ttl-create-lambda", "rootPath": "lambdas/ttl-create-lambda/" },
       { "name": "ttl-handle-expiry-lambda", "rootPath": "lambdas/ttl-handle-expiry-lambda" },
       { "name": "ttl-poll-lambda", "rootPath": "lambdas/ttl-poll-lambda" },
diff --git a/src/python-schema-generator/.gitignore b/src/python-schema-generator/.gitignore
@@ -0,0 +1 @@
+schemas
diff --git a/src/python-schema-generator/Makefile b/src/python-schema-generator/Makefile
@@ -1,7 +1,8 @@
 # Variables
 SCHEMA_SRC_DIR := ../../schemas/digital-letters/2025-10-draft/events
 OUTPUT_DIR := ../digital-letters-events/digital_letters_events/models
-SCRIPTS_DIR := scripts
+SRC_DIR := src
+SCHEMAS_DIR := $(SRC_DIR)/schemas
 
 # Default target
 .PHONY: all clean generate install install-dev test coverage help
@@ -10,9 +11,12 @@ all: generate
 
 # Install production dependencies
 install:
-	@echo "Installing production dependencies..."
+	@echo "Installing Python production dependencies..."
 	@pip install -r requirements.txt
 	@echo "Production dependencies installed!"
+	@echo "Installing Node production dependencies..."
+	@npm install
+	@echo "Node production dependencies installed!"
 
 # Install development dependencies
 install-dev:
@@ -24,8 +28,9 @@ install-dev:
 generate: install
 	@echo "Generating Pydantic models from JSON schemas..."
 	@mkdir -p $(OUTPUT_DIR)
-	@python $(SCRIPTS_DIR)/generate_models.py \
-		--input-dir $(SCHEMA_SRC_DIR) \
+	@npm run merge
+	@python $(SRC_DIR)/generate_models.py \
+		--input-dir $(SCHEMAS_DIR) \
 		--output-dir $(OUTPUT_DIR)
 	@echo "Pydantic models generated in $(OUTPUT_DIR)/"
 
@@ -50,6 +55,7 @@ coverage: install-dev
 clean:
 	@echo "Cleaning generated models..."
 	@rm -rf $(OUTPUT_DIR) htmlcov .pytest_cache coverage.xml
+	@rm -rf $(SCHEMAS_DIR)
 	@echo "Clean complete!"
 
 # Show help
diff --git a/src/python-schema-generator/README.md b/src/python-schema-generator/README.md
@@ -104,7 +104,7 @@ if event_data.type == "uk.nhs.notify.digital.letters.letter.available.v1":
 
 The generator consists of:
 
-- `scripts/generate_models.py` - Main CLI script
-- `scripts/schema_processor.py` - Schema loading and processing
-- `scripts/model_generator.py` - Pydantic model generation logic
-- `scripts/file_utils.py` - File system utilities
+- `src/generate_models.py` - Main CLI script
+- `src/schema_processor.py` - Schema loading and processing
+- `src/model_generator.py` - Pydantic model generation logic
+- `src/file_utils.py` - File system utilities
diff --git a/src/python-schema-generator/jest.config.ts b/src/python-schema-generator/jest.config.ts
@@ -0,0 +1,11 @@
+import { baseJestConfig } from '../../jest.config.base';
+
+const config = {
+  ...baseJestConfig,
+  coveragePathIgnorePatterns: [
+    ...(baseJestConfig.coveragePathIgnorePatterns ?? []),
+    'src/merge-allof-cli.ts',
+  ],
+};
+
+export default config;
diff --git a/src/python-schema-generator/package.json b/src/python-schema-generator/package.json
@@ -0,0 +1,26 @@
+{
+  "dependencies": {
+    "json-schema-merge-allof": "^0.8.1",
+    "utils": "^0.0.1"
+  },
+  "description": "A tool to generate Python classes for the application's event schemas.",
+  "devDependencies": {
+    "@tsconfig/node22": "^22.0.5",
+    "@types/jest": "^29.5.14",
+    "@types/json-schema-merge-allof": "^0.6.5",
+    "@types/node": "^25.0.2",
+    "eslint": "^9.39.2",
+    "jest": "^29.7.0",
+    "mock-fs": "^5.5.0"
+  },
+  "name": "python-schema-generator",
+  "private": true,
+  "scripts": {
+    "lint": "eslint scripts",
+    "lint:fix": "eslint scripts --fix",
+    "merge": "tsx src/merge-allof-cli.ts",
+    "test:unit": "jest",
+    "typecheck": "tsc --noEmit"
+  },
+  "version": "1.0.0"
+}
diff --git a/src/python-schema-generator/src/__init__.py b/src/python-schema-generator/src/__init__.py
diff --git a/src/python-schema-generator/src/__tests__/merge-allof.test.ts b/src/python-schema-generator/src/__tests__/merge-allof.test.ts
@@ -0,0 +1,45 @@
+/* eslint-disable security/detect-non-literal-fs-filename */
+
+import { eventSchemasDir } from 'file-utils';
+import { mergeAllOfInSchemas } from 'merge-allof';
+import mockFs from 'mock-fs';
+import { readdirSync } from 'node:fs';
+import path from 'node:path';
+
+jest.mock('json-schema-to-typescript');
+
+describe('merge-allof', () => {
+  const outputDir = path.resolve(__dirname, '..', 'schemas');
+
+  beforeEach(() => {
+    mockFs({
+      [eventSchemasDir]: {
+        'one.flattened.schema.json': '{"title": "One"}',
+        'two.flattened.schema.json': '{"title": "Two"}',
+        'three.flattened.schema.json': '{"title": "Three"}',
+      },
+    });
+
+    jest.spyOn(console, 'log').mockImplementation(() => {});
+    jest.spyOn(console, 'group').mockImplementation(() => {});
+  });
+
+  afterEach(() => {
+    mockFs.restore();
+  });
+
+  it('should generate a schema with merged allOfs for each schema', () => {
+    mergeAllOfInSchemas();
+
+    const mergedSchemas = readdirSync(outputDir);
+
+    expect(mergedSchemas.length).toBe(3);
+    expect(mergedSchemas).toEqual(
+      expect.arrayContaining([
+        'one.flattened.schema.json',
+        'two.flattened.schema.json',
+        'three.flattened.schema.json',
+      ]),
+    );
+  });
+});
diff --git a/src/python-schema-generator/src/file-utils.ts b/src/python-schema-generator/src/file-utils.ts
@@ -0,0 +1,68 @@
+// We don't accept user input, so path traversal attacks should not be a risk.
+/* eslint-disable security/detect-non-literal-fs-filename */
+
+import { mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+
+export const eventSchemasDir = path.resolve(
+  __dirname,
+  '..',
+  '..',
+  '..',
+  'schemas',
+  'digital-letters',
+  '2025-10-draft',
+  'events',
+);
+
+export const destinationPackageName = 'digital-letters-events';
+
+/**
+ * Lists all event schema filenames in the digital letters schemas directory.
+ *
+ * @returns An array of schema filenames.
+ */
+export function listEventSchemas(): string[] {
+  const flattenedSchemaFiles = readdirSync(eventSchemasDir).filter((f) =>
+    f.endsWith('.flattened.schema.json'),
+  );
+
+  return flattenedSchemaFiles;
+}
+
+/**
+ * Loads and parses a JSON schema from the specified file path.
+ *
+ * @param schemaPath The path to the JSON schema file.
+ * @returns The parsed JSON schema object.
+ */
+export function loadSchema(schemaPath: string): any {
+  return JSON.parse(readFileSync(schemaPath, 'utf8'));
+}
+
+/**
+ * Creates the specified output directory if it doesn't exist.
+ *
+ * @param dirName The name of the directory to create.
+ * @returns The resolved path to the created directory.
+ */
+export function createOutputDir(dirName: string): string {
+  const outputDir = path.resolve(__dirname, dirName);
+  mkdirSync(outputDir, { recursive: true });
+  return outputDir;
+}
+
+/**
+ * Write a file with the specified content to the given path and filename.
+ *
+ * @param outputDir The directory the file will be written to.
+ * @param fileName The name of the file to write.
+ * @param content The content to write to the file.
+ */
+export function writeFile(
+  outputDir: string,
+  fileName: string,
+  content: string,
+): void {
+  writeFileSync(path.join(outputDir, fileName), content);
+}
diff --git a/src/python-schema-generator/src/file_utils.py b/src/python-schema-generator/src/file_utils.py
diff --git a/src/python-schema-generator/src/generate_models.py b/src/python-schema-generator/src/generate_models.py
diff --git a/src/python-schema-generator/src/merge-allof-cli.ts b/src/python-schema-generator/src/merge-allof-cli.ts
@@ -0,0 +1,3 @@
+import { mergeAllOfInSchemas } from 'merge-allof';
+
+mergeAllOfInSchemas();
diff --git a/src/python-schema-generator/src/merge-allof.ts b/src/python-schema-generator/src/merge-allof.ts
@@ -0,0 +1,34 @@
+/* eslint-disable no-console */
+import {
+  createOutputDir,
+  eventSchemasDir,
+  listEventSchemas,
+  loadSchema,
+  writeFile,
+} from 'file-utils';
+import mergeAllOf from 'json-schema-merge-allof';
+import path from 'node:path';
+
+export function mergeAllOfInSchemas(): void {
+  const eventSchemaFilenames = listEventSchemas();
+
+  const outputDir = createOutputDir('schemas');
+
+  console.log(`Output directory created at ${outputDir}`);
+
+  console.group('Merging allOf entries in schemas:');
+  for (const eventSchemaFilename of eventSchemaFilenames) {
+    const eventSchemaPath = path.join(eventSchemasDir, eventSchemaFilename);
+    const eventSchema = loadSchema(eventSchemaPath);
+
+    const merged = mergeAllOf(eventSchema, {
+      resolvers: {
+        defaultResolver: mergeAllOf.options.resolvers.title,
+      },
+    });
+
+    writeFile(outputDir, eventSchemaFilename, JSON.stringify(merged, null, 2));
+    console.log(eventSchemaFilename);
+  }
+  console.groupEnd();
+}
diff --git a/src/python-schema-generator/src/model_generator.py b/src/python-schema-generator/src/model_generator.py
diff --git a/src/python-schema-generator/src/schema_processor.py b/src/python-schema-generator/src/schema_processor.py
diff --git a/src/python-schema-generator/tests/test_file_utils.py b/src/python-schema-generator/tests/test_file_utils.py
@@ -4,7 +4,7 @@
 
 import pytest
 
-from scripts.file_utils import (
+from src.file_utils import (
     list_json_schemas,
     load_json_schema,
     write_init_file,
diff --git a/src/python-schema-generator/tests/test_model_generator.py b/src/python-schema-generator/tests/test_model_generator.py
@@ -4,13 +4,13 @@
 
 import pytest
 
-from scripts.model_generator import generate_pydantic_model
+from src.model_generator import generate_pydantic_model
 
 
 class TestGeneratePydanticModel:
     """Tests for generate_pydantic_model function."""
 
-    @patch("scripts.model_generator.subprocess.run")
+    @patch("src.model_generator.subprocess.run")
     def test_calls_datamodel_codegen_with_expected_arguments(self, mock_run):
         """Test successful model generation."""
         # Arrange
@@ -32,7 +32,7 @@ def test_calls_datamodel_codegen_with_expected_arguments(self, mock_run):
         assert "--class-name" in cmd_args[5]
         assert "TestModel" in cmd_args[6]
 
-    @patch("scripts.model_generator.subprocess.run")
+    @patch("src.model_generator.subprocess.run")
     def test_raises_error_on_generation_failure(self, mock_run):
         """Test that it raises error when generation fails."""
         schema_path = "test_model.json"
diff --git a/src/python-schema-generator/tests/test_schema_processor.py b/src/python-schema-generator/tests/test_schema_processor.py
@@ -2,7 +2,7 @@
 
 import pytest
 
-from scripts.schema_processor import (
+from src.schema_processor import (
     extract_event_type,
     extract_model_name,
 )
diff --git a/src/python-schema-generator/tsconfig.json b/src/python-schema-generator/tsconfig.json
@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "baseUrl": "./src/",
+    "isolatedModules": true,
+    "outDir": "dist"
+  },
+  "exclude": [
+    "node_modules"
+  ],
+  "extends": "@tsconfig/node22/tsconfig.json",
+  "include": [
+    "src/**/*",
+    "./jest.config.ts"
+  ]
+}

Original file line number	Diff line number	Diff line change
`@@ -61,6 +61,7 @@`
`61`	`61`	`"utils/sender-management",`
`62`	`62`	`"src/cloudevents",`
`63`	`63`	`"src/digital-letters-events",`
	`64`	`+ "src/python-schema-generator",`
`64`	`65`	`"src/typescript-schema-generator",`
`65`	`66`	`"tests/playwright"`
`66`	`67`	`]`