Skip to content

Commit c6af37c

Browse files
committed
Add pre-processing script to enable all our code generation tools to work with $refs
The root problem is that our TS-type-generation tool, jsonschema-to-typescript, expects $ref paths to be relative to the schema root directory, while our Python-type-generation tool, datamodel-codegen, expects $ref paths to be relative to the file containing the $ref. To work around this, I'm adding a pre-processing step to the Python type generation code which converts the paths to look like datamodel-codegen expects.
1 parent d2c433c commit c6af37c

File tree

4 files changed

+128
-4
lines changed

4 files changed

+128
-4
lines changed

packages/schema/package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@
2828
"build": "jlpm build:schema && jlpm build:lib",
2929
"build:schema": "jlpm run build:processing && node ./cacheGeoJSONSchema.js && jlpm build:schema:js && jlpm build:schema:py",
3030
"build:processing": "python scripts/process.py",
31-
"build:schema:js": "json2ts -i src/schema -o src/_interface --no-unknownAny --unreachableDefinitions --cwd ./src/schema && cd src/schema && node ../../schema.js",
32-
"build:schema:py": "datamodel-codegen --input ./src/schema --output ../../python/jupytergis_core/jupytergis_core/schema/interfaces --output-model-type pydantic_v2.BaseModel --input-file-type jsonschema",
31+
"build:schema:js": "echo 'Generating TypeScript types from schema...' && json2ts -i src/schema -o src/_interface --no-unknownAny --unreachableDefinitions --cwd ./src/schema && cd src/schema && node ../../schema.js",
32+
"build:schema:py": "echo 'Generating Python types from schema...' && node scripts/preprocess-schemas-for-python-type-generation.js && datamodel-codegen --input ./temp-schema --output ../../python/jupytergis_core/jupytergis_core/schema/interfaces --output-model-type pydantic_v2.BaseModel --input-file-type jsonschema && rm -rf ./temp-schema",
3333
"build:prod": "jlpm run clean && jlpm build:schema && jlpm run build:lib",
3434
"build:lib": "tsc -b",
3535
"build:dev": "jlpm run build",
Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
#!/usr/bin/env node
2+
3+
/**
4+
* Preprocesses JSON schemas for Python type generation by replacing
5+
* $ref paths which are relative to the schema root directory with paths which
6+
* are relative to the file containing the $ref.
7+
*
8+
* This is a difference between how jsonschema-to-typescript and
9+
* datamodel-codegen process $refs; I believe there is no way to write a $ref
10+
* containing a path which is compatible with both unless our schemas are all
11+
* in a flat directory structure.
12+
*/
13+
14+
const fs = require('fs');
15+
const path = require('path');
16+
17+
const schemaRoot = path.join(__dirname, '..', 'src', 'schema');
18+
const tempDir = path.join(__dirname, '..', 'temp-schema');
19+
20+
/*
21+
* Rewrite `refValue`, if it contains a path, to be relative to `schemaDir`
22+
* instead of `schemaRoot`.
23+
*/
24+
function updateRefPath(refValue, schemaDir, schemaRoot) {
25+
// Handle $ref with optional fragment (e.g., "path/to/file.json#/definitions/something")
26+
const [refPath, fragment] = refValue.split('#');
27+
28+
// Check if the referenced file exists
29+
const absoluteRefPath = path.resolve(schemaRoot, refPath);
30+
if (!fs.existsSync(absoluteRefPath)) {
31+
throw new Error(`Referenced file does not exist: ${refPath} (resolved to ${absoluteRefPath})`);
32+
}
33+
34+
// Convert schemaRoot-relative path to schemaDir-relative path
35+
const relativeToCurrentDir = path.relative(schemaDir, absoluteRefPath);
36+
37+
// Just in case we're on Windows, replace backslashes.
38+
const newRef = relativeToCurrentDir.replace(/\\/g, '/');
39+
40+
return fragment ? `${newRef}#${fragment}` : newRef;
41+
}
42+
43+
/*
44+
* Recursively process `schema` (JSON content) to rewrite `$ref`s containing paths.
45+
*
46+
* Any path will be modified to be relative to `schemaDir` instead of relative
47+
* to `schemaRoot`.
48+
*/
49+
function processSchema(schema, schemaDir, schemaRoot) {
50+
if (Array.isArray(schema)) {
51+
// Recurse!
52+
return schema.map(item => processSchema(item, schemaDir, schemaRoot));
53+
}
54+
55+
if (Object.prototype.toString.call(schema) !== '[object Object]') {
56+
return schema;
57+
}
58+
59+
// `schema` is an "object":
60+
const result = {};
61+
for (const [key, value] of Object.entries(schema)) {
62+
if (key === '$ref' && typeof value === 'string' && !value.startsWith('#')) {
63+
result[key] = updateRefPath(value, schemaDir, schemaRoot);
64+
} else {
65+
// Recurse!
66+
result[key] = processSchema(value, schemaDir, schemaRoot);
67+
}
68+
}
69+
70+
return result;
71+
}
72+
73+
/*
74+
* Recursively rewrite schema files in `src` to `dest`.
75+
*
76+
* For each schema, rewrite the paths in JSONSchema `$ref`s to be relative to
77+
* that schema's parent directory instead of `schemaRoot`.
78+
*/
79+
function preProcessSchemaDirectory(src, dest, schemaRoot) {
80+
if (!fs.existsSync(dest)) {
81+
fs.mkdirSync(dest, { recursive: true });
82+
}
83+
84+
const children = fs.readdirSync(src, { withFileTypes: true });
85+
86+
for (const child of children) {
87+
const srcChild = path.join(src, child.name);
88+
const destChild = path.join(dest, child.name);
89+
90+
if (child.isDirectory()) {
91+
// Recurse!
92+
preProcessSchemaDirectory(srcChild, destChild, schemaRoot);
93+
} else if (child.isFile() && child.name.endsWith('.json')) {
94+
// Process schema JSON to modify $ref paths
95+
const content = fs.readFileSync(srcChild, 'utf8');
96+
const schema = JSON.parse(content);
97+
const processedSchema = processSchema(schema, src, schemaRoot);
98+
99+
fs.writeFileSync(destChild, JSON.stringify(processedSchema, null, 2));
100+
} else {
101+
// There should be no non-JSON files in the schema directory!
102+
throw new Error(`Non-JSON file detected in schema directory: ${child.parentPath}/${child.name}`);
103+
}
104+
}
105+
}
106+
107+
function preProcessSchemas() {
108+
fs.rmSync(tempDir, { recursive: true, force: true });
109+
preProcessSchemaDirectory(schemaRoot, tempDir, schemaRoot);
110+
}
111+
112+
console.log(`Pre-processing JSONSchemas for Python type generation (writing to ${tempDir})...`)
113+
114+
preProcessSchemas();
115+
116+
console.log('Schemas pre-processed for Python type generation.');

packages/schema/src/schema/project/layers/symbology/vectorColor.json

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
},
1111
"selectedAttribute": {
1212
"type": "string",
13-
"title": "Attribute",
1413
"description": "The selected attribute for varying the color"
1514
},
1615
"colorRamp": {

packages/schema/src/schema/project/layers/vectorLayer.json

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,16 @@
1919
},
2020
"symbology": {
2121
"type": "object",
22-
"description": "The user inputs for symbology options"
22+
"description": "The user inputs for symbology options",
23+
"additionalProperties": false,
24+
"properties": {
25+
"color": {
26+
"$ref": "project/layers/symbology/vectorColor.json"
27+
},
28+
"size": {
29+
"$ref": "project/layers/symbology/vectorSize.json"
30+
}
31+
}
2332
}
2433
}
2534
}

0 commit comments

Comments
 (0)