Skip to content

Commit 8db28fc

Browse files
authored
Move GeoJSON source schema to correct dir (#801)
The root problem is that our TS-type-generation tool, jsonschema-to-typescript, expects $ref paths to be relative to the schema root directory, while our Python-type-generation tool, datamodel-codegen, expects $ref paths to be relative to the file containing the $ref. To work around this, I'm adding a pre-processing step to the Python type generation code which converts the paths to look like datamodel-codegen expects.
1 parent 56d6352 commit 8db28fc

File tree

7 files changed

+122
-4
lines changed

7 files changed

+122
-4
lines changed

.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,8 @@ untitled*
141141

142142
# GeoJSON schema
143143
packages/schema/src/schema/geojson.json
144+
# Schema pre-processing
145+
packages/schema/temp-schema
144146

145147
# Automatically generated file for processing
146148

packages/schema/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
"build:schema": "jlpm run build:processing && node ./cacheGeoJSONSchema.js && jlpm build:schema:js && jlpm build:schema:py",
3030
"build:processing": "python scripts/process.py",
3131
"build:schema:js": "echo 'Generating TypeScript types from schema...' && json2ts -i src/schema -o src/_interface --no-unknownAny --unreachableDefinitions --cwd ./src/schema && cd src/schema && node ../../schema.js",
32-
"build:schema:py": "echo 'Generating Python types from schema...' && datamodel-codegen --input ./src/schema --output ../../python/jupytergis_core/jupytergis_core/schema/interfaces --output-model-type pydantic_v2.BaseModel --input-file-type jsonschema",
32+
"build:schema:py": "echo 'Generating Python types from schema...' && node scripts/preprocess-schemas-for-python-type-generation.js && datamodel-codegen --input ./temp-schema --output ../../python/jupytergis_core/jupytergis_core/schema/interfaces --output-model-type pydantic_v2.BaseModel --input-file-type jsonschema && rm -rf ./temp-schema",
3333
"build:prod": "jlpm run clean && jlpm build:schema && jlpm run build:lib",
3434
"build:lib": "tsc -b",
3535
"build:dev": "jlpm run build",
Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
#!/usr/bin/env node
2+
3+
/**
4+
* Preprocesses JSON schemas for Python type generation by replacing
5+
* $ref paths which are relative to the schema root directory with paths which
6+
* are relative to the file containing the $ref.
7+
*
8+
* This is a difference between how jsonschema-to-typescript and
9+
* datamodel-codegen process $refs; I believe there is no way to write a $ref
10+
* containing a path which is compatible with both unless our schemas are all
11+
* in a flat directory structure.
12+
*/
13+
14+
const fs = require('fs');
15+
const path = require('path');
16+
17+
const schemaRoot = path.join(__dirname, '..', 'src', 'schema');
18+
const tempDir = path.join(__dirname, '..', 'temp-schema');
19+
20+
/*
21+
* Rewrite `refValue`, if it contains a path, to be relative to `schemaDir`
22+
* instead of `schemaRoot`.
23+
*/
24+
function updateRefPath(refValue, schemaDir, schemaRoot) {
25+
// Handle $ref with optional fragment (e.g., "path/to/file.json#/definitions/something")
26+
const [refPath, fragment] = refValue.split('#');
27+
28+
// Check if the referenced file exists
29+
const absoluteRefPath = path.resolve(schemaRoot, refPath);
30+
if (!fs.existsSync(absoluteRefPath)) {
31+
throw new Error(`Referenced file does not exist: ${refPath} (resolved to ${absoluteRefPath})`);
32+
}
33+
34+
// Convert schemaRoot-relative path to schemaDir-relative path
35+
const relativeToCurrentDir = path.relative(schemaDir, absoluteRefPath);
36+
37+
// Just in case we're on Windows, replace backslashes.
38+
const newRef = relativeToCurrentDir.replace(/\\/g, '/');
39+
40+
return fragment ? `${newRef}#${fragment}` : newRef;
41+
}
42+
43+
/*
44+
* Recursively process `schema` (JSON content) to rewrite `$ref`s containing paths.
45+
*
46+
* Any path will be modified to be relative to `schemaDir` instead of relative
47+
* to `schemaRoot`.
48+
*/
49+
function processSchema(schema, schemaDir, schemaRoot) {
50+
if (Array.isArray(schema)) {
51+
// Recurse!
52+
return schema.map(item => processSchema(item, schemaDir, schemaRoot));
53+
}
54+
55+
if (Object.prototype.toString.call(schema) !== '[object Object]') {
56+
return schema;
57+
}
58+
59+
// `schema` is an "object":
60+
const result = {};
61+
for (const [key, value] of Object.entries(schema)) {
62+
if (key === '$ref' && typeof value === 'string' && !value.startsWith('#')) {
63+
result[key] = updateRefPath(value, schemaDir, schemaRoot);
64+
} else {
65+
// Recurse!
66+
result[key] = processSchema(value, schemaDir, schemaRoot);
67+
}
68+
}
69+
70+
return result;
71+
}
72+
73+
/*
74+
* Recursively rewrite schema files in `src` to `dest`.
75+
*
76+
* For each schema, rewrite the paths in JSONSchema `$ref`s to be relative to
77+
* that schema's parent directory instead of `schemaRoot`.
78+
*/
79+
function preProcessSchemaDirectory(src, dest, schemaRoot) {
80+
if (!fs.existsSync(dest)) {
81+
fs.mkdirSync(dest, { recursive: true });
82+
}
83+
84+
const children = fs.readdirSync(src, { withFileTypes: true });
85+
86+
for (const child of children) {
87+
const srcChild = path.join(src, child.name);
88+
const destChild = path.join(dest, child.name);
89+
90+
if (child.isDirectory()) {
91+
// Recurse!
92+
preProcessSchemaDirectory(srcChild, destChild, schemaRoot);
93+
} else if (child.isFile() && child.name.endsWith('.json')) {
94+
// Process schema JSON to modify $ref paths
95+
const content = fs.readFileSync(srcChild, 'utf8');
96+
const schema = JSON.parse(content);
97+
const processedSchema = processSchema(schema, src, schemaRoot);
98+
99+
fs.writeFileSync(destChild, JSON.stringify(processedSchema, null, 2));
100+
} else {
101+
// There should be no non-JSON files in the schema directory!
102+
throw new Error(`Non-JSON file detected in schema directory: ${child.parentPath}/${child.name}`);
103+
}
104+
}
105+
}
106+
107+
function preProcessSchemas() {
108+
fs.rmSync(tempDir, { recursive: true, force: true });
109+
preProcessSchemaDirectory(schemaRoot, tempDir, schemaRoot);
110+
}
111+
112+
console.log(`Pre-processing JSONSchemas for Python type generation (writing to ${tempDir})...`)
113+
114+
preProcessSchemas();
115+
116+
console.log('Schemas pre-processed for Python type generation.');

packages/schema/src/interfaces.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ import {
2727
SourceType,
2828
} from './_interface/project/jgis';
2929
import { IRasterSource } from './_interface/project/sources/rasterSource';
30-
export { IGeoJSONSource } from './_interface/geoJsonSource';
30+
export { IGeoJSONSource } from './_interface/project/sources/geoJsonSource';
3131

3232
export type JgisCoordinates = { x: number; y: number };
3333

packages/schema/src/types.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
export * from './_interface/project/jgis';
22

33
// Sources
4-
export * from './_interface/geoJsonSource';
4+
export * from './_interface/project/sources/geoJsonSource';
55
export * from './_interface/project/sources/geoTiffSource';
66
export * from './_interface/project/sources/imageSource';
77
export * from './_interface/project/sources/rasterDemSource';

python/jupytergis_core/jupytergis_core/schema/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
from .interfaces.project.sources.vectorTileSource import IVectorTileSource # noqa
1212
from .interfaces.project.sources.rasterSource import IRasterSource # noqa
13-
from .interfaces.geoJsonSource import IGeoJSONSource # noqa
13+
from .interfaces.project.sources.geoJsonSource import IGeoJSONSource # noqa
1414
from .interfaces.project.sources.videoSource import IVideoSource # noqa
1515
from .interfaces.project.sources.imageSource import IImageSource # noqa
1616
from .interfaces.project.sources.geoTiffSource import IGeoTiffSource # noqa

0 commit comments

Comments
 (0)