Skip to content

Commit d3849dd

Browse files
acrodrigacrodrig
committed
Refactoring out all of Schema generation
1 parent 3eca6da commit d3849dd

File tree

6 files changed

+276
-252
lines changed

6 files changed

+276
-252
lines changed

‎src/ddl.ts‎

Lines changed: 2 additions & 169 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import { eTag } from "@std/http/etag";
21
import { hub } from "hub";
32
import type { Column, Constraint, Index, Relation, Schema } from "./types.ts";
43
import DB from "./db.ts";
@@ -21,20 +20,8 @@ const serialType = {
2120
postgres: "",
2221
};
2322

24-
const _BaseSchema: DB.Schema = {
25-
table: "_BaseSchema",
26-
properties: {
27-
id: { type: "integer", primaryKey: true, description: "Unique identifier, auto-generated. It's the primary key." },
28-
etag: { type: "string", maxLength: 1024, description: "Possible ETag for all resources that are external. Allows for better synch-ing." },
29-
inserted: { type: "date", dateOn: "insert", index: ["inserted"], description: "Timestamp when current record is inserted" },
30-
updated: { type: "date", dateOn: "update", index: ["updated"], description: "Timestamp when current record is updated" },
31-
},
32-
};
33-
3423
export class DDL {
3524
static EXTENSIONS = ["as", "constraint", "dateOn", "fullText", "index", "primaryKey", "relations", "unique", "table"];
36-
static TS_OPTIONS = { lib: ["es2022"], module: "es2022", target: "es2022" };
37-
static TJS_OPTIONS = { required: true, ignoreErrors: true, defaultNumberType: "integer", validationKeywords: DDL.EXTENSIONS };
3825

3926
static padWidth = 4;
4027
static defaultWidth = 128;
@@ -49,160 +36,6 @@ export class DDL {
4936
if (!values.includes(provider)) throw new Error(message);
5037
}
5138

52-
/**
53-
* Generator function that creates a map of schemas from class files
54-
* @param classFiles - a map of class names to file paths
55-
* @param base - the base directory where the files are located
56-
* @param extensions - additional extensions to be used by the generator
57-
* @example
58-
*
59-
* Below is an example of how to define the generator function using :
60-
*
61-
* ```ts
62-
* DDL.generator = async function(classFiles: Record<string, string>, base?: string) {
63-
* const TJS = (await import("npm:typescript-json-schema@0.65.1")).default;
64-
* const program = TJS.getProgramFromFiles(Object.values(classFiles), DDL.TS_OPTIONS, base);
65-
* const entries = Object.keys(classFiles).map((c) => [c, TJS.generateSchema(program, c, DDL.TJS_OPTIONS)]);
66-
* return Object.fromEntries(entries);
67-
* };
68-
* ```
69-
*/
70-
static generator: (classFiles: Record<string, string>, base?: string, extensions?: string[]) => Promise<Record<string, Schema>>;
71-
72-
static async ensureSchemas(
73-
schemas: Record<string, Schema>,
74-
classFiles: Record<string, string>,
75-
base?: string,
76-
enhance = false,
77-
schemasFile?: string,
78-
): Promise<Record<string, Schema>> {
79-
const outdated = !schemas ? undefined : await DDL.outdatedSchemas(schemas, base);
80-
if (!outdated || outdated.length > 0) return schemas;
81-
82-
// Generate and save
83-
schemas = await DDL.generateSchemas(classFiles, base, enhance);
84-
if (schemasFile) await Deno.writeTextFile(schemasFile, JSON.stringify(schemas, null, 2));
85-
return schemas;
86-
}
87-
88-
/**
89-
* Generate schemas from class files
90-
*
91-
* @param classFiles - a map of class names to file paths
92-
* @param base - the base directory where the files are located, needed for relative URLs in schema
93-
* @param enhance - if true schemas will be enhanced with standard properties
94-
* @returns a map of class names to schemas
95-
*/
96-
static async generateSchemas(classFiles: Record<string, string>, base?: string, enhance?: boolean): Promise<Record<string, Schema>> {
97-
log.debug({ method: "generateSchemas", classFiles, base, enhance });
98-
99-
// If DDL has no generator, throw an error
100-
if (!DDL.generator) throw new Error("DDL.generator must be set to a function that generates schemas from class files");
101-
102-
// Generate schemas and clean them and enhance them
103-
const schemas = await DDL.generator(classFiles, base);
104-
for (const [c, f] of Object.entries(classFiles)) {
105-
const etag = await eTag(await Deno.stat(f));
106-
const file = f.startsWith("/") ? f : "./" + f;
107-
schemas[c] = DDL.#cleanSchema(schemas[c], c, undefined, "file://" + file, etag);
108-
if (enhance) schemas[c] = DDL.enhanceSchema(schemas[c]);
109-
}
110-
111-
// Return schema map (from class/type to schema)
112-
return schemas;
113-
}
114-
115-
/**
116-
* When using tools such as [TJS](https://github.com/YousefED/typescript-json-schema) to
117-
* generate JSON schemas from TypeScript classes, the resulting schema may need some
118-
* cleaning up. This function does that.
119-
*
120-
* @param schema - the tool-generated schema
121-
* @param type - the type of the schema which we may need to correct/override
122-
* @param table - the table name which we may need to correct/override
123-
* @param $id - the URL of the schema file
124-
* @param etag - the etag of the source file
125-
*/
126-
static #cleanSchema(schema: Schema, type?: string, table?: string, $id?: string, etag?: string): Schema {
127-
if (type) schema.type = type;
128-
if (table) schema.table = table;
129-
130-
// By default the table name is the same as the type name
131-
if (!schema.table) schema.table = type?.toLowerCase() ?? schema.type?.toLowerCase();
132-
133-
// Set $id to the file URL of the schema and the date time it was created (as the hash)
134-
if ($id) schema.$id = $id + ($id.includes("#") ? "" : "#" + new Date().toISOString().substring(0, 19));
135-
136-
// Generate an etag (based on the file etag)
137-
if (etag) schema.etag = etag;
138-
139-
if (typeof (schema.fullText) === "string") schema.fullText = (schema.fullText as string).split(",").map((s) => s.trim());
140-
Object.entries(schema.properties).forEach(([n, c]) => {
141-
// If 'description' spans multiple lines, use the first line as the description
142-
if (c.description?.includes("\n")) c.description = c.description.split("\n")[0];
143-
144-
// If there is no type, assume it is a string
145-
if (!c.type) c.type = "string";
146-
147-
// Make primary key and uniqye attributes boolean
148-
if (typeof c.primaryKey === "string") c.primaryKey = true;
149-
if (typeof c.unique === "string") c.unique = true;
150-
151-
// Use the format as a way to discover a date type
152-
if (c.format === "date-time") c.type = "date";
153-
154-
// Build the index into a proper string array
155-
if (typeof c.index === "string") c.index = (c.index ? c.index : n).split(",").map((s) => s.trim());
156-
});
157-
return schema;
158-
}
159-
160-
static async outdatedSchemas(schemas: Record<string, Schema>, base = ""): Promise<string[]> {
161-
const outdated: string[] = [];
162-
for (const [c, s] of Object.entries(schemas)) {
163-
if (!(await DDL.outdatedSchema(s, base))) continue;
164-
outdated.push(c);
165-
}
166-
return outdated;
167-
}
168-
169-
/**
170-
* Check if the schema is outdated by comparing the etag with the content etag
171-
* Returns the file if it is outdated, or undefined if it is not.
172-
* @param schema - the schema to check
173-
* @param base - the directory where the schema file is located
174-
*/
175-
static async outdatedSchema(schema: Schema, base = ""): Promise<boolean> {
176-
if (!base.startsWith("/")) throw new Error("Base must be absolute within the system");
177-
if (!schema.$id) throw new Error("Schema must have an '$id' property to test if it is outdated");
178-
179-
// Get file and schema create date from $id
180-
const url = new URL(schema.$id);
181-
const file = ((schema.$id.startsWith("file://./") ? base : "") + url.pathname).replace(/\/\//g, "/");
182-
const fileInfo = await Deno.stat(file);
183-
const schemaDate = url.hash.substring(1);
184-
185-
// First compare dates
186-
const fileDate = fileInfo.mtime!.toISOString().substring(0, 19);
187-
// console.log(url.pathname, " --- ", schemaDate, " : ", fileDate, " -> ", schemaDate < fileInfo.mtime!.toISOString());
188-
if (schemaDate < fileDate) return true;
189-
190-
// If the date comparison is not enough to tell, then compare etags
191-
const etag = await eTag(await Deno.stat(file));
192-
return schema.etag !== etag;
193-
}
194-
195-
// Enhance schema with standard properties
196-
static enhanceSchema(schema: Schema, selected: string[] = ["id", "inserted", "updated"]): Schema {
197-
// Select properties that match the selected columns and add them to the schema
198-
if (!schema.properties) schema.properties = {};
199-
for (const name of selected) {
200-
if (schema.properties[name]) continue;
201-
schema.properties[name] = _BaseSchema.properties[name];
202-
}
203-
return schema;
204-
}
205-
20639
static #defaultValue(column: Column, provider: string) {
20740
const cd = column.default;
20841

@@ -322,7 +155,7 @@ export class DDL {
322155
}
323156

324157
// Uses the most standard MySQL syntax, and then it is fixed afterward
325-
static createTable(schema: Schema, provider: DB.Provider, nameOverride?: string): string {
158+
static createTable(schema: Schema, provider: DB.Provider, nameOverride?: string, safe?: boolean): string {
326159
log.debug({ method: "createTable", schema, provider, nameOverride });
327160
this.#ensureProvider(provider);
328161

@@ -345,7 +178,7 @@ export class DDL {
345178
const constraints = [...columnConstraints, ...independentConstraints].sort().join("");
346179

347180
// Create sql
348-
let sql = `CREATE TABLE IF NOT EXISTS ${table} (\n${columns}${relations}${constraints})`;
181+
let sql = `CREATE TABLE ${safe ? " IF NOT EXISTS" : ""}${table} (\n${columns}${relations}${constraints})`;
349182

350183
// Independent indexes (and sort lines for consistency)
351184
const indices = schema.indices?.slice() ?? [];

‎src/schemas.ts‎

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
import { eTag } from "@std/http/etag";
2+
import { hub } from "hub";
3+
import { DDL } from "./ddl.ts";
4+
import type { Schema } from "./types.ts";
5+
6+
const log = hub("dbx:schema");
7+
8+
const _BaseSchema: Schema = {
9+
table: "_BaseSchema",
10+
properties: {
11+
id: { type: "integer", primaryKey: true, description: "Unique identifier, auto-generated. It's the primary key." },
12+
etag: { type: "string", maxLength: 1024, description: "Possible ETag for all resources that are external. Allows for better synch-ing." },
13+
inserted: { type: "date", dateOn: "insert", index: ["inserted"], description: "Timestamp when current record is inserted" },
14+
updated: { type: "date", dateOn: "update", index: ["updated"], description: "Timestamp when current record is updated" },
15+
},
16+
};
17+
18+
/**
19+
* Generator function that creates a map of schemas from class files
20+
* @param classFiles - a map of class names to file paths
21+
* @param base - the base directory where the files are located
22+
* @param extensions - additional extensions to be used by the generator
23+
* @example
24+
*
25+
* Below is an example of how to define the generator function using :
26+
*
27+
* ```ts
28+
* async function generator (classFiles: Record<string, string>, base?: string) {
29+
* const TJS = (await import("npm:typescript-json-schema@0.65.1")).default;
30+
* const program = TJS.getProgramFromFiles(Object.values(classFiles), Generator.TS_OPTIONS, base);
31+
* const entries = Object.keys(classFiles).map((c) => [c, TJS.generateSchema(program, c, Generator.TJS_OPTIONS)]);
32+
* return Object.fromEntries(entries);
33+
* };
34+
* ```
35+
*/
36+
37+
type Generator = (classFiles: Record<string, string>, base?: string, extensions?: string[]) => Promise<Record<string, Schema>>;
38+
39+
export class Schemas {
40+
static TS_OPTIONS = { lib: ["es2022"], module: "es2022", target: "es2022" };
41+
static TJS_OPTIONS = { required: true, ignoreErrors: true, defaultNumberType: "integer", validationKeywords: DDL.EXTENSIONS };
42+
43+
/**
44+
* When using tools such as [TJS](https://github.com/YousefED/typescript-json-schema) to
45+
* generate JSON schemas from TypeScript classes, the resulting schema may need some
46+
* cleaning up. This function does that.
47+
*
48+
* @param schema - the tool-generated schema
49+
* @param type - the type of the schema which we may need to correct/override
50+
* @param table - the table name which we may need to correct/override
51+
* @param $id - the URL of the schema file
52+
* @param etag - the etag of the source file
53+
*/
54+
static #clean(schema: Schema, type?: string, table?: string, $id?: string, etag?: string): Schema {
55+
if (type) schema.type = type;
56+
if (table) schema.table = table;
57+
58+
// By default the table name is the same as the type name
59+
if (!schema.table) schema.table = type?.toLowerCase() ?? schema.type?.toLowerCase();
60+
61+
// Set $id to the file URL of the schema and the date time it was created (as the hash)
62+
if ($id) schema.$id = $id + ($id.includes("#") ? "" : "#" + new Date().toISOString().substring(0, 19));
63+
64+
// Generate an etag (based on the file etag)
65+
if (etag) schema.etag = etag;
66+
67+
if (typeof (schema.fullText) === "string") schema.fullText = (schema.fullText as string).split(",").map((s) => s.trim());
68+
Object.entries(schema.properties).forEach(([n, c]) => {
69+
// If 'description' spans multiple lines, use the first line as the description
70+
if (c.description?.includes("\n")) c.description = c.description.split("\n")[0];
71+
72+
// If there is no type, assume it is a string
73+
if (!c.type) c.type = "string";
74+
75+
// Make primary key and uniqye attributes boolean
76+
if (typeof c.primaryKey === "string") c.primaryKey = true;
77+
if (typeof c.unique === "string") c.unique = true;
78+
79+
// Use the format as a way to discover a date type
80+
if (c.format === "date-time") c.type = "date";
81+
82+
// Build the index into a proper string array
83+
if (typeof c.index === "string") c.index = (c.index ? c.index : n).split(",").map((s) => s.trim());
84+
});
85+
return schema;
86+
}
87+
88+
static async ensure(
89+
schemas: Record<string, Schema>,
90+
classFiles: Record<string, string>,
91+
generator: Generator,
92+
base?: string,
93+
enhance = false,
94+
schemasFile?: string,
95+
): Promise<Record<string, Schema>> {
96+
const outdated = !schemas ? undefined : await Schemas.outdatedSchemas(schemas, base);
97+
if (!outdated || outdated.length > 0) return schemas;
98+
99+
// Generate and save
100+
schemas = await Schemas.generate(classFiles, generator, base, enhance);
101+
if (schemasFile) await Deno.writeTextFile(schemasFile, JSON.stringify(schemas, null, 2));
102+
return schemas;
103+
}
104+
105+
/**
106+
* Generate schemas from class files
107+
*
108+
* @param classFiles - a map of class names to file paths
109+
* @param base - the base directory where the files are located, needed for relative URLs in schema
110+
* @param enhance - if true schemas will be enhanced with standard properties
111+
* @returns a map of class names to schemas
112+
*/
113+
static async generate(classFiles: Record<string, string>, generator: Generator, base?: string, enhance?: boolean): Promise<Record<string, Schema>> {
114+
log.debug({ method: "generateSchemas", classFiles, base, enhance });
115+
116+
// If Generator has no generator, throw an error
117+
if (!generator) throw new Error("Generator must be set to a function that generates schemas from class files");
118+
119+
// Generate schemas and clean them and enhance them
120+
const schemas = await generator(classFiles, base);
121+
for (const [c, f] of Object.entries(classFiles)) {
122+
const etag = await eTag(await Deno.stat(f));
123+
const file = f.startsWith("/") ? f : "./" + f;
124+
schemas[c] = Schemas.#clean(schemas[c], c, undefined, "file://" + file, etag);
125+
if (enhance) schemas[c] = Schemas.enhanceSchema(schemas[c]);
126+
}
127+
128+
// Return schema map (from class/type to schema)
129+
return schemas;
130+
}
131+
132+
static async outdatedSchemas(schemas: Record<string, Schema>, base = ""): Promise<string[]> {
133+
const outdated: string[] = [];
134+
for (const [c, s] of Object.entries(schemas)) {
135+
if (!(await Schemas.outdated(s, base))) continue;
136+
outdated.push(c);
137+
}
138+
return outdated;
139+
}
140+
141+
/**
142+
* Check if the schema is outdated by comparing the etag with the content etag
143+
* Returns the file if it is outdated, or undefined if it is not.
144+
* @param schema - the schema to check
145+
* @param base - the directory where the schema file is located
146+
*/
147+
static async outdated(schema: Schema, base = ""): Promise<boolean> {
148+
if (!base.startsWith("/")) throw new Error("Base must be absolute within the system");
149+
if (!schema.$id) throw new Error("Schema must have an '$id' property to test if it is outdated");
150+
151+
// Get file and schema create date from $id
152+
const url = new URL(schema.$id);
153+
const file = ((schema.$id.startsWith("file://./") ? base : "") + url.pathname).replace(/\/\//g, "/");
154+
const fileInfo = await Deno.stat(file);
155+
const schemaDate = url.hash.substring(1);
156+
157+
// First compare dates
158+
const fileDate = fileInfo.mtime!.toISOString().substring(0, 19);
159+
// console.log(url.pathname, " --- ", schemaDate, " : ", fileDate, " -> ", schemaDate < fileInfo.mtime!.toISOString());
160+
if (schemaDate < fileDate) return true;
161+
162+
// If the date comparison is not enough to tell, then compare etags
163+
const etag = await eTag(await Deno.stat(file));
164+
return schema.etag !== etag;
165+
}
166+
167+
// Enhance schema with standard properties
168+
static enhanceSchema(schema: Schema, selected: string[] = ["id", "inserted", "updated"]): Schema {
169+
// Select properties that match the selected columns and add them to the schema
170+
if (!schema.properties) schema.properties = {};
171+
for (const name of selected) {
172+
if (schema.properties[name]) continue;
173+
schema.properties[name] = _BaseSchema.properties[name];
174+
}
175+
return schema;
176+
}
177+
}

0 commit comments

Comments
 (0)