Better alignment with JSON Schema

Author: acrodrig

resources/account.json

@@ -1,26 +1,28 @@
 {
-    "name": "accounts",
+    "table": "accounts",
     "type": "Account",
     "properties": {
-        "id":          { "type": "integer", "required": true,  "primaryKey": true, "description": "Unique identifier, auto-generated. It's the primary key." },
-        "inserted":    { "type": "date",    "required": false, "dateOn": "insert", "description": "Timestamp when current record is inserted" },
-        "updated":     { "type": "date",    "required": false, "dateOn": "update", "description": "Timestamp when current record is updated" },
-        "etag":        { "type": "string",  "required": false, "maxLength": 1024, "description": "Possible ETag for all resources that are external. Allows for better synch-ing." },
-        "comments":    { "type": "string",  "required": false, "maxLength": 8192, "fullText": true, "description": "General comments. Can be used for anything useful related to the instance." },
-        "country":     { "type": "string",  "required": true,  "maxLength": 16, "default": "'US'", "fullText": true, "description": "Country code" },
-        "email":       { "type": "string",  "required": false, "maxLength": 128, "unique": true, "description": "Main email to communicate for that account" },
-        "established": { "type": "date",    "required": false, "maxLength": 6, "minimum": "2020-01-01", "description": "Date on which the account was established" },
-        "enabled":     { "type": "boolean", "required": true,  "default": true, "description": "Whether it is enabled or not. Disabled instances will not be used." },
-        "externalId":  { "type": "string",  "required": false, "maxLength": 512, "unique": true, "description": "External unique ID, used to refer to external accounts" },
-        "phone":       { "type": "string",  "required": false, "maxLength": 128, "fullText": true, "description": "Handle associated with the account" },
-        "name":        { "type": "string",  "required": true,  "unique": true, "fullText": true, "description": "Descriptive name to identify the instance" },
-        "preferences": { "type": "json",    "required": true,  "default": { "wrap": true, "minAge": 18 }, "description": "All the general options associated with the account." },
-        "valueList":   { "type": "json",    "asExpression": "JSON_EXTRACT(preferences, '$.*')", "generatedType": "stored" }
+        "id":          { "type": "integer", "primaryKey": true, "description": "Unique identifier, auto-generated. It's the primary key." },
+        "inserted":    { "type": "date",    "dateOn": "insert", "description": "Timestamp when current record is inserted" },
+        "updated":     { "type": "date",    "dateOn": "update", "description": "Timestamp when current record is updated" },
+        "etag":        { "type": "string",  "description": "Possible ETag for all resources that are external. Allows for better synch-ing." },
+        "comments":    { "type": "string",  "maxLength": 8192, "description": "General comments. Can be used for anything useful related to the instance." },
+        "country":     { "type": "string",  "default": "'US'", "description": "Country code" },
+        "email":       { "type": "string",  "uniqueItems": true, "description": "Main email to communicate for that account" },
+        "established": { "type": "date",    "maxLength": 6, "minimum": "2020-01-01", "description": "Date on which the account was established" },
+        "enabled":     { "type": "boolean", "default": true, "description": "Whether it is enabled or not. Disabled instances will not be used." },
+        "externalId":  { "type": "string",  "maxLength": 512, "uniqueItems": true, "description": "External unique ID, used to refer to external accounts" },
+        "phone":       { "type": "string",  "description": "Handle associated with the account" },
+        "name":        { "type": "string",  "uniqueItems": true, "description": "Descriptive name to identify the instance" },
+        "preferences": { "type": "object",  "default": { "wrap": true, "minAge": 18 }, "description": "All the general options associated with the account." },
+        "valueList":   { "type": "object",  "as": "JSON_EXTRACT(preferences, '$.*')", "description": "Auto-generated field with values" }
     },
+    "required": ["id", "country", "enabled", "name", "preferences"],
+    "fullText": ["comments", "country", "phone", "name"],
     "indices": [
-        { "name": "inserted", "properties": ["inserted"] },
-        { "name": "updated", "properties": ["updated"] },
-        { "name": "valueList", "properties": ["id", "valueList", "enabled"], "array": 1 }
+        { "properties": ["inserted"] },
+        { "properties": ["updated"] },
+        { "properties": ["id", "valueList", "enabled"], "array": 1 }
     ],
     "constraints": [
         { "name": "email", "check": "email IS NULL OR email RLIKE '^[^@]+@[^@]+[.][^@]{2,}$'" },

resources/account.ts

@@ -1,21 +1,54 @@
+/**
+ * Accounts class definition
+ * @table accounts
+ * @fullText comments, country, phone, name
+ */
 export default class Account {
+  /** Unique identifier, auto-generated. It's the primary key. @primaryKey */
+  id!: number;
+
+  /** General comments. Can be used for anything useful related to the instance. @maxLength 8192  */
+  comments?: string;
+
+  /** Country code */
+  country = "US";
+
   /**
-   * Internal auto-incremented ID
+   * Main email to communicate for that account
+   * @uniqueItems
+   * @constraint email - email IS NULL OR email RLIKE '^[^@]+@[^@]+[.][^@]{2,}$'
+   * */
+  email?: string;
+
+  /** Date on which the account was established @maxLength 6 @minimum 2020-01-01 */
+  established? = new Date();
+
+  /** Whether it is enabled or not. Disabled instances will not be used. */
+  enabled = true;
+
+  /** External unique ID, used to refer to external accounts @maxLength 512 @uniqueItems */
+  externalId?: string;
+
+  /** Handle associated with the account */
+  name!: string;
+
+  /**
+   * Descriptive name to identify the instance
+   * @constraint phone - phone IS NULL OR phone RLIKE '^[0-9]{8,16}$'
    */
-  id!: number;
-    comments?: string;
-    country = "US";
-    email?: string;
-    established = new Date();
-    enabled = true;
-    externalId?: string;
-    name!: string;
-    phone?: string;
-    preferences: { [key: string]: boolean|number|string; } = { wrap: true, minAge: 18 };
-    valueList: string[] = [];
-    provider?: string;
-
-    constructor(data?: Pick<Account, "name"> & Partial<Account>) {
-        Object.assign(this, data);
-    }
+  phone?: string;
+
+  /** All the general options associated with the account. */
+  preferences: { [key: string]: boolean|number|string; } = { wrap: true, minAge: 18 };
+
+  /**
+   * Auto-generated field with values
+   * @as JSON_EXTRACT(preferences, '$.*')
+   * @index id, valueList, enabled
+   * */
+  valueList: string[] = [];
+
+  constructor(data?: Pick<Account, "name"> & Partial<Account>) {
+      Object.assign(this, data);
+  }
 }

src/db.ts

@@ -162,7 +162,7 @@ export class DB {
   static async connect(config: ClientConfig, schemas?: Schema[]): Promise<Client> {
     // Iterate over the schemas and map them by name and type if it exists
     schemas?.forEach((s) => {
-      DB.#schemas.set(s.name, s);
+      DB.#schemas.set(s.table!, s);
       if (s.type) DB.#schemas.set(s.type, s);
     });
     if (DB.client) return Promise.resolve(DB.client);
@@ -270,7 +270,7 @@ export class DB {
     const name = typeof target === "string" ? target : target.name;
     if (typeof target === "string") target = Object as unknown as Class<T>;
     if (!schema) schema = DB.#schemas.get(name);
-    repository = new Repository(target, schema, schema?.name ?? name);
+    repository = new Repository(target, schema, schema?.table ?? name);
     this.#repositories.set(target, repository as Repository<Identifiable>);
 
     // Return repository

src/ddl.ts

@@ -2,10 +2,11 @@ import type { Column, Constraint, Index, Relation, Schema } from "./types.ts";
 import DB from "./db.ts";
 
 const dataTypes = {
+  array: "JSON",
   boolean: "BOOLEAN",
   date: "DATETIME",
   integer: "INTEGER",
-  json: "JSON",
+  object: "JSON",
   number: "DOUBLE",
   string: "VARCHAR",
 };
@@ -17,22 +18,48 @@ const serialType = {
 };
 
 const _BaseSchema: DB.Schema = {
-  name: "_BaseSchema",
+  table: "_BaseSchema",
   properties: {
-    id: { type: "integer", required: true, primaryKey: true, description: "Unique identifier, auto-generated. It's the primary key." },
-    insertedAt: { type: "date", required: false, dateOn: "insert", description: "Timestamp when current record is inserted" },
-    updatedAt: { type: "date", required: false, dateOn: "update", description: "Timestamp when current record is updated" },
-    etag: { type: "string", required: false, maxLength: 1024, description: "Possible ETag for all resources that are external. Allows for better synch-ing." },
+    id: { type: "integer", primaryKey: true, description: "Unique identifier, auto-generated. It's the primary key." },
+    insertedAt: { type: "date", dateOn: "insert", description: "Timestamp when current record is inserted" },
+    updatedAt: { type: "date", dateOn: "update", description: "Timestamp when current record is updated" },
+    etag: { type: "string", maxLength: 1024, description: "Possible ETag for all resources that are external. Allows for better synch-ing." },
   },
   indices: [
-    { name: "insertedAt", properties: ["insertedAt"] },
-    { name: "updatedAt", properties: ["updatedAt"] },
+    { properties: ["insertedAt"] },
+    { properties: ["updatedAt"] },
   ],
 };
 
 export class DDL {
+  static EXTENSIONS = ["as", "constraints", "dateOn", "fullText", "index", "primaryKey", "relations", "table"];
+
   static padWidth = 4;
-  static defaultWidth = 256;
+  static defaultWidth = 128;
+  static textWidth = 128;
+
+  /**
+   * When using tools such as [TJS](https://github.com/YousefED/typescript-json-schema) to
+   * generate JSON schemas from TypeScript classes, the resulting schema may need some
+   * cleaning up. This function does that.
+   *
+   * @param schema - the tool-generated schema
+   * @param type - the type of the schema which we may need to correct/override
+   */
+  static cleanSchema(schema: Schema, type?: string, table?: string) {
+    if (type) schema.type = type;
+    if (table) schema.table = table;
+    if (typeof(schema.fullText) === "string") schema.fullText = (schema.fullText as string).split(",").map(s => s.trim());
+    schema.indices ??= [];
+    Object.values(schema.properties).forEach((c) => {
+      if (!c.type) c.type = "string";
+      if (typeof c.primaryKey === "string") c.primaryKey = true;
+      if (typeof c.uniqueItems === "string") c.uniqueItems = true;
+      if (c.format === "date-time") c.type = "date";
+      if (typeof c.index === "string") schema.indices!.push({ properties: (c.index as string).split(",").map(s => s.trim()) });
+    });
+    return schema;
+  }
 
   // Enhance schema with standard properties
   static enhanceSchema(schema: Schema, selected: string[] = ["id", "insertedAt", "updatedAt"]): Schema {
@@ -42,29 +69,30 @@ export class DDL {
 
     // Select indices
     if (!schema.indices) schema.indices = [];
-    for (const name of selected) {
-      const index = _BaseSchema.indices?.find((i) => i.name === name);
-      if (index) schema.indices.push(index);
-    }
+    if (selected.includes("insertedAt")) schema.indices.push({ properties: ["insertedAt"] });
+    if (selected.includes("updatedAt")) schema.indices.push({ properties: ["updatedAt"] });
 
     return schema;
   }
 
   // Column generator
-  static createColumn(dbType: string, name: string, column: Column, namePad: number, padWidth = DDL.padWidth, defaultWidth = DDL.defaultWidth): string {
+  static createColumn(dbType: string, name: string, column: Column, required: boolean, namePad: number, padWidth = DDL.padWidth, defaultWidth = DDL.defaultWidth): string {
+    if (typeof (column.default) === "string" && !column.default.startsWith("'") && !column.default.endsWith("'")) column.default = "'" + column.default + "'";
     if (typeof (column.default) === "object") column.default = "('" + JSON.stringify(column.default) + "')";
     if (column.dateOn === "insert") column.default = "CURRENT_TIMESTAMP";
     if (column.dateOn === "update") column.default = "CURRENT_TIMESTAMP" + ((dbType !== DB.Provider.MYSQL) ? "" : " ON UPDATE CURRENT_TIMESTAMP");
     const pad = "".padEnd(padWidth);
     let type = dataTypes[column.type as keyof typeof dataTypes];
-    const autoIncrement = column.primaryKey && column.type === "integer";
-    const length = column.maxLength || type.endsWith("CHAR") ? "(" + (column.maxLength ?? defaultWidth) + ")" : "";
-    const nullable = column.primaryKey || column.required ? " NOT NULL" : "";
+    if (column.maxLength! > this.textWidth) type = "TEXT";
+    const primaryKey = (column.primaryKey !== undefined);
+    const autoIncrement = primaryKey && column.type === "integer";
+    const length = column.maxLength! < this.textWidth || type.endsWith("CHAR") ? "(" + (column.maxLength ?? defaultWidth) + ")" : "";
+    const nullable = primaryKey || required ? " NOT NULL" : "";
     const gen = autoIncrement ? serialType[dbType as keyof typeof serialType] : "";
-    const asExpression = column.asExpression && (typeof column.asExpression === "string" ? DB._sqlFilter(column.asExpression) : column.asExpression[dbType]);
-    const as = asExpression ? " GENERATED ALWAYS AS (" + asExpression + ") " + (column.generatedType?.toUpperCase() || "VIRTUAL") : "";
+    const expr = column.as && (typeof column.as === "string" ? DB._sqlFilter(column.as) : column.as[dbType]);
+    const as = expr ? " GENERATED ALWAYS AS (" + expr + ") STORED" : "";
     const def = Object.hasOwn(column, "default") ? " DEFAULT " + column.default : "";
-    const key = column.primaryKey ? " PRIMARY KEY" : (column.unique ? " UNIQUE" : "");
+    const key = primaryKey ? " PRIMARY KEY" : (column.uniqueItems !== undefined ? " UNIQUE" : "");
     const comment = (dbType === DB.Provider.MYSQL) && column.description ? " COMMENT '" + column.description.replace(/'/g, "''") + "'" : "";
 
     // Correct Postgres JSON type
@@ -75,19 +103,19 @@ export class DDL {
   }
 
   // Index generator
-  static createIndex(dbType: string, indice: Index, padWidth = 4, table: string): string {
+  static createIndex(dbType: string, index: Index, padWidth = 4, table: string): string {
     const pad = "".padEnd(padWidth);
-    const columns = [...indice.properties] as string[];
+    const columns = [...index.properties] as string[];
+    const name = columns.join("_");
 
     // If there is an array expression, replace the column by it
     // TODO: multivalued indexes only supported on MYSQL for now, Postgres and SQLite will use the entire
-    const subType = indice.subType ?? "CHAR(32)";
-    if (indice.array !== undefined) {
-      columns[indice.array] = "(CAST(" + columns[indice.array] + " AS " + subType + (dbType === DB.Provider.MYSQL ? " ARRAY" : "") + "))";
+    const subType = index.subType ?? "CHAR(32)";
+    if (index.array !== undefined) {
+      columns[index.array] = "(CAST(" + columns[index.array] + " AS " + subType + (dbType === DB.Provider.MYSQL ? " ARRAY" : "") + "))";
     }
 
-    const name = indice.name ?? "";
-    const unique = indice.unique ? "UNIQUE " : "";
+    const unique = index.unique ? "UNIQUE " : "";
     return `${pad}CREATE ${unique}INDEX ${table}_${name} ON ${table} (${columns.join(",")});\n`;
   }
 
@@ -138,14 +166,15 @@ export class DDL {
     const sqlite = dbType === DB.Provider.SQLITE;
 
     // Create SQL
-    const table = nameOverride ?? schema.name;
-    const columns = Object.entries(schema.properties).map(([n, c]) => this.createColumn(dbType, n, c!, namePad)).join("");
-    const relations = !sqlite && Object.entries(schema.relations || []).map(([n, r]) => this.createRelation(dbType, schema.name, n, r!)).join("") || "";
+    const table = nameOverride ?? schema.table! ?? schema.type?.toLowerCase();
+    const required = (n: string) => schema.required?.includes(n) || false;
+    const columns = Object.entries(schema.properties).map(([n, c]) => this.createColumn(dbType, n, c!, required(n), namePad)).join("");
+    const relations = !sqlite && Object.entries(schema.relations || []).map(([n, r]) => this.createRelation(dbType, table, n, r!)).join("") || "";
 
     // Create constraints
     const filter = (c: Constraint) => !c.provider || c.provider === dbType;
-    const columnConstraints = Object.entries(schema.properties || {}).map(([n, c]) => this.createColumnConstraint(dbType, schema.name, n, c));
-    const independentConstraints = (schema.constraints || []).filter(filter).map((c) => this.createIndependentConstraint(dbType, schema.name, c));
+    const columnConstraints = Object.entries(schema.properties || {}).map(([n, c]) => this.createColumnConstraint(dbType, table, n, c));
+    const independentConstraints = (schema.constraints || []).filter(filter).map((c) => this.createIndependentConstraint(dbType, table, c));
     const constraints = !sqlite && [...columnConstraints, ...independentConstraints].join("") || "";
 
     // Create sql
@@ -155,8 +184,7 @@ export class DDL {
     if (schema.indices) sql += "\n" + schema.indices?.map((i) => this.createIndex(dbType, i, 0, table)).join("");
 
     // Full text index
-    const fullTextColumns = Object.entries(schema.properties).filter(([_, c]) => c.fullText).map(([n, _]) => n);
-    if (fullTextColumns.length) sql += this.createFullTextIndex(dbType, fullTextColumns, 0, table);
+    if (schema.fullText?.length) sql += this.createFullTextIndex(dbType, schema.fullText, 0, table);
 
     const fixDanglingComma = (sql: string) => sql.replace(/,\n\)/, "\n);");
     if (dbType === DB.Provider.POSTGRES) sql = this.#postgres(sql);