Use generic schema definitions

.changeset/weak-cats-hug.md

@@ -0,0 +1,5 @@
+---
+'@powersync/service-sync-rules': minor
+---
+
+Optionally include original types in generated schemas as a comment.

modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts

@@ -275,14 +275,14 @@ GROUP BY schemaname, tablename, quoted_name`
     );
     const rows = pgwire.pgwireRows(results);
 
-    let schemas: Record<string, any> = {};
+    let schemas: Record<string, service_types.DatabaseSchema> = {};
 
     for (let row of rows) {
       const schema = (schemas[row.schemaname] ??= {
         name: row.schemaname,
         tables: []
       });
-      const table = {
+      const table: service_types.TableSchema = {
         name: row.tablename,
         columns: [] as any[]
       };
@@ -296,7 +296,9 @@ GROUP BY schemaname, tablename, quoted_name`
         }
         table.columns.push({
           name: column.attname,
+          sqlite_type: sync_rules.expressionTypeFromPostgresType(pg_type).typeFlags,
           type: column.data_type,
+          internal_type: column.data_type,
           pg_type: pg_type
         });
       }

packages/sync-rules/src/DartSchemaGenerator.ts

@@ -1,5 +1,5 @@
 import { ColumnDefinition, TYPE_INTEGER, TYPE_REAL, TYPE_TEXT } from './ExpressionType.js';
-import { SchemaGenerator } from './SchemaGenerator.js';
+import { GenerateSchemaOptions, SchemaGenerator } from './SchemaGenerator.js';
 import { SqlSyncRules } from './SqlSyncRules.js';
 import { SourceSchema } from './types.js';
 
@@ -9,18 +9,34 @@ export class DartSchemaGenerator extends SchemaGenerator {
   readonly mediaType = 'text/x-dart';
   readonly fileName = 'schema.dart';
 
-  generate(source: SqlSyncRules, schema: SourceSchema): string {
+  generate(source: SqlSyncRules, schema: SourceSchema, options?: GenerateSchemaOptions): string {
     const tables = super.getAllTables(source, schema);
 
     return `Schema([
-  ${tables.map((table) => this.generateTable(table.name, table.columns)).join(',\n  ')}
+  ${tables.map((table) => this.generateTable(table.name, table.columns, options)).join(',\n  ')}
 ]);
 `;
   }
 
-  private generateTable(name: string, columns: ColumnDefinition[]): string {
+  private generateTable(name: string, columns: ColumnDefinition[], options?: GenerateSchemaOptions): string {
+    const generated = columns.map((c, i) => {
+      const last = i == columns.length - 1;
+      const base = this.generateColumn(c);
+      let withFormatting: string;
+      if (last) {
+        withFormatting = `    ${base}`;
+      } else {
+        withFormatting = `    ${base},`;
+      }
+
+      if (options?.includeTypeComments && c.originalType != null) {
+        return `${withFormatting} // ${c.originalType}`;
+      } else {
+        return withFormatting;
+      }
+    });
     return `Table('${name}', [
-    ${columns.map((c) => this.generateColumn(c)).join(',\n    ')}
+${generated.join('\n')}
   ])`;
   }
 

packages/sync-rules/src/ExpressionType.ts

@@ -4,11 +4,12 @@ export const TYPE_TEXT = 2;
 export const TYPE_INTEGER = 4;
 export const TYPE_REAL = 8;
 
-export type SqliteType = 'null' | 'blob' | 'text' | 'integer' | 'real';
+export type SqliteType = 'null' | 'blob' | 'text' | 'integer' | 'real' | 'numeric';
 
 export interface ColumnDefinition {
   name: string;
   type: ExpressionType;
+  originalType?: string;
 }
 
 export class ExpressionType {
@@ -34,7 +35,7 @@ export class ExpressionType {
     return new ExpressionType(typeFlags);
   }
 
-  static fromTypeText(type: SqliteType | 'numeric') {
+  static fromTypeText(type: SqliteType) {
     if (type == 'null') {
       return ExpressionType.NONE;
     } else if (type == 'blob') {
@@ -72,3 +73,28 @@ export class ExpressionType {
     return this.typeFlags == TYPE_NONE;
   }
 }
+
+/**
+ * Here only for backwards-compatibility only.
+ */
+export function expressionTypeFromPostgresType(type: string | undefined): ExpressionType {
+  if (type?.endsWith('[]')) {
+    return ExpressionType.TEXT;
+  }
+  switch (type) {
+    case 'bool':
+      return ExpressionType.INTEGER;
+    case 'bytea':
+      return ExpressionType.BLOB;
+    case 'int2':
+    case 'int4':
+    case 'int8':
+    case 'oid':
+      return ExpressionType.INTEGER;
+    case 'float4':
+    case 'float8':
+      return ExpressionType.REAL;
+    default:
+      return ExpressionType.TEXT;
+  }
+}

packages/sync-rules/src/SchemaGenerator.ts

@@ -2,6 +2,10 @@ import { ColumnDefinition } from './ExpressionType.js';
 import { SqlSyncRules } from './SqlSyncRules.js';
 import { SourceSchema } from './types.js';
 
+export interface GenerateSchemaOptions {
+  includeTypeComments?: boolean;
+}
+
 export abstract class SchemaGenerator {
   protected getAllTables(source: SqlSyncRules, schema: SourceSchema) {
     let tables: Record<string, Record<string, ColumnDefinition>> = {};
@@ -33,5 +37,5 @@ export abstract class SchemaGenerator {
   abstract readonly mediaType: string;
   abstract readonly fileName: string;
 
-  abstract generate(source: SqlSyncRules, schema: SourceSchema): string;
+  abstract generate(source: SqlSyncRules, schema: SourceSchema, options?: GenerateSchemaOptions): string;
 }

packages/sync-rules/src/SqlDataQuery.ts

@@ -123,7 +123,9 @@ export class SqlDataQuery {
             output[name] = clause.evaluate(tables);
           },
           getTypes(schema, into) {
-            into[name] = { name, type: clause.getType(schema) };
+            const def = clause.getColumnDefinition(schema);
+
+            into[name] = { name, type: def?.type ?? ExpressionType.NONE, originalType: def?.originalType };
           }
         });
       } else {
@@ -152,7 +154,7 @@ export class SqlDataQuery {
           // Not performing schema-based validation - assume there is an id
           hasId = true;
         } else {
-          const idType = querySchema.getType(alias, 'id');
+          const idType = querySchema.getColumn(alias, 'id')?.type ?? ExpressionType.NONE;
           if (!idType.isNone()) {
             hasId = true;
           }
@@ -296,12 +298,12 @@ export class SqlDataQuery {
 
   private getColumnOutputsFor(schemaTable: SourceSchemaTable, output: Record<string, ColumnDefinition>) {
     const querySchema: QuerySchema = {
-      getType: (table, column) => {
+      getColumn: (table, column) => {
         if (table == this.table!) {
-          return schemaTable.getType(column) ?? ExpressionType.NONE;
+          return schemaTable.getColumn(column);
         } else {
           // TODO: bucket parameters?
-          return ExpressionType.NONE;
+          return undefined;
         }
       },
       getColumns: (table) => {

packages/sync-rules/src/StaticSchema.ts

@@ -1,4 +1,4 @@
-import { ColumnDefinition, ExpressionType } from './ExpressionType.js';
+import { ColumnDefinition, ExpressionType, expressionTypeFromPostgresType, SqliteType } from './ExpressionType.js';
 import { SourceTableInterface } from './SourceTableInterface.js';
 import { TablePattern } from './TablePattern.js';
 import { SourceSchema, SourceSchemaTable } from './types.js';
@@ -14,11 +14,28 @@ export interface SourceTableDefinition {
 }
 
 export interface SourceColumnDefinition {
+  /**
+   * Column name.
+   */
   name: string;
+
+  /**
+   * Option 1: SQLite type flags - see ExpressionType.typeFlags.
+   * Option 2: SQLite type name in lowercase - 'text' | 'integer' | 'real' | 'numeric' | 'blob' | 'null'
+   */
+  sqlite_type?: number | SqliteType;
+
   /**
-   * Postgres type.
+   * Type name from the source database, e.g. "character varying(255)[]"
    */
-  pg_type: string;
+  internal_type?: string;
+
+  /**
+   * Postgres type, kept for backwards-compatibility.
+   *
+   * @deprecated - use internal_type instead
+   */
+  pg_type?: string;
 }
 
 export interface SourceConnectionDefinition {
@@ -43,8 +60,8 @@ class SourceTableDetails implements SourceTableInterface, SourceSchemaTable {
     );
   }
 
-  getType(column: string): ExpressionType | undefined {
-    return this.columns[column]?.type;
+  getColumn(column: string): ColumnDefinition | undefined {
+    return this.columns[column];
   }
 
   getColumns(): ColumnDefinition[] {
@@ -75,28 +92,20 @@ export class StaticSchema implements SourceSchema {
 function mapColumn(column: SourceColumnDefinition): ColumnDefinition {
   return {
     name: column.name,
-    type: mapType(column.pg_type)
+    type: mapColumnType(column),
+    originalType: column.internal_type
   };
 }
 
-function mapType(type: string | undefined): ExpressionType {
-  if (type?.endsWith('[]')) {
-    return ExpressionType.TEXT;
-  }
-  switch (type) {
-    case 'bool':
-      return ExpressionType.INTEGER;
-    case 'bytea':
-      return ExpressionType.BLOB;
-    case 'int2':
-    case 'int4':
-    case 'int8':
-    case 'oid':
-      return ExpressionType.INTEGER;
-    case 'float4':
-    case 'float8':
-      return ExpressionType.REAL;
-    default:
-      return ExpressionType.TEXT;
+function mapColumnType(column: SourceColumnDefinition): ExpressionType {
+  if (typeof column.sqlite_type == 'number') {
+    return ExpressionType.of(column.sqlite_type);
+  } else if (typeof column.sqlite_type == 'string') {
+    return ExpressionType.fromTypeText(column.sqlite_type);
+  } else if (column.pg_type != null) {
+    // We still handle these types for backwards-compatibility of old schemas
+    return expressionTypeFromPostgresType(column.pg_type);
+  } else {
+    throw new Error(`Cannot determine SQLite type of ${JSON.stringify(column)}`);
   }
 }

packages/sync-rules/src/TableQuerySchema.ts

@@ -1,23 +1,20 @@
-import { ColumnDefinition, ExpressionType } from './ExpressionType.js';
+import { ColumnDefinition } from './ExpressionType.js';
 import { QuerySchema, SourceSchemaTable } from './types.js';
 
 export class TableQuerySchema implements QuerySchema {
-  constructor(
-    private tables: SourceSchemaTable[],
-    private alias: string
-  ) {}
+  constructor(private tables: SourceSchemaTable[], private alias: string) {}
 
-  getType(table: string, column: string): ExpressionType {
+  getColumn(table: string, column: string): ColumnDefinition | undefined {
     if (table != this.alias) {
-      return ExpressionType.NONE;
+      return undefined;
     }
     for (let table of this.tables) {
-      const t = table.getType(column);
+      const t = table.getColumn(column);
       if (t != null) {
         return t;
       }
     }
-    return ExpressionType.NONE;
+    return undefined;
   }
 
   getColumns(table: string): ColumnDefinition[] {

packages/sync-rules/src/TsSchemaGenerator.ts

@@ -1,5 +1,5 @@
 import { ColumnDefinition, TYPE_INTEGER, TYPE_REAL, TYPE_TEXT } from './ExpressionType.js';
-import { SchemaGenerator } from './SchemaGenerator.js';
+import { GenerateSchemaOptions, SchemaGenerator } from './SchemaGenerator.js';
 import { SqlSyncRules } from './SqlSyncRules.js';
 import { SourceSchema } from './types.js';
 
@@ -47,12 +47,12 @@ export class TsSchemaGenerator extends SchemaGenerator {
     }
   }
 
-  generate(source: SqlSyncRules, schema: SourceSchema): string {
+  generate(source: SqlSyncRules, schema: SourceSchema, options?: GenerateSchemaOptions): string {
     const tables = super.getAllTables(source, schema);
 
     return `${this.generateImports()}
 
-${tables.map((table) => this.generateTable(table.name, table.columns)).join('\n\n')}
+${tables.map((table) => this.generateTable(table.name, table.columns, options)).join('\n\n')}
 
 export const AppSchema = new Schema({
   ${tables.map((table) => table.name).join(',\n  ')}
@@ -81,11 +81,28 @@ ${this.generateTypeExports()}`;
     }
   }
 
-  private generateTable(name: string, columns: ColumnDefinition[]): string {
+  private generateTable(name: string, columns: ColumnDefinition[], options?: GenerateSchemaOptions): string {
+    const generated = columns.map((c, i) => {
+      const last = i == columns.length - 1;
+      const base = this.generateColumn(c);
+      let withFormatting: string;
+      if (last) {
+        withFormatting = `    ${base}`;
+      } else {
+        withFormatting = `    ${base},`;
+      }
+
+      if (options?.includeTypeComments && c.originalType != null) {
+        return `${withFormatting} // ${c.originalType}`;
+      } else {
+        return withFormatting;
+      }
+    });
+
     return `const ${name} = new Table(
   {
     // id column (text) is automatically included
-    ${columns.map((c) => this.generateColumn(c)).join(',\n    ')}
+${generated.join('\n')}
   },
   { indexes: {} }
 );`;