From df2725ac623b031d9e11fb72d0006457120d6909 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 25 Sep 2024 16:54:24 +0200 Subject: [PATCH 1/5] Refactor schema definitions to not be postgres-specific. --- .../src/api/PostgresRouteAPIAdapter.ts | 6 ++- packages/sync-rules/src/ExpressionType.ts | 29 +++++++++- packages/sync-rules/src/StaticSchema.ts | 54 +++++++++++-------- packages/types/src/definitions.ts | 33 +++++++++++- 4 files changed, 93 insertions(+), 29 deletions(-) diff --git a/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts b/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts index 205a2c744..49f53c415 100644 --- a/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts +++ b/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts @@ -275,14 +275,14 @@ GROUP BY schemaname, tablename, quoted_name` ); const rows = pgwire.pgwireRows(results); - let schemas: Record = {}; + let schemas: Record = {}; for (let row of rows) { const schema = (schemas[row.schemaname] ??= { name: row.schemaname, tables: [] }); - const table = { + const table: service_types.DatabaseSchema['tables'][0] = { name: row.tablename, columns: [] as any[] }; @@ -296,7 +296,9 @@ GROUP BY schemaname, tablename, quoted_name` } table.columns.push({ name: column.attname, + sqlite_type: sync_rules.expressionTypeFromPostgresType(pg_type).typeFlags, type: column.data_type, + original_type: column.data_type, pg_type: pg_type }); } diff --git a/packages/sync-rules/src/ExpressionType.ts b/packages/sync-rules/src/ExpressionType.ts index 87b52c6e9..2121abbb0 100644 --- a/packages/sync-rules/src/ExpressionType.ts +++ b/packages/sync-rules/src/ExpressionType.ts @@ -4,7 +4,7 @@ export const TYPE_TEXT = 2; export const TYPE_INTEGER = 4; export const TYPE_REAL = 8; -export type SqliteType = 'null' | 'blob' | 'text' | 'integer' | 'real'; +export type SqliteType = 'null' | 'blob' | 'text' | 'integer' | 'real' | 'numeric'; export interface ColumnDefinition { name: string; @@ -34,7 +34,7 @@ export class ExpressionType { return new ExpressionType(typeFlags); } - static fromTypeText(type: SqliteType | 'numeric') { + static fromTypeText(type: SqliteType) { if (type == 'null') { return ExpressionType.NONE; } else if (type == 'blob') { @@ -72,3 +72,28 @@ export class ExpressionType { return this.typeFlags == TYPE_NONE; } } + +/** + * Here only for backwards-compatibility only. + */ +export function expressionTypeFromPostgresType(type: string | undefined): ExpressionType { + if (type?.endsWith('[]')) { + return ExpressionType.TEXT; + } + switch (type) { + case 'bool': + return ExpressionType.INTEGER; + case 'bytea': + return ExpressionType.BLOB; + case 'int2': + case 'int4': + case 'int8': + case 'oid': + return ExpressionType.INTEGER; + case 'float4': + case 'float8': + return ExpressionType.REAL; + default: + return ExpressionType.TEXT; + } +} diff --git a/packages/sync-rules/src/StaticSchema.ts b/packages/sync-rules/src/StaticSchema.ts index a807d471c..997b6da2f 100644 --- a/packages/sync-rules/src/StaticSchema.ts +++ b/packages/sync-rules/src/StaticSchema.ts @@ -1,4 +1,4 @@ -import { ColumnDefinition, ExpressionType } from './ExpressionType.js'; +import { ColumnDefinition, ExpressionType, expressionTypeFromPostgresType, SqliteType } from './ExpressionType.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { TablePattern } from './TablePattern.js'; import { SourceSchema, SourceSchemaTable } from './types.js'; @@ -14,11 +14,28 @@ export interface SourceTableDefinition { } export interface SourceColumnDefinition { + /** + * Column name. + */ name: string; + + /** + * Option 1: SQLite type flags - see ExpressionType.typeFlags. + * Option 2: SQLite type name in lowercase - 'text' | 'integer' | 'real' | 'numeric' | 'blob' | 'null' + */ + sqlite_type?: number | SqliteType; + /** - * Postgres type. + * Type name from the source database, e.g. "character varying(255)[]" */ - pg_type: string; + original_type?: string; + + /** + * Postgres type, kept for backwards-compatibility. + * + * @deprecated - use original_type instead + */ + pg_type?: string; } export interface SourceConnectionDefinition { @@ -75,28 +92,19 @@ export class StaticSchema implements SourceSchema { function mapColumn(column: SourceColumnDefinition): ColumnDefinition { return { name: column.name, - type: mapType(column.pg_type) + type: mapColumnType(column) }; } -function mapType(type: string | undefined): ExpressionType { - if (type?.endsWith('[]')) { - return ExpressionType.TEXT; - } - switch (type) { - case 'bool': - return ExpressionType.INTEGER; - case 'bytea': - return ExpressionType.BLOB; - case 'int2': - case 'int4': - case 'int8': - case 'oid': - return ExpressionType.INTEGER; - case 'float4': - case 'float8': - return ExpressionType.REAL; - default: - return ExpressionType.TEXT; +function mapColumnType(column: SourceColumnDefinition): ExpressionType { + if (typeof column.sqlite_type == 'number') { + return ExpressionType.of(column.sqlite_type); + } else if (typeof column.sqlite_type == 'string') { + return ExpressionType.fromTypeText(column.sqlite_type); + } else if (column.pg_type != null) { + // We still handle these types for backwards-compatibility of old schemas + return expressionTypeFromPostgresType(column.pg_type); + } else { + throw new Error(`Cannot determine SQLite type of ${JSON.stringify(column)}`); } } diff --git a/packages/types/src/definitions.ts b/packages/types/src/definitions.ts index 35dc64177..b7eff1a27 100644 --- a/packages/types/src/definitions.ts +++ b/packages/types/src/definitions.ts @@ -91,6 +91,15 @@ export const ConnectionStatusV2 = t.object({ }); export type ConnectionStatusV2 = t.Encoded; +export enum SqliteSchemaTypeText { + null = 'null', + blob = 'blob', + text = 'text', + integer = 'integer', + real = 'real', + numeric = 'numeric' +} + export const DatabaseSchema = t.object({ name: t.string, tables: t.array( @@ -99,14 +108,34 @@ export const DatabaseSchema = t.object({ columns: t.array( t.object({ name: t.string, + + /** + * Option 1: SQLite type flags - see ExpressionType.typeFlags. + * Option 2: SQLite type name in lowercase - 'text' | 'integer' | 'real' | 'numeric' | 'blob' | 'null' + */ + sqlite_type: t.number.or(t.Enum(SqliteSchemaTypeText)), + + /** + * Type name from the source database, e.g. "character varying(255)[]" + */ + original_type: t.string, + + /** + * Description for the field if available. + */ + description: t.string.optional(), + /** * Full type name, e.g. "character varying(255)[]" + * @deprecated - use original_type */ - type: t.string, + type: t.string.optional(), + /** * Internal postgres type, e.g. "varchar[]". + * @deprecated - use original_type */ - pg_type: t.string + pg_type: t.string.optional() }) ) }) From a197710fab3dadf06e05b35705153e9488af9884 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 25 Sep 2024 16:58:26 +0200 Subject: [PATCH 2/5] Rename original_type -> internal_type. --- modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts | 2 +- packages/sync-rules/src/StaticSchema.ts | 4 ++-- packages/types/src/definitions.ts | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts b/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts index 49f53c415..3862daec5 100644 --- a/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts +++ b/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts @@ -298,7 +298,7 @@ GROUP BY schemaname, tablename, quoted_name` name: column.attname, sqlite_type: sync_rules.expressionTypeFromPostgresType(pg_type).typeFlags, type: column.data_type, - original_type: column.data_type, + internal_type: column.data_type, pg_type: pg_type }); } diff --git a/packages/sync-rules/src/StaticSchema.ts b/packages/sync-rules/src/StaticSchema.ts index 997b6da2f..5176190e0 100644 --- a/packages/sync-rules/src/StaticSchema.ts +++ b/packages/sync-rules/src/StaticSchema.ts @@ -28,12 +28,12 @@ export interface SourceColumnDefinition { /** * Type name from the source database, e.g. "character varying(255)[]" */ - original_type?: string; + internal_type?: string; /** * Postgres type, kept for backwards-compatibility. * - * @deprecated - use original_type instead + * @deprecated - use internal_type instead */ pg_type?: string; } diff --git a/packages/types/src/definitions.ts b/packages/types/src/definitions.ts index b7eff1a27..bdbf88d29 100644 --- a/packages/types/src/definitions.ts +++ b/packages/types/src/definitions.ts @@ -118,7 +118,7 @@ export const DatabaseSchema = t.object({ /** * Type name from the source database, e.g. "character varying(255)[]" */ - original_type: t.string, + internal_type: t.string, /** * Description for the field if available. @@ -127,13 +127,13 @@ export const DatabaseSchema = t.object({ /** * Full type name, e.g. "character varying(255)[]" - * @deprecated - use original_type + * @deprecated - use internal_type */ type: t.string.optional(), /** * Internal postgres type, e.g. "varchar[]". - * @deprecated - use original_type + * @deprecated - use internal_type instead */ pg_type: t.string.optional() }) From b0a4411b0b56e806cbb05d7ff464f342d10eb529 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 25 Sep 2024 17:57:37 +0200 Subject: [PATCH 3/5] Include source types in the generated schema comments (optional). --- .../sync-rules/src/DartSchemaGenerator.ts | 26 ++- packages/sync-rules/src/ExpressionType.ts | 1 + packages/sync-rules/src/SchemaGenerator.ts | 6 +- packages/sync-rules/src/SqlDataQuery.ts | 12 +- packages/sync-rules/src/StaticSchema.ts | 7 +- packages/sync-rules/src/TableQuerySchema.ts | 15 +- packages/sync-rules/src/TsSchemaGenerator.ts | 27 ++- packages/sync-rules/src/sql_filters.ts | 23 ++- packages/sync-rules/src/sql_support.ts | 20 +- packages/sync-rules/src/types.ts | 6 +- .../test/src/generate_schema.test.ts | 172 ++++++++++++++++++ .../sync-rules/test/src/sync_rules.test.ts | 106 ----------- 12 files changed, 267 insertions(+), 154 deletions(-) create mode 100644 packages/sync-rules/test/src/generate_schema.test.ts diff --git a/packages/sync-rules/src/DartSchemaGenerator.ts b/packages/sync-rules/src/DartSchemaGenerator.ts index e25fedf20..1a90884de 100644 --- a/packages/sync-rules/src/DartSchemaGenerator.ts +++ b/packages/sync-rules/src/DartSchemaGenerator.ts @@ -1,5 +1,5 @@ import { ColumnDefinition, TYPE_INTEGER, TYPE_REAL, TYPE_TEXT } from './ExpressionType.js'; -import { SchemaGenerator } from './SchemaGenerator.js'; +import { GenerateSchemaOptions, SchemaGenerator } from './SchemaGenerator.js'; import { SqlSyncRules } from './SqlSyncRules.js'; import { SourceSchema } from './types.js'; @@ -9,18 +9,34 @@ export class DartSchemaGenerator extends SchemaGenerator { readonly mediaType = 'text/x-dart'; readonly fileName = 'schema.dart'; - generate(source: SqlSyncRules, schema: SourceSchema): string { + generate(source: SqlSyncRules, schema: SourceSchema, options?: GenerateSchemaOptions): string { const tables = super.getAllTables(source, schema); return `Schema([ - ${tables.map((table) => this.generateTable(table.name, table.columns)).join(',\n ')} + ${tables.map((table) => this.generateTable(table.name, table.columns, options)).join(',\n ')} ]); `; } - private generateTable(name: string, columns: ColumnDefinition[]): string { + private generateTable(name: string, columns: ColumnDefinition[], options?: GenerateSchemaOptions): string { + const generated = columns.map((c, i) => { + const last = i == columns.length - 1; + const base = this.generateColumn(c); + let withFormatting: string; + if (last) { + withFormatting = ` ${base}`; + } else { + withFormatting = ` ${base},`; + } + + if (options?.includeTypeComments && c.originalType != null) { + return `${withFormatting} // ${c.originalType}`; + } else { + return withFormatting; + } + }); return `Table('${name}', [ - ${columns.map((c) => this.generateColumn(c)).join(',\n ')} +${generated.join('\n')} ])`; } diff --git a/packages/sync-rules/src/ExpressionType.ts b/packages/sync-rules/src/ExpressionType.ts index 2121abbb0..ad46c4408 100644 --- a/packages/sync-rules/src/ExpressionType.ts +++ b/packages/sync-rules/src/ExpressionType.ts @@ -9,6 +9,7 @@ export type SqliteType = 'null' | 'blob' | 'text' | 'integer' | 'real' | 'numeri export interface ColumnDefinition { name: string; type: ExpressionType; + originalType?: string; } export class ExpressionType { diff --git a/packages/sync-rules/src/SchemaGenerator.ts b/packages/sync-rules/src/SchemaGenerator.ts index 91bf48942..18e111b57 100644 --- a/packages/sync-rules/src/SchemaGenerator.ts +++ b/packages/sync-rules/src/SchemaGenerator.ts @@ -2,6 +2,10 @@ import { ColumnDefinition } from './ExpressionType.js'; import { SqlSyncRules } from './SqlSyncRules.js'; import { SourceSchema } from './types.js'; +export interface GenerateSchemaOptions { + includeTypeComments?: boolean; +} + export abstract class SchemaGenerator { protected getAllTables(source: SqlSyncRules, schema: SourceSchema) { let tables: Record> = {}; @@ -33,5 +37,5 @@ export abstract class SchemaGenerator { abstract readonly mediaType: string; abstract readonly fileName: string; - abstract generate(source: SqlSyncRules, schema: SourceSchema): string; + abstract generate(source: SqlSyncRules, schema: SourceSchema, options?: GenerateSchemaOptions): string; } diff --git a/packages/sync-rules/src/SqlDataQuery.ts b/packages/sync-rules/src/SqlDataQuery.ts index b5377d436..e9d6cb04c 100644 --- a/packages/sync-rules/src/SqlDataQuery.ts +++ b/packages/sync-rules/src/SqlDataQuery.ts @@ -123,7 +123,9 @@ export class SqlDataQuery { output[name] = clause.evaluate(tables); }, getTypes(schema, into) { - into[name] = { name, type: clause.getType(schema) }; + const def = clause.getColumnDefinition(schema); + + into[name] = { name, type: def?.type ?? ExpressionType.NONE, originalType: def?.originalType }; } }); } else { @@ -152,7 +154,7 @@ export class SqlDataQuery { // Not performing schema-based validation - assume there is an id hasId = true; } else { - const idType = querySchema.getType(alias, 'id'); + const idType = querySchema.getColumn(alias, 'id')?.type ?? ExpressionType.NONE; if (!idType.isNone()) { hasId = true; } @@ -296,12 +298,12 @@ export class SqlDataQuery { private getColumnOutputsFor(schemaTable: SourceSchemaTable, output: Record) { const querySchema: QuerySchema = { - getType: (table, column) => { + getColumn: (table, column) => { if (table == this.table!) { - return schemaTable.getType(column) ?? ExpressionType.NONE; + return schemaTable.getColumn(column); } else { // TODO: bucket parameters? - return ExpressionType.NONE; + return undefined; } }, getColumns: (table) => { diff --git a/packages/sync-rules/src/StaticSchema.ts b/packages/sync-rules/src/StaticSchema.ts index 5176190e0..aa27114c6 100644 --- a/packages/sync-rules/src/StaticSchema.ts +++ b/packages/sync-rules/src/StaticSchema.ts @@ -60,8 +60,8 @@ class SourceTableDetails implements SourceTableInterface, SourceSchemaTable { ); } - getType(column: string): ExpressionType | undefined { - return this.columns[column]?.type; + getColumn(column: string): ColumnDefinition | undefined { + return this.columns[column]; } getColumns(): ColumnDefinition[] { @@ -92,7 +92,8 @@ export class StaticSchema implements SourceSchema { function mapColumn(column: SourceColumnDefinition): ColumnDefinition { return { name: column.name, - type: mapColumnType(column) + type: mapColumnType(column), + originalType: column.internal_type }; } diff --git a/packages/sync-rules/src/TableQuerySchema.ts b/packages/sync-rules/src/TableQuerySchema.ts index 01a7a8cf4..c223371b3 100644 --- a/packages/sync-rules/src/TableQuerySchema.ts +++ b/packages/sync-rules/src/TableQuerySchema.ts @@ -1,23 +1,20 @@ -import { ColumnDefinition, ExpressionType } from './ExpressionType.js'; +import { ColumnDefinition } from './ExpressionType.js'; import { QuerySchema, SourceSchemaTable } from './types.js'; export class TableQuerySchema implements QuerySchema { - constructor( - private tables: SourceSchemaTable[], - private alias: string - ) {} + constructor(private tables: SourceSchemaTable[], private alias: string) {} - getType(table: string, column: string): ExpressionType { + getColumn(table: string, column: string): ColumnDefinition | undefined { if (table != this.alias) { - return ExpressionType.NONE; + return undefined; } for (let table of this.tables) { - const t = table.getType(column); + const t = table.getColumn(column); if (t != null) { return t; } } - return ExpressionType.NONE; + return undefined; } getColumns(table: string): ColumnDefinition[] { diff --git a/packages/sync-rules/src/TsSchemaGenerator.ts b/packages/sync-rules/src/TsSchemaGenerator.ts index 9e3e56e51..1a0dba58c 100644 --- a/packages/sync-rules/src/TsSchemaGenerator.ts +++ b/packages/sync-rules/src/TsSchemaGenerator.ts @@ -1,5 +1,5 @@ import { ColumnDefinition, TYPE_INTEGER, TYPE_REAL, TYPE_TEXT } from './ExpressionType.js'; -import { SchemaGenerator } from './SchemaGenerator.js'; +import { GenerateSchemaOptions, SchemaGenerator } from './SchemaGenerator.js'; import { SqlSyncRules } from './SqlSyncRules.js'; import { SourceSchema } from './types.js'; @@ -47,12 +47,12 @@ export class TsSchemaGenerator extends SchemaGenerator { } } - generate(source: SqlSyncRules, schema: SourceSchema): string { + generate(source: SqlSyncRules, schema: SourceSchema, options?: GenerateSchemaOptions): string { const tables = super.getAllTables(source, schema); return `${this.generateImports()} -${tables.map((table) => this.generateTable(table.name, table.columns)).join('\n\n')} +${tables.map((table) => this.generateTable(table.name, table.columns, options)).join('\n\n')} export const AppSchema = new Schema({ ${tables.map((table) => table.name).join(',\n ')} @@ -81,11 +81,28 @@ ${this.generateTypeExports()}`; } } - private generateTable(name: string, columns: ColumnDefinition[]): string { + private generateTable(name: string, columns: ColumnDefinition[], options?: GenerateSchemaOptions): string { + const generated = columns.map((c, i) => { + const last = i == columns.length - 1; + const base = this.generateColumn(c); + let withFormatting: string; + if (last) { + withFormatting = ` ${base}`; + } else { + withFormatting = ` ${base},`; + } + + if (options?.includeTypeComments && c.originalType != null) { + return `${withFormatting} // ${c.originalType}`; + } else { + return withFormatting; + } + }); + return `const ${name} = new Table( { // id column (text) is automatically included - ${columns.map((c) => this.generateColumn(c)).join(',\n ')} +${generated.join('\n')} }, { indexes: {} } );`; diff --git a/packages/sync-rules/src/sql_filters.ts b/packages/sync-rules/src/sql_filters.ts index 67c4b1603..10ea88053 100644 --- a/packages/sync-rules/src/sql_filters.ts +++ b/packages/sync-rules/src/sql_filters.ts @@ -198,8 +198,8 @@ export class SqlTools { evaluate(tables: QueryParameters): SqliteValue { return tables[table]?.[column]; }, - getType(schema) { - return schema.getType(table, column); + getColumnDefinition(schema) { + return schema.getColumn(table, column); } } satisfies RowValueClause; } else { @@ -514,8 +514,8 @@ export class SqlTools { private checkRef(table: string, ref: ExprRef) { if (this.schema) { - const type = this.schema.getType(table, ref.name); - if (type.typeFlags == TYPE_NONE) { + const type = this.schema.getColumn(table, ref.name); + if (type == null) { this.warn(`Column not found: ${ref.name}`, ref); } } @@ -613,9 +613,11 @@ export class SqlTools { const args = argClauses.map((e) => (e as RowValueClause).evaluate(tables)); return fnImpl.call(...args); }, - getType(schema) { - const argTypes = argClauses.map((e) => (e as RowValueClause).getType(schema)); - return fnImpl.getReturnType(argTypes); + getColumnDefinition(schema) { + const argTypes = argClauses.map( + (e) => (e as RowValueClause).getColumnDefinition(schema)?.type ?? ExpressionType.NONE + ); + return { name: `${fnImpl}()`, type: fnImpl.getReturnType(argTypes) }; } } satisfies RowValueClause; } else if (argsType == 'param') { @@ -671,8 +673,11 @@ function staticValueClause(value: SqliteValue): StaticValueClause { value: value, // RowValueClause compatibility evaluate: () => value, - getType() { - return ExpressionType.fromTypeText(sqliteTypeOf(value)); + getColumnDefinition() { + return { + name: 'literal', + type: ExpressionType.fromTypeText(sqliteTypeOf(value)) + }; }, // ParamterValueClause compatibility key: JSONBig.stringify(value), diff --git a/packages/sync-rules/src/sql_support.ts b/packages/sync-rules/src/sql_support.ts index 9d381f363..e2b074c55 100644 --- a/packages/sync-rules/src/sql_support.ts +++ b/packages/sync-rules/src/sql_support.ts @@ -66,10 +66,14 @@ export function compileStaticOperator(op: string, left: RowValueClause, right: R const rightValue = right.evaluate(tables); return evaluateOperator(op, leftValue, rightValue); }, - getType(schema) { - const typeLeft = left.getType(schema); - const typeRight = right.getType(schema); - return getOperatorReturnType(op, typeLeft, typeRight); + getColumnDefinition(schema) { + const typeLeft = left.getColumnDefinition(schema)?.type ?? ExpressionType.NONE; + const typeRight = right.getColumnDefinition(schema)?.type ?? ExpressionType.NONE; + const type = getOperatorReturnType(op, typeLeft, typeRight); + return { + name: '?', + type + }; } }; } @@ -95,8 +99,8 @@ export function andFilters(a: CompiledClause, b: CompiledClause): CompiledClause const bValue = sqliteBool(b.evaluate(tables)); return sqliteBool(aValue && bValue); }, - getType() { - return ExpressionType.INTEGER; + getColumnDefinition() { + return { name: 'and', type: ExpressionType.INTEGER }; } } satisfies RowValueClause; } @@ -156,8 +160,8 @@ export function orFilters(a: CompiledClause, b: CompiledClause): CompiledClause const bValue = sqliteBool(b.evaluate(tables)); return sqliteBool(aValue || bValue); }, - getType() { - return ExpressionType.INTEGER; + getColumnDefinition() { + return { name: 'or', type: ExpressionType.INTEGER }; } } satisfies RowValueClause; } diff --git a/packages/sync-rules/src/types.ts b/packages/sync-rules/src/types.ts index e27489435..5aa8ea378 100644 --- a/packages/sync-rules/src/types.ts +++ b/packages/sync-rules/src/types.ts @@ -280,7 +280,7 @@ export interface ParameterValueClause { } export interface QuerySchema { - getType(table: string, column: string): ExpressionType; + getColumn(table: string, column: string): ColumnDefinition | undefined; getColumns(table: string): ColumnDefinition[]; } @@ -292,7 +292,7 @@ export interface QuerySchema { */ export interface RowValueClause { evaluate(tables: QueryParameters): SqliteValue; - getType(schema: QuerySchema): ExpressionType; + getColumnDefinition(schema: QuerySchema): ColumnDefinition | undefined; } /** @@ -322,7 +322,7 @@ export interface QueryBucketIdOptions { export interface SourceSchemaTable { table: string; - getType(column: string): ExpressionType | undefined; + getColumn(column: string): ColumnDefinition | undefined; getColumns(): ColumnDefinition[]; } export interface SourceSchema { diff --git a/packages/sync-rules/test/src/generate_schema.test.ts b/packages/sync-rules/test/src/generate_schema.test.ts new file mode 100644 index 000000000..d742a7f8b --- /dev/null +++ b/packages/sync-rules/test/src/generate_schema.test.ts @@ -0,0 +1,172 @@ +import { describe, expect, test } from 'vitest'; +import { + DEFAULT_TAG, + DartSchemaGenerator, + JsLegacySchemaGenerator, + SqlSyncRules, + StaticSchema, + TsSchemaGenerator +} from '../../src/index.js'; + +import { PARSE_OPTIONS } from './util.js'; + +describe('schema generation', () => { + const schema = new StaticSchema([ + { + tag: DEFAULT_TAG, + schemas: [ + { + name: 'test_schema', + tables: [ + { + name: 'assets', + columns: [ + { name: 'id', sqlite_type: 'text', internal_type: 'uuid' }, + { name: 'name', sqlite_type: 'text', internal_type: 'text' }, + { name: 'count', sqlite_type: 'integer', internal_type: 'int4' }, + { name: 'owner_id', sqlite_type: 'text', internal_type: 'uuid' } + ] + } + ] + } + ] + } + ]); + + const rules = SqlSyncRules.fromYaml( + ` +bucket_definitions: + mybucket: + data: + - SELECT * FROM assets as assets1 + - SELECT id, name, count FROM assets as assets2 + - SELECT id, owner_id as other_id, foo FROM assets as ASSETS2 + `, + PARSE_OPTIONS + ); + + test('dart', () => { + expect(new DartSchemaGenerator().generate(rules, schema)).toEqual(`Schema([ + Table('assets1', [ + Column.text('name'), + Column.integer('count'), + Column.text('owner_id') + ]), + Table('assets2', [ + Column.text('name'), + Column.integer('count'), + Column.text('other_id'), + Column.text('foo') + ]) +]); +`); + + expect(new DartSchemaGenerator().generate(rules, schema, { includeTypeComments: true })).toEqual(`Schema([ + Table('assets1', [ + Column.text('name'), // text + Column.integer('count'), // int4 + Column.text('owner_id') // uuid + ]), + Table('assets2', [ + Column.text('name'), // text + Column.integer('count'), // int4 + Column.text('other_id'), // uuid + Column.text('foo') + ]) +]); +`); + }); + + test('js legacy', () => { + expect(new JsLegacySchemaGenerator().generate(rules, schema)).toEqual(`new Schema([ + new Table({ + name: 'assets1', + columns: [ + new Column({ name: 'name', type: ColumnType.TEXT }), + new Column({ name: 'count', type: ColumnType.INTEGER }), + new Column({ name: 'owner_id', type: ColumnType.TEXT }) + ] + }), + new Table({ + name: 'assets2', + columns: [ + new Column({ name: 'name', type: ColumnType.TEXT }), + new Column({ name: 'count', type: ColumnType.INTEGER }), + new Column({ name: 'other_id', type: ColumnType.TEXT }), + new Column({ name: 'foo', type: ColumnType.TEXT }) + ] + }) +]) +`); + }); + + test('ts', () => { + expect(new TsSchemaGenerator().generate(rules, schema, {})).toEqual( + `import { column, Schema, Table } from '@powersync/web'; +// OR: import { column, Schema, Table } from '@powersync/react-native'; + +const assets1 = new Table( + { + // id column (text) is automatically included + name: column.text, + count: column.integer, + owner_id: column.text + }, + { indexes: {} } +); + +const assets2 = new Table( + { + // id column (text) is automatically included + name: column.text, + count: column.integer, + other_id: column.text, + foo: column.text + }, + { indexes: {} } +); + +export const AppSchema = new Schema({ + assets1, + assets2 +}); + +export type Database = (typeof AppSchema)['types']; +` + ); + + expect(new TsSchemaGenerator().generate(rules, schema, { includeTypeComments: true })).toEqual( + `import { column, Schema, Table } from '@powersync/web'; +// OR: import { column, Schema, Table } from '@powersync/react-native'; + +const assets1 = new Table( + { + // id column (text) is automatically included + name: column.text, // text + count: column.integer, // int4 + owner_id: column.text // uuid + }, + { indexes: {} } +); + +const assets2 = new Table( + { + // id column (text) is automatically included + name: column.text, // text + count: column.integer, // int4 + other_id: column.text, // uuid + foo: column.text + }, + { indexes: {} } +); + +export const AppSchema = new Schema({ + assets1, + assets2 +}); + +export type Database = (typeof AppSchema)['types']; +` + ); + }); +}); diff --git a/packages/sync-rules/test/src/sync_rules.test.ts b/packages/sync-rules/test/src/sync_rules.test.ts index c7e88ad46..26ff3adb3 100644 --- a/packages/sync-rules/test/src/sync_rules.test.ts +++ b/packages/sync-rules/test/src/sync_rules.test.ts @@ -786,110 +786,4 @@ bucket_definitions: expect(rules.errors).toEqual([]); }); - - test('schema generation', () => { - const schema = new StaticSchema([ - { - tag: DEFAULT_TAG, - schemas: [ - { - name: 'test_schema', - tables: [ - { - name: 'assets', - columns: [ - { name: 'id', pg_type: 'uuid' }, - { name: 'name', pg_type: 'text' }, - { name: 'count', pg_type: 'int4' }, - { name: 'owner_id', pg_type: 'uuid' } - ] - } - ] - } - ] - } - ]); - - const rules = SqlSyncRules.fromYaml( - ` -bucket_definitions: - mybucket: - data: - - SELECT * FROM assets as assets1 - - SELECT id, name, count FROM assets as assets2 - - SELECT id, owner_id as other_id, foo FROM assets as ASSETS2 - `, - PARSE_OPTIONS - ); - - expect(new DartSchemaGenerator().generate(rules, schema)).toEqual(`Schema([ - Table('assets1', [ - Column.text('name'), - Column.integer('count'), - Column.text('owner_id') - ]), - Table('assets2', [ - Column.text('name'), - Column.integer('count'), - Column.text('other_id'), - Column.text('foo') - ]) -]); -`); - - expect(new JsLegacySchemaGenerator().generate(rules, schema)).toEqual(`new Schema([ - new Table({ - name: 'assets1', - columns: [ - new Column({ name: 'name', type: ColumnType.TEXT }), - new Column({ name: 'count', type: ColumnType.INTEGER }), - new Column({ name: 'owner_id', type: ColumnType.TEXT }) - ] - }), - new Table({ - name: 'assets2', - columns: [ - new Column({ name: 'name', type: ColumnType.TEXT }), - new Column({ name: 'count', type: ColumnType.INTEGER }), - new Column({ name: 'other_id', type: ColumnType.TEXT }), - new Column({ name: 'foo', type: ColumnType.TEXT }) - ] - }) -]) -`); - - expect(new TsSchemaGenerator().generate(rules, schema)).toEqual( - `import { column, Schema, Table } from '@powersync/web'; -// OR: import { column, Schema, Table } from '@powersync/react-native'; - -const assets1 = new Table( - { - // id column (text) is automatically included - name: column.text, - count: column.integer, - owner_id: column.text - }, - { indexes: {} } -); - -const assets2 = new Table( - { - // id column (text) is automatically included - name: column.text, - count: column.integer, - other_id: column.text, - foo: column.text - }, - { indexes: {} } -); - -export const AppSchema = new Schema({ - assets1, - assets2 -}); - -export type Database = (typeof AppSchema)['types']; -` - ); - }); }); From 3da71a9e1fdb8c545efcc3a8ffcab981afe84fdc Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 25 Sep 2024 18:00:07 +0200 Subject: [PATCH 4/5] Explicit TableSchema type. --- .../src/api/PostgresRouteAPIAdapter.ts | 2 +- packages/types/src/definitions.ts | 71 ++++++++++--------- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts b/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts index 3862daec5..8a7e6e40c 100644 --- a/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts +++ b/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts @@ -282,7 +282,7 @@ GROUP BY schemaname, tablename, quoted_name` name: row.schemaname, tables: [] }); - const table: service_types.DatabaseSchema['tables'][0] = { + const table: service_types.TableSchema = { name: row.tablename, columns: [] as any[] }; diff --git a/packages/types/src/definitions.ts b/packages/types/src/definitions.ts index bdbf88d29..d387fb87e 100644 --- a/packages/types/src/definitions.ts +++ b/packages/types/src/definitions.ts @@ -100,47 +100,48 @@ export enum SqliteSchemaTypeText { numeric = 'numeric' } -export const DatabaseSchema = t.object({ +export const TableSchema = t.object({ name: t.string, - tables: t.array( + columns: t.array( t.object({ name: t.string, - columns: t.array( - t.object({ - name: t.string, - - /** - * Option 1: SQLite type flags - see ExpressionType.typeFlags. - * Option 2: SQLite type name in lowercase - 'text' | 'integer' | 'real' | 'numeric' | 'blob' | 'null' - */ - sqlite_type: t.number.or(t.Enum(SqliteSchemaTypeText)), - - /** - * Type name from the source database, e.g. "character varying(255)[]" - */ - internal_type: t.string, - - /** - * Description for the field if available. - */ - description: t.string.optional(), - - /** - * Full type name, e.g. "character varying(255)[]" - * @deprecated - use internal_type - */ - type: t.string.optional(), - - /** - * Internal postgres type, e.g. "varchar[]". - * @deprecated - use internal_type instead - */ - pg_type: t.string.optional() - }) - ) + + /** + * Option 1: SQLite type flags - see ExpressionType.typeFlags. + * Option 2: SQLite type name in lowercase - 'text' | 'integer' | 'real' | 'numeric' | 'blob' | 'null' + */ + sqlite_type: t.number.or(t.Enum(SqliteSchemaTypeText)), + + /** + * Type name from the source database, e.g. "character varying(255)[]" + */ + internal_type: t.string, + + /** + * Description for the field if available. + */ + description: t.string.optional(), + + /** + * Full type name, e.g. "character varying(255)[]" + * @deprecated - use internal_type + */ + type: t.string.optional(), + + /** + * Internal postgres type, e.g. "varchar[]". + * @deprecated - use internal_type instead + */ + pg_type: t.string.optional() }) ) }); +export type TableSchema = t.Encoded; + +export const DatabaseSchema = t.object({ + name: t.string, + tables: t.array(TableSchema) +}); export type DatabaseSchema = t.Encoded; export const InstanceSchema = t.object({ From 4ecaee26ada0be290575649cbeee153386f7b861 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 25 Sep 2024 18:04:46 +0200 Subject: [PATCH 5/5] Add changeset. --- .changeset/weak-cats-hug.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/weak-cats-hug.md diff --git a/.changeset/weak-cats-hug.md b/.changeset/weak-cats-hug.md new file mode 100644 index 000000000..df152386e --- /dev/null +++ b/.changeset/weak-cats-hug.md @@ -0,0 +1,5 @@ +--- +'@powersync/service-sync-rules': minor +--- + +Optionally include original types in generated schemas as a comment.