fix(policy): run pg/sqlite tests, misc dual db compatibility fixes

.github/workflows/build-test.yml

@@ -34,6 +34,7 @@ jobs:
         strategy:
             matrix:
                 node-version: [20.x]
+                provider: [sqlite, postgresql]
 
         steps:
             - name: Checkout
@@ -76,4 +77,4 @@ jobs:
               run: pnpm run lint
 
             - name: Test
-              run: pnpm run test
+              run: TEST_DB_PROVIDER=${{ matrix.provider }} pnpm run test

package.json

@@ -7,7 +7,7 @@
         "build": "turbo run build",
         "watch": "turbo run watch build",
         "lint": "turbo run lint",
-        "test": "turbo run test",
+        "test": "vitest run",
         "format": "prettier --write \"**/*.{ts,tsx,md}\"",
         "pr": "gh pr create --fill-first --base dev",
         "merge-main": "gh pr create --title \"merge dev to main\" --body \"\" --base main --head dev",

packages/language/res/stdlib.zmodel

@@ -123,8 +123,10 @@ function future(): Any {
 } @@@expressionContext([AccessPolicy])
 
 /**
- * If the field value contains the search string. By default, the search is case-sensitive,
- * but you can override the behavior with the "caseInSensitive" argument.
+ * Checks if the field value contains the search string. By default, the search is case-sensitive, and
+ * "LIKE" operator is used to match. If `caseInSensitive` is true, "ILIKE" operator is used if
+ * supported, otherwise it still falls back to "LIKE" and delivers whatever the database's 
+ * behavior is.
  */
 function contains(field: String, search: String, caseInSensitive: Boolean?): Boolean {
 } @@@expressionContext([AccessPolicy, ValidationRule])
@@ -136,15 +138,21 @@ function search(field: String, search: String): Boolean {
 } @@@expressionContext([AccessPolicy])
 
 /**
- * If the field value starts with the search string
+ * Checks the field value starts with the search string. By default, the search is case-sensitive, and
+ * "LIKE" operator is used to match. If `caseInSensitive` is true, "ILIKE" operator is used if
+ * supported, otherwise it still falls back to "LIKE" and delivers whatever the database's 
+ * behavior is.
  */
-function startsWith(field: String, search: String): Boolean {
+function startsWith(field: String, search: String, caseInSensitive: Boolean?): Boolean {
 } @@@expressionContext([AccessPolicy, ValidationRule])
 
 /**
- * If the field value ends with the search string
+ * Checks if the field value ends with the search string. By default, the search is case-sensitive, and
+ * "LIKE" operator is used to match. If `caseInSensitive` is true, "ILIKE" operator is used if
+ * supported, otherwise it still falls back to "LIKE" and delivers whatever the database's 
+ * behavior is.
  */
-function endsWith(field: String, search: String): Boolean {
+function endsWith(field: String, search: String, caseInSensitive: Boolean?): Boolean {
 } @@@expressionContext([AccessPolicy, ValidationRule])
 
 /**

packages/language/test/delegate.test.ts

@@ -5,6 +5,11 @@ import { loadSchema, loadSchemaWithError } from './utils';
 describe('Delegate Tests', () => {
     it('supports inheriting from delegate', async () => {
         const model = await loadSchema(`
+        datasource db {
+            provider = 'sqlite'
+            url      = 'file:./dev.db'
+        }
+
         model A {
             id Int @id @default(autoincrement())
             x String
@@ -24,6 +29,11 @@ describe('Delegate Tests', () => {
     it('rejects inheriting from non-delegate models', async () => {
         await loadSchemaWithError(
             `
+        datasource db {
+            provider = 'sqlite'
+            url      = 'file:./dev.db'
+        }
+
         model A {
             id Int @id @default(autoincrement())
             x String
@@ -40,6 +50,11 @@ describe('Delegate Tests', () => {
     it('can detect cyclic inherits', async () => {
         await loadSchemaWithError(
             `
+        datasource db {
+            provider = 'sqlite'
+            url      = 'file:./dev.db'
+        }
+
         model A extends B {
             x String
             @@delegate(x)
@@ -57,6 +72,11 @@ describe('Delegate Tests', () => {
     it('can detect duplicated fields from base model', async () => {
         await loadSchemaWithError(
             `
+        datasource db {
+            provider = 'sqlite'
+            url      = 'file:./dev.db'
+        }
+
         model A {
             id String @id
             x String
@@ -74,6 +94,11 @@ describe('Delegate Tests', () => {
     it('can detect duplicated attributes from base model', async () => {
         await loadSchemaWithError(
             `
+        datasource db {
+            provider = 'sqlite'
+            url      = 'file:./dev.db'
+        }
+
         model A {
             id String @id
             x String

packages/language/test/import.test.ts

@@ -12,6 +12,11 @@
         fs.writeFileSync(
             path.join(name, 'a.zmodel'),
             `
+datasource db {
+  provider = 'sqlite'
+  url      = 'file:./dev.db'
+}
+
 model A {
   id Int @id
   name String
@@ -48,6 +53,12 @@
             path.join(name, 'b.zmodel'),
             `
 import './a'
+
+datasource db {
+  provider = 'sqlite'
+  url      = 'file:./dev.db'
+}
+
 model User {
   id Int @id
   role Role
@@ -56,7 +67,7 @@
         );
 
         const model = await expectLoaded(path.join(name, 'b.zmodel'));
         expect((model.declarations[0] as DataModel).fields[1].type.reference?.ref?.name).toBe('Role');
     });
 
     it('supports cyclic imports', async () => {
@@ -65,6 +76,12 @@
             path.join(name, 'a.zmodel'),
             `
 import './b'
+
+datasource db {
+  provider = 'sqlite'
+  url      = 'file:./dev.db'
+}
+
 model A {
   id Int @id
   b B?
@@ -86,7 +103,7 @@
         const modelB = await expectLoaded(path.join(name, 'b.zmodel'));
         expect((modelB.declarations[0] as DataModel).fields[1].type.reference?.ref?.name).toBe('A');
         const modelA = await expectLoaded(path.join(name, 'a.zmodel'));
         expect((modelA.declarations[0] as DataModel).fields[1].type.reference?.ref?.name).toBe('B');
     });
 
     async function expectLoaded(file: string) {

packages/language/test/mixin.test.ts

@@ -5,6 +5,11 @@ import { DataModel, TypeDef } from '../src/ast';
 describe('Mixin Tests', () => {
     it('supports model mixing types to Model', async () => {
         const model = await loadSchema(`
+        datasource db {
+            provider = 'sqlite'
+            url      = 'file:./dev.db'
+        }
+
         type A {
             x String
         }
@@ -25,6 +30,11 @@ describe('Mixin Tests', () => {
 
     it('supports model mixing types to type', async () => {
         const model = await loadSchema(`
+        datasource db {
+            provider = 'sqlite'
+            url      = 'file:./dev.db'
+        }
+
         type A {
             x String
         }
@@ -52,6 +62,11 @@ describe('Mixin Tests', () => {
     it('can detect cyclic mixins', async () => {
         await loadSchemaWithError(
             `
+        datasource db {
+            provider = 'sqlite'
+            url      = 'file:./dev.db'
+        }
+
         type A with B {
             x String
         }

packages/runtime/package.json

@@ -8,6 +8,8 @@
         "watch": "tsup-node --watch",
         "lint": "eslint src --ext ts",
         "test": "vitest run && pnpm test:typecheck",
+        "test:sqlite": "TEST_DB_PROVIDER=sqlite vitest run",
+        "test:postgresql": "TEST_DB_PROVIDER=postgresql vitest run",
         "test:generate": "tsx test/scripts/generate.ts",
         "test:typecheck": "tsc --project tsconfig.test.json",
         "pack": "pnpm pack"

packages/runtime/src/client/client-impl.ts

@@ -61,7 +61,7 @@ export class ClientImpl<Schema extends SchemaDef> {
         executor?: QueryExecutor,
     ) {
         this.$schema = schema;
-        this.$options = options ?? ({} as ClientOptions<Schema>);
+        this.$options = options;
 
         this.$options.functions = {
             ...BuiltinFunctions,
@@ -326,7 +326,7 @@ export class ClientImpl<Schema extends SchemaDef> {
 
 function createClientProxy<Schema extends SchemaDef>(client: ClientImpl<Schema>): ClientImpl<Schema> {
     const inputValidator = new InputValidator(client.$schema);
-    const resultProcessor = new ResultProcessor(client.$schema);
+    const resultProcessor = new ResultProcessor(client.$schema, client.$options);
 
     return new Proxy(client, {
         get: (target, prop, receiver) => {

packages/runtime/src/client/crud/dialects/base-dialect.ts

@@ -45,6 +45,10 @@ export abstract class BaseCrudDialect<Schema extends SchemaDef> {
         return value;
     }
 
+    transformOutput(value: unknown, _type: BuiltinType) {
+        return value;
+    }
+
     // #region common query builders
 
     buildSelectModel(eb: ExpressionBuilder<any, any>, model: string, modelAlias: string) {
@@ -1255,5 +1259,15 @@ export abstract class BaseCrudDialect<Schema extends SchemaDef> {
      */
     abstract get supportInsertWithDefault(): boolean;
 
+    /**
+     * Gets the SQL column type for the given field definition.
+     */
+    abstract getFieldSqlType(fieldDef: FieldDef): string;
+
+    /*
+     * Gets the string casing behavior for the dialect.
+     */
+    abstract getStringCasingBehavior(): { supportsILike: boolean; likeCaseSensitive: boolean };
+
     // #endregion
 }

packages/runtime/src/client/crud/dialects/postgresql.ts

@@ -1,4 +1,5 @@
 import { invariant } from '@zenstackhq/common-helpers';
+import Decimal from 'decimal.js';
 import {
     sql,
     type Expression,
@@ -11,6 +12,8 @@ import { match } from 'ts-pattern';
 import type { BuiltinType, FieldDef, GetModels, SchemaDef } from '../../../schema';
 import { DELEGATE_JOINED_FIELD_PREFIX } from '../../constants';
 import type { FindArgs } from '../../crud-types';
+import { QueryError } from '../../errors';
+import type { ClientOptions } from '../../options';
 import {
     buildJoinPairs,
     getDelegateDescendantModels,
@@ -23,6 +26,10 @@ import {
 import { BaseCrudDialect } from './base-dialect';
 
 export class PostgresCrudDialect<Schema extends SchemaDef> extends BaseCrudDialect<Schema> {
+    constructor(schema: Schema, options: ClientOptions<Schema>) {
+        super(schema, options);
+    }
+
     override get provider() {
         return 'postgresql' as const;
     }
@@ -44,13 +51,69 @@ export class PostgresCrudDialect<Schema extends SchemaDef> extends BaseCrudDiale
         } else {
             return match(type)
                 .with('DateTime', () =>
-                    value instanceof Date ? value : typeof value === 'string' ? new Date(value) : value,
+                    value instanceof Date
+                        ? value.toISOString()
+                        : typeof value === 'string'
+                          ? new Date(value).toISOString()
+                          : value,
                 )
                 .with('Decimal', () => (value !== null ? value.toString() : value))
                 .otherwise(() => value);
         }
     }
 
+    override transformOutput(value: unknown, type: BuiltinType) {
+        if (value === null || value === undefined) {
+            return value;
+        }
+        return match(type)
+            .with('DateTime', () => this.transformOutputDate(value))
+            .with('Bytes', () => this.transformOutputBytes(value))
+            .with('BigInt', () => this.transformOutputBigInt(value))
+            .with('Decimal', () => this.transformDecimal(value))
+            .otherwise(() => super.transformOutput(value, type));
+    }
+
+    private transformOutputBigInt(value: unknown) {
+        if (typeof value === 'bigint') {
+            return value;
+        }
+        invariant(
+            typeof value === 'string' || typeof value === 'number',
+            `Expected string or number, got ${typeof value}`,
+        );
+        return BigInt(value);
+    }
+
+    private transformDecimal(value: unknown) {
+        if (value instanceof Decimal) {
+            return value;
+        }
+        invariant(
+            typeof value === 'string' || typeof value === 'number' || value instanceof Decimal,
+            `Expected string, number or Decimal, got ${typeof value}`,
+        );
+        return new Decimal(value);
+    }
+
+    private transformOutputDate(value: unknown) {
+        if (typeof value === 'string') {
+            return new Date(value);
+        } else if (value instanceof Date && this.options.fixPostgresTimezone !== false) {
+            // SPECIAL NOTES:
+            // node-pg has a terrible quirk that it returns the date value in local timezone
+            // as a `Date` object although for `DateTime` field the data in DB is stored in UTC
+            // see: https://github.com/brianc/node-postgres/issues/429
+            return new Date(value.getTime() - value.getTimezoneOffset() * 60 * 1000);
+        } else {
+            return value;
+        }
+    }
+
+    private transformOutputBytes(value: unknown) {
+        return Buffer.isBuffer(value) ? Uint8Array.from(value) : value;
+    }
+
     override buildRelationSelection(
         query: SelectQueryBuilder<any, any, any>,
         model: string,
@@ -370,4 +433,42 @@ export class PostgresCrudDialect<Schema extends SchemaDef> extends BaseCrudDiale
     override get supportInsertWithDefault() {
         return true;
     }
+
+    override getFieldSqlType(fieldDef: FieldDef) {
+        // TODO: respect `@db.x` attributes
+        if (fieldDef.relation) {
+            throw new QueryError('Cannot get SQL type of a relation field');
+        }
+
+        let result: string;
+
+        if (this.schema.enums?.[fieldDef.type]) {
+            // enums are treated as text
+            result = 'text';
+        } else {
+            result = match(fieldDef.type)
+                .with('String', () => 'text')
+                .with('Boolean', () => 'boolean')
+                .with('Int', () => 'integer')
+                .with('BigInt', () => 'bigint')
+                .with('Float', () => 'double precision')
+                .with('Decimal', () => 'decimal')
+                .with('DateTime', () => 'timestamp')
+                .with('Bytes', () => 'bytea')
+                .with('Json', () => 'jsonb')
+                // fallback to text
+                .otherwise(() => 'text');
+        }
+
+        if (fieldDef.array) {
+            result += '[]';
+        }
+
+        return result;
+    }
+
+    override getStringCasingBehavior() {
+        // Postgres `LIKE` is case-sensitive, `ILIKE` is case-insensitive
+        return { supportsILike: true, likeCaseSensitive: true };
+    }
 }