Mark interfaces as experimental. Add warning if schema changed while triggers are active. Fix DELETE triggers not filtering columns with JSON fragment. Add throttleMs as option.

Author: stevensJourney

packages/common/src/client/AbstractPowerSyncDatabase.ts

@@ -194,9 +194,11 @@ export abstract class AbstractPowerSyncDatabase extends BaseObserver<PowerSyncDB
   protected runExclusiveMutex: Mutex;
 
   /**
+   * @experimental
    * Allows creating SQLite triggers which can be used to track various operations on SQLite tables.
    */
   readonly triggers: TriggerManager;
+
   logger: ILogger;
 
   constructor(options: PowerSyncDatabaseOptionsWithDBAdapter);

packages/common/src/client/triggers/TriggerManager.ts

@@ -2,6 +2,7 @@ import { LockContext } from '../../db/DBAdapter.js';
 
 /**
  * SQLite operations to track changes for with {@link TriggerManager}
+ * @experimental
  */
 export enum DiffTriggerOperation {
   INSERT = 'INSERT',
@@ -10,6 +11,7 @@ export enum DiffTriggerOperation {
 }
 
 /**
+ * @experimental
  * Diffs created by {@link TriggerManager#createDiffTrigger} are stored in a temporary table.
  * This is the base record structure for all diff records.
  */
@@ -30,6 +32,7 @@ export interface BaseTriggerDiffRecord {
 }
 
 /**
+ * @experimental
  * Represents a diff record for a SQLite UPDATE operation.
  * This record contains the new value and optionally the previous value.
  * Values are stored as JSON strings.
@@ -47,6 +50,7 @@ export interface TriggerDiffUpdateRecord extends BaseTriggerDiffRecord {
 }
 
 /**
+ * @experimental
  * Represents a diff record for a SQLite INSERT operation.
  * This record contains the new value represented as a JSON string.
  */
@@ -59,6 +63,7 @@ export interface TriggerDiffInsertRecord extends BaseTriggerDiffRecord {
 }
 
 /**
+ * @experimental
  * Represents a diff record for a SQLite DELETE operation.
  * This record contains the new value represented as a JSON string.
  */
@@ -71,6 +76,7 @@ export interface TriggerDiffDeleteRecord extends BaseTriggerDiffRecord {
 }
 
 /**
+ * @experimental
  * Diffs created by {@link TriggerManager#createDiffTrigger} are stored in a temporary table.
  * This is the record structure for all diff records.
  *
@@ -85,6 +91,7 @@ export interface TriggerDiffDeleteRecord extends BaseTriggerDiffRecord {
 export type TriggerDiffRecord = TriggerDiffUpdateRecord | TriggerDiffInsertRecord | TriggerDiffDeleteRecord;
 
 /**
+ * @experimental
  * Querying the DIFF table directly with {@link TriggerDiffHandlerContext#withExtractedDiff} will return records
  * with the tracked columns extracted from the JSON value.
  * This type represents the structure of such records.
@@ -101,6 +108,7 @@ export type ExtractedTriggerDiffRecord<T> = T & {
 };
 
 /**
+ * @experimental
  * Hooks used in the creation of a table diff trigger.
  */
 export interface TriggerCreationHooks {
@@ -161,6 +169,7 @@ interface BaseCreateDiffTriggerOptions {
 }
 
 /**
+ * @experimental
  * Options for {@link TriggerManager#createDiffTrigger}.
  */
 export interface CreateDiffTriggerOptions extends BaseCreateDiffTriggerOptions {
@@ -173,11 +182,13 @@ export interface CreateDiffTriggerOptions extends BaseCreateDiffTriggerOptions {
 }
 
 /**
+ * @experimental
  * Callback to drop a trigger after it has been created.
  */
 export type TriggerRemoveCallback = () => Promise<void>;
 
 /**
+ * @experimental
  * Context for the `onChange` handler provided to {@link TriggerManager#trackTableDiff}.
  */
 export interface TriggerDiffHandlerContext extends LockContext {
@@ -225,6 +236,9 @@ export interface TriggerDiffHandlerContext extends LockContext {
    * This is similar to {@link withDiff} but extracts the row columns from the tracked JSON value. The diff operation
    * data is aliased as `__` columns to avoid column conflicts.
    *
+   * For {@link DiffTriggerOperation#DELETE} operations the previous_value columns are extracted for convenience.
+   *
+   *
    * ```sql
    * CREATE TEMP TABLE DIFF (
    *       id TEXT,
@@ -251,6 +265,7 @@ export interface TriggerDiffHandlerContext extends LockContext {
 }
 
 /**
+ * @experimental
  * Options for tracking changes to a table with {@link TriggerManager#trackTableDiff}.
  */
 export interface TrackDiffOptions extends BaseCreateDiffTriggerOptions {
@@ -260,10 +275,20 @@ export interface TrackDiffOptions extends BaseCreateDiffTriggerOptions {
    * Diff items are automatically cleared after the handler is invoked.
    */
   onChange: (context: TriggerDiffHandlerContext) => Promise<void>;
+
+  /**
+   * The minimum interval, in milliseconds, between {@link onChange} invocations.
+   *  @default {@link DEFAULT_WATCH_THROTTLE_MS}
+   */
+  throttleMs?: number;
 }
 
+/**
+ * @experimental
+ */
 export interface TriggerManager {
   /**
+   * @experimental
    * Creates a temporary trigger which tracks changes to a source table
    * and writes changes to a destination table.
    * The temporary destination table is created internally and will be dropped when the trigger is removed.
@@ -301,6 +326,7 @@ export interface TriggerManager {
   createDiffTrigger(options: CreateDiffTriggerOptions): Promise<TriggerRemoveCallback>;
 
   /**
+   * @experimental
    * Tracks changes for a table. Triggering a provided handler on changes.
    * Uses {@link createDiffTrigger} internally to create a temporary destination table.
    *

packages/common/src/client/triggers/TriggerManagerImpl.ts

@@ -1,6 +1,7 @@
 import { LockContext } from '../../db/DBAdapter.js';
 import { Schema } from '../../db/schema/Schema.js';
 import { type AbstractPowerSyncDatabase } from '../AbstractPowerSyncDatabase.js';
+import { DEFAULT_WATCH_THROTTLE_MS } from '../watched/WatchedQuery.js';
 import {
   CreateDiffTriggerOptions,
   DiffTriggerOperation,
@@ -103,16 +104,26 @@ export class TriggerManagerImpl implements TriggerManager {
       }
     };
 
+    const disposeWarningListener = this.db.registerListener({
+      schemaChanged: () => {
+        this.db.logger.warn(
+          `The PowerSync schema has changed while previously configured triggers are still operational. This might cause unexpected results.`
+        );
+      }
+    });
+
     /**
      * Declare the cleanup function early since if any of the init steps fail,
      * we need to ensure we can cleanup the created resources.
      * We unfortunately cannot rely on transaction rollback.
      */
-    const cleanup = async () =>
-      this.db.writeLock(async (tx) => {
+    const cleanup = async () => {
+      disposeWarningListener();
+      return this.db.writeLock(async (tx) => {
         await this.removeTriggers(tx, triggerIds);
         await tx.execute(/* sql */ `DROP TABLE IF EXISTS ${destination};`);
       });
+    };
 
     const setup = async (tx: LockContext) => {
       // Allow user code to execute in this lock context before the trigger is created.
@@ -187,7 +198,7 @@ export class TriggerManagerImpl implements TriggerManager {
               OLD.id,
               'DELETE',
               strftime ('%Y-%m-%dT%H:%M:%fZ', 'now'),
-              OLD.data
+              ${jsonFragment('OLD')}
             );
 
           END;
@@ -205,7 +216,7 @@ export class TriggerManagerImpl implements TriggerManager {
   }
 
   async trackTableDiff(options: TrackDiffOptions): Promise<TriggerRemoveCallback> {
-    const { source, when, columns, operations, hooks } = options;
+    const { source, when, columns, operations, hooks, throttleMs = DEFAULT_WATCH_THROTTLE_MS } = options;
 
     await this.db.waitForReady();
 
@@ -284,7 +295,7 @@ export class TriggerManagerImpl implements TriggerManager {
           });
         }
       },
-      { tables: [destination], signal: abortController.signal }
+      { tables: [destination], signal: abortController.signal, throttleMs }
     );
 
     try {

packages/node/tests/trigger.test.ts

@@ -1,7 +1,10 @@
 import {
+  column,
   DiffTriggerOperation,
   ExtractedTriggerDiffRecord,
   sanitizeUUID,
+  Schema,
+  Table,
   TriggerDiffRecord,
   whenClause
 } from '@powersync/common';
@@ -425,7 +428,7 @@ describe('Triggers', () => {
     );
   });
 
-  databaseTest('Should allow tracking 0 columns', { timeout: 1000 }, async ({ database }) => {
+  databaseTest('Should allow tracking 0 columns', async ({ database }) => {
     /**
      * Tracks the ids of todos reported via the trigger
      */
@@ -499,4 +502,96 @@ describe('Triggers', () => {
       { timeout: 1000, interval: 100 }
     );
   });
+
+  databaseTest('Should only track listed columns', async ({ database }) => {
+    const newSchema = new Schema({
+      todos: new Table({
+        content: column.text,
+        columnA: column.text,
+        columnB: column.text
+      })
+    });
+    await database.updateSchema(newSchema);
+
+    type NewTodoRecord = (typeof newSchema)['types']['todos'];
+
+    const changes: ExtractedTriggerDiffRecord<NewTodoRecord>[] = [];
+
+    const createTodo = async (content: string, columnA = 'A', columnB = 'B'): Promise<NewTodoRecord> => {
+      // Create todos for both lists
+      return database.writeLock(async (tx) => {
+        const result = await tx.execute(
+          /* sql */ `
+            INSERT INTO
+              todos (id, content, columnA, columnB)
+            VALUES
+              (uuid (), ?, ?, ?) RETURNING id
+          `,
+          [content, columnA, columnB]
+        );
+        return result.rows?._array?.[0];
+      });
+    };
+
+    await database.triggers.trackTableDiff({
+      source: 'todos',
+      operations: [DiffTriggerOperation.INSERT, DiffTriggerOperation.UPDATE, DiffTriggerOperation.DELETE],
+      columns: ['columnA'],
+      onChange: async (context) => {
+        // Fetches the content of the records at the time of the operation
+        const extractedDiff = await context.withExtractedDiff<ExtractedTriggerDiffRecord<NewTodoRecord>>(/* sql */ `
+          SELECT
+            *
+          FROM
+            DIFF
+        `);
+        changes.push(...extractedDiff);
+      }
+    });
+
+    await createTodo('todo 1');
+    await createTodo('todo 2');
+    await createTodo('todo 3');
+
+    // Do an update operation to ensure only the tracked columns of updated values are stored
+    await database.execute(/* sql */ `
+      UPDATE todos
+      SET
+        content = 'todo 4'
+      WHERE
+        content = 'todo 3'
+    `);
+
+    // Do a delete operation to ensure only the tracked columns of updated values are stored
+    await database.execute(/* sql */ `
+      DELETE FROM todos
+      WHERE
+        content = 'todo 4'
+    `);
+
+    // Wait for all the changes to be recorded
+    await vi.waitFor(
+      async () => {
+        expect(changes.length).toEqual(5);
+      },
+      { timeout: 1000, interval: 100 }
+    );
+
+    // Inserts should only have the tracked columns
+    expect(changes[0].__operation).eq(DiffTriggerOperation.INSERT);
+    expect(changes[1].__operation).eq(DiffTriggerOperation.INSERT);
+    expect(changes[2].__operation).eq(DiffTriggerOperation.INSERT);
+    // Should not track this column
+    expect(changes[0].columnB).toBeUndefined();
+
+    expect(changes[3].__operation).eq(DiffTriggerOperation.UPDATE);
+    expect(changes[3].columnB).toBeUndefined();
+    expect(changes[3].__previous_value).toBeDefined();
+    expect(Object.keys(JSON.parse(changes[3].__previous_value))).to.deep.equal(['columnA']);
+
+    // For deletes we extract the old value for convenience (there is no new value)
+    expect(changes[4].__operation).eq(DiffTriggerOperation.DELETE);
+    expect(changes[4].columnB).toBeUndefined();
+    expect(changes[4].__previous_value).toBeNull();
+  });
 });

pnpm-workspace.yaml

@@ -1,5 +1,11 @@
 packages:
-  - "demos/*"
-  - "packages/*"
-  - "tools/*"
-  - "docs/"
+  - demos/*
+  - packages/*
+  - tools/*
+  - docs/
+
+onlyBuiltDependencies:
+  - '@journeyapps/wa-sqlite'
+  - esbuild
+  - react-native-elements
+  - vue-demi