diff --git a/.changeset/giant-ladybugs-dress.md b/.changeset/giant-ladybugs-dress.md new file mode 100644 index 000000000..d6011c9b8 --- /dev/null +++ b/.changeset/giant-ladybugs-dress.md @@ -0,0 +1,8 @@ +--- +'@powersync/common': minor +--- + +- Add `trackPreviousValues` option on `Table` which sets `CrudEntry.previousValues` to previous values on updates. +- Add `trackMetadata` option on `Table` which adds a `_metadata` column that can be used for updates. + The configured metadata is available through `CrudEntry.metadata`. +- Add `ignoreEmptyUpdates` option which skips creating CRUD entries for updates that don't change any values. diff --git a/packages/common/src/client/sync/bucket/CrudEntry.ts b/packages/common/src/client/sync/bucket/CrudEntry.ts index ae862aca4..1d0133486 100644 --- a/packages/common/src/client/sync/bucket/CrudEntry.ts +++ b/packages/common/src/client/sync/bucket/CrudEntry.ts @@ -25,9 +25,11 @@ export type CrudEntryJSON = { type CrudEntryDataJSON = { data: Record; + old?: Record; op: UpdateType; type: string; id: string; + metadata?: string; }; /** @@ -62,6 +64,13 @@ export class CrudEntry { * Data associated with the change. */ opData?: Record; + + /** + * For tables where the `trackPreviousValues` option has been enabled, this tracks previous values for + * `UPDATE` and `DELETE` statements. + */ + previousValues?: Record; + /** * Table that contained the change. */ @@ -71,9 +80,26 @@ export class CrudEntry { */ transactionId?: number; + /** + * Client-side metadata attached with this write. + * + * This field is only available when the `trackMetadata` option was set to `true` when creating a table + * and the insert or update statement set the `_metadata` column. + */ + metadata?: string; + static fromRow(dbRow: CrudEntryJSON) { const data: CrudEntryDataJSON = JSON.parse(dbRow.data); - return new CrudEntry(parseInt(dbRow.id), data.op, data.type, data.id, dbRow.tx_id, data.data); + return new CrudEntry( + parseInt(dbRow.id), + data.op, + data.type, + data.id, + dbRow.tx_id, + data.data, + data.old, + data.metadata + ); } constructor( @@ -82,7 +108,9 @@ export class CrudEntry { table: string, id: string, transactionId?: number, - opData?: Record + opData?: Record, + previousValues?: Record, + metadata?: string ) { this.clientId = clientId; this.id = id; @@ -90,6 +118,8 @@ export class CrudEntry { this.opData = opData; this.table = table; this.transactionId = transactionId; + this.previousValues = previousValues; + this.metadata = metadata; } /** diff --git a/packages/common/src/db/schema/Schema.ts b/packages/common/src/db/schema/Schema.ts index 8d283a080..0aa175c3a 100644 --- a/packages/common/src/db/schema/Schema.ts +++ b/packages/common/src/db/schema/Schema.ts @@ -53,15 +53,7 @@ export class Schema { private convertToClassicTables(props: S) { return Object.entries(props).map(([name, table]) => { - const convertedTable = new Table({ - name, - columns: table.columns, - indexes: table.indexes, - localOnly: table.localOnly, - insertOnly: table.insertOnly, - viewName: table.viewNameOverride || name - }); - return convertedTable; + return table.copyWithName(name); }); } } diff --git a/packages/common/src/db/schema/Table.ts b/packages/common/src/db/schema/Table.ts index 5f91afaf2..ae729681d 100644 --- a/packages/common/src/db/schema/Table.ts +++ b/packages/common/src/db/schema/Table.ts @@ -10,16 +10,34 @@ import { Index } from './Index.js'; import { IndexedColumn } from './IndexedColumn.js'; import { TableV2 } from './TableV2.js'; -export interface TableOptions { +interface SharedTableOptions { + localOnly?: boolean; + insertOnly?: boolean; + viewName?: string; + trackPrevious?: boolean | TrackPreviousOptions; + trackMetadata?: boolean; + ignoreEmptyUpdates?: boolean; +} + +/** Whether to include previous column values when PowerSync tracks local changes. + * + * Including old values may be helpful for some backend connector implementations, which is + * why it can be enabled on per-table or per-columm basis. + */ +export interface TrackPreviousOptions { + /** When defined, a list of column names for which old values should be tracked. */ + columns?: string[]; + /** When enabled, only include values that have actually been changed by an update. */ + onlyWhenChanged?: boolean; +} + +export interface TableOptions extends SharedTableOptions { /** * The synced table name, matching sync rules */ name: string; columns: Column[]; indexes?: Index[]; - localOnly?: boolean; - insertOnly?: boolean; - viewName?: string; } export type RowType> = { @@ -30,17 +48,17 @@ export type RowType> = { export type IndexShorthand = Record; -export interface TableV2Options { +export interface TableV2Options extends SharedTableOptions { indexes?: IndexShorthand; - localOnly?: boolean; - insertOnly?: boolean; - viewName?: string; } export const DEFAULT_TABLE_OPTIONS = { indexes: [], insertOnly: false, - localOnly: false + localOnly: false, + trackPrevious: false, + trackMetadata: false, + ignoreEmptyUpdates: false }; export const InvalidSQLCharacters = /["'%,.#\s[\]]/; @@ -137,6 +155,13 @@ export class Table { } } + copyWithName(name: string): Table { + return new Table({ + ...this.options, + name + }); + } + private isTableV1(arg: TableOptions | Columns): arg is TableOptions { return 'columns' in arg && Array.isArray(arg.columns); } @@ -144,10 +169,9 @@ export class Table { private initTableV1(options: TableOptions) { this.options = { ...options, - indexes: options.indexes || [], - insertOnly: options.insertOnly ?? DEFAULT_TABLE_OPTIONS.insertOnly, - localOnly: options.localOnly ?? DEFAULT_TABLE_OPTIONS.localOnly + indexes: options.indexes || [] }; + this.applyDefaultOptions(); } private initTableV2(columns: Columns, options?: TableV2Options) { @@ -173,14 +197,26 @@ export class Table { name: '', columns: convertedColumns, indexes: convertedIndexes, - insertOnly: options?.insertOnly ?? DEFAULT_TABLE_OPTIONS.insertOnly, - localOnly: options?.localOnly ?? DEFAULT_TABLE_OPTIONS.localOnly, - viewName: options?.viewName + viewName: options?.viewName, + insertOnly: options?.insertOnly, + localOnly: options?.localOnly, + trackPrevious: options?.trackPrevious, + trackMetadata: options?.trackMetadata, + ignoreEmptyUpdates: options?.ignoreEmptyUpdates }; + this.applyDefaultOptions(); this._mappedColumns = columns; } + private applyDefaultOptions() { + this.options.insertOnly ??= DEFAULT_TABLE_OPTIONS.insertOnly; + this.options.localOnly ??= DEFAULT_TABLE_OPTIONS.localOnly; + this.options.trackPrevious ??= DEFAULT_TABLE_OPTIONS.trackPrevious; + this.options.trackMetadata ??= DEFAULT_TABLE_OPTIONS.trackMetadata; + this.options.ignoreEmptyUpdates ??= DEFAULT_TABLE_OPTIONS.ignoreEmptyUpdates; + } + get name() { return this.options.name; } @@ -212,11 +248,23 @@ export class Table { } get localOnly() { - return this.options.localOnly ?? false; + return this.options.localOnly!; } get insertOnly() { - return this.options.insertOnly ?? false; + return this.options.insertOnly!; + } + + get trackPrevious() { + return this.options.trackPrevious!; + } + + get trackMetadata() { + return this.options.trackMetadata!; + } + + get ignoreEmptyUpdates() { + return this.options.ignoreEmptyUpdates!; } get internalName() { @@ -250,6 +298,13 @@ export class Table { throw new Error(`Table has too many columns. The maximum number of columns is ${MAX_AMOUNT_OF_COLUMNS}.`); } + if (this.trackMetadata && this.localOnly) { + throw new Error(`Can't include metadata for local-only tables.`); + } + if (this.trackPrevious != false && this.localOnly) { + throw new Error(`Can't include old values for local-only tables.`); + } + const columnNames = new Set(); columnNames.add('id'); for (const column of this.columns) { @@ -286,11 +341,17 @@ export class Table { } toJSON() { + const trackPrevious = this.trackPrevious; + return { name: this.name, view_name: this.viewName, local_only: this.localOnly, insert_only: this.insertOnly, + include_old: trackPrevious && ((trackPrevious as any).columns ?? true), + include_old_only_when_changed: typeof trackPrevious == 'object' && trackPrevious.onlyWhenChanged == true, + include_metadata: this.trackMetadata, + ignore_empty_update: this.ignoreEmptyUpdates, columns: this.columns.map((c) => c.toJSON()), indexes: this.indexes.map((e) => e.toJSON(this)) }; diff --git a/packages/common/tests/db/schema/Schema.test.ts b/packages/common/tests/db/schema/Schema.test.ts index 51bbf5aa9..c9f303535 100644 --- a/packages/common/tests/db/schema/Schema.test.ts +++ b/packages/common/tests/db/schema/Schema.test.ts @@ -90,6 +90,10 @@ describe('Schema', () => { view_name: 'users', local_only: false, insert_only: false, + ignore_empty_update: false, + include_metadata: false, + include_old: false, + include_old_only_when_changed: false, columns: [ { name: 'name', type: 'TEXT' }, { name: 'age', type: 'INTEGER' } @@ -101,6 +105,10 @@ describe('Schema', () => { view_name: 'posts', local_only: false, insert_only: false, + ignore_empty_update: false, + include_metadata: false, + include_old: false, + include_old_only_when_changed: false, columns: [ { name: 'title', type: 'TEXT' }, { name: 'content', type: 'TEXT' } diff --git a/packages/common/tests/db/schema/Table.test.ts b/packages/common/tests/db/schema/Table.test.ts index 3f213517c..f5a51560e 100644 --- a/packages/common/tests/db/schema/Table.test.ts +++ b/packages/common/tests/db/schema/Table.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { Table } from '../../../src/db/schema/Table'; +import { Table, TableV2Options } from '../../../src/db/schema/Table'; import { column, Column, ColumnType } from '../../../src/db/schema/Column'; import { Index } from '../../../src/db/schema/Index'; import { IndexedColumn } from '../../../src/db/schema/IndexedColumn'; @@ -103,6 +103,10 @@ describe('Table', () => { view_name: 'customView', local_only: false, insert_only: false, + ignore_empty_update: false, + include_metadata: false, + include_old: false, + include_old_only_when_changed: false, columns: [ { name: 'name', type: 'TEXT' }, { name: 'age', type: 'INTEGER' } @@ -126,6 +130,27 @@ describe('Table', () => { expect(table.indexes[0].columns[0].ascending).toBe(false); }); + it('should handle options', () => { + function createTable(options: TableV2Options) { + return new Table({ name: column.text }, options); + } + + expect(createTable({}).toJSON().include_metadata).toBe(false); + expect(createTable({ trackMetadata: true }).toJSON().include_metadata).toBe(true); + + expect(createTable({ trackPrevious: true }).toJSON().include_old).toBe(true); + expect(createTable({ trackPrevious: true }).toJSON().include_old_only_when_changed).toBe(false); + + const complexIncldueOld = createTable({ trackPrevious: { + columns: ['foo', 'bar'], + onlyWhenChanged: true, + } }); + expect(complexIncldueOld.toJSON().include_old).toStrictEqual(['foo', 'bar']); + expect(complexIncldueOld.toJSON().include_old_only_when_changed).toBe(true); + + expect(createTable({ ignoreEmptyUpdates: true }).toJSON().ignore_empty_update).toBe(true); + }); + describe('validate', () => { it('should throw an error for invalid view names', () => { expect(() => { @@ -173,5 +198,36 @@ describe('Table', () => { }).validate() ).toThrowError('Invalid characters in column name: #invalid-name'); }); + + it('should throw an error for local-only tables with metadata', () => { + expect(() => + new Table( + { + name: column.text + }, + { localOnly: true, trackMetadata: true } + ).validate() + ).toThrowError("Can't include metadata for local-only tables."); + }); + + it('should throw an error for local-only tables tracking old values', () => { + expect(() => + new Table( + { + name: column.text + }, + { localOnly: true, trackPrevious: true } + ).validate() + ).toThrowError("Can't include old values for local-only tables."); + + expect(() => + new Table( + { + name: column.text + }, + { localOnly: true, trackPrevious: { onlyWhenChanged: false } } + ).validate() + ).toThrowError("Can't include old values for local-only tables."); + }); }); }); diff --git a/packages/common/tests/db/schema/TableV2.test.ts b/packages/common/tests/db/schema/TableV2.test.ts index 6e42f07b5..55895b63d 100644 --- a/packages/common/tests/db/schema/TableV2.test.ts +++ b/packages/common/tests/db/schema/TableV2.test.ts @@ -79,6 +79,10 @@ describe('TableV2', () => { view_name: 'customView', local_only: false, insert_only: false, + ignore_empty_update: false, + include_metadata: false, + include_old: false, + include_old_only_when_changed: false, columns: [ { name: 'name', type: 'TEXT' }, { name: 'age', type: 'INTEGER' } diff --git a/packages/node/tests/crud.test.ts b/packages/node/tests/crud.test.ts new file mode 100644 index 000000000..d9f35c57d --- /dev/null +++ b/packages/node/tests/crud.test.ts @@ -0,0 +1,98 @@ +import { expect } from 'vitest'; +import { column, Schema, Table } from '@powersync/common'; +import { databaseTest } from './utils'; + +databaseTest('include metadata', async ({ database }) => { + await database.init(); + const schema = new Schema({ + lists: new Table( + { + name: column.text + }, + { trackMetadata: true } + ) + }); + await database.updateSchema(schema); + await database.execute('INSERT INTO lists (id, name, _metadata) VALUES (uuid(), ?, ?);', ['entry', 'so meta']); + + const batch = await database.getNextCrudTransaction(); + expect(batch?.crud[0].metadata).toBe('so meta'); +}); + +databaseTest('include old values', async ({ database }) => { + await database.init(); + const schema = new Schema({ + lists: new Table( + { + name: column.text + }, + { trackPrevious: true } + ) + }); + await database.updateSchema(schema); + await database.execute('INSERT INTO lists (id, name) VALUES (uuid(), ?);', ['entry']); + await database.execute('DELETE FROM ps_crud;'); + await database.execute('UPDATE lists SET name = ?', ['new name']); + + const batch = await database.getNextCrudTransaction(); + expect(batch?.crud[0].previousValues).toStrictEqual({name: 'entry'}); +}); + +databaseTest('include old values with column filter', async ({ database }) => { + await database.init(); + const schema = new Schema({ + lists: new Table( + { + name: column.text, + content: column.text + }, + { trackPrevious: { columns: ['name'] } } + ) + }); + await database.updateSchema(schema); + await database.execute('INSERT INTO lists (id, name, content) VALUES (uuid(), ?, ?);', ['name', 'content']); + await database.execute('DELETE FROM ps_crud;'); + await database.execute('UPDATE lists SET name = ?, content = ?', ['new name', 'new content']); + + const batch = await database.getNextCrudTransaction(); + expect(batch?.crud[0].previousValues).toStrictEqual({name: 'name'}); +}); + +databaseTest('include old values when changed', async ({ database }) => { + await database.init(); + const schema = new Schema({ + lists: new Table( + { + name: column.text, + content: column.text + }, + { trackPrevious: { onlyWhenChanged: true } } + ) + }); + await database.updateSchema(schema); + await database.execute('INSERT INTO lists (id, name, content) VALUES (uuid(), ?, ?);', ['name', 'content']); + await database.execute('DELETE FROM ps_crud;'); + await database.execute('UPDATE lists SET name = ?', ['new name']); + + const batch = await database.getNextCrudTransaction(); + expect(batch?.crud[0].previousValues).toStrictEqual({name: 'name'}); +}); + +databaseTest('ignore empty update', async ({ database }) => { + await database.init(); + const schema = new Schema({ + lists: new Table( + { + name: column.text + }, + { ignoreEmptyUpdates: true } + ) + }); + await database.updateSchema(schema); + await database.execute('INSERT INTO lists (id, name) VALUES (uuid(), ?);', ['name']); + await database.execute('DELETE FROM ps_crud;'); + await database.execute('UPDATE lists SET name = ?', ['name']); + + const batch = await database.getNextCrudTransaction(); + expect(batch).toBeNull(); +});