diff --git a/.changeset/big-books-remember.md b/.changeset/big-books-remember.md new file mode 100644 index 000000000..97f75dc27 --- /dev/null +++ b/.changeset/big-books-remember.md @@ -0,0 +1,7 @@ +--- +'@powersync/service-module-mongodb-storage': minor +'@powersync/service-core-tests': minor +'@powersync/service-core': minor +--- + +Updated BucketStorageFactory to use AsyncDisposable diff --git a/.changeset/green-trainers-yell.md b/.changeset/green-trainers-yell.md new file mode 100644 index 000000000..a6b35c77a --- /dev/null +++ b/.changeset/green-trainers-yell.md @@ -0,0 +1,8 @@ +--- +'@powersync/service-module-postgres-storage': minor +'@powersync/service-module-postgres': minor +'@powersync/lib-service-postgres': minor +'@powersync/service-core': minor +--- + +Initial release of Postgres bucket storage. diff --git a/.changeset/heavy-llamas-move.md b/.changeset/heavy-llamas-move.md new file mode 100644 index 000000000..26ff6abfd --- /dev/null +++ b/.changeset/heavy-llamas-move.md @@ -0,0 +1,6 @@ +--- +'@powersync/service-core-tests': minor +'@powersync/lib-services-framework': minor +--- + +Improved migrations logic. Up migrations can be executed correctly after down migrations. diff --git a/.env.template b/.env.template index a3c5fd5d0..4b9d70da3 100644 --- a/.env.template +++ b/.env.template @@ -1,3 +1,5 @@ # Connections for tests MONGO_TEST_URL="mongodb://localhost:27017/powersync_test" -PG_TEST_URL="postgres://postgres:postgres@localhost:5432/powersync_test" \ No newline at end of file +PG_TEST_URL="postgres://postgres:postgres@localhost:5432/powersync_test" +# Note that this uses a separate server on a different port +PG_STORAGE_TEST_URL="postgres://postgres:postgres@localhost:5431/powersync_storage_test" \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9f5fbd055..e2d3a3358 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -104,6 +104,18 @@ jobs: -d postgres:${{ matrix.postgres-version }} \ -c wal_level=logical + - name: Start PostgreSQL (Storage) + run: | + docker run \ + --health-cmd pg_isready \ + --health-interval 10s \ + --health-timeout 5s \ + --health-retries 5 \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=powersync_storage_test \ + -p 5431:5432 \ + -d postgres:${{ matrix.postgres-version }} + - name: Start MongoDB uses: supercharge/mongodb-github-action@1.8.0 with: @@ -176,6 +188,18 @@ jobs: mongodb-version: '6.0' mongodb-replica-set: test-rs + - name: Start PostgreSQL (Storage) + run: | + docker run \ + --health-cmd pg_isready \ + --health-interval 10s \ + --health-timeout 5s \ + --health-retries 5 \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=powersync_storage_test \ + -p 5431:5432 \ + -d postgres:16 + - name: Setup NodeJS uses: actions/setup-node@v4 with: @@ -229,6 +253,18 @@ jobs: mongodb-version: ${{ matrix.mongodb-version }} mongodb-replica-set: test-rs + - name: Start PostgreSQL (Storage) + run: | + docker run \ + --health-cmd pg_isready \ + --health-interval 10s \ + --health-timeout 5s \ + --health-retries 5 \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=powersync_storage_test \ + -p 5431:5432 \ + -d postgres:16 + - name: Setup Node.js uses: actions/setup-node@v4 with: diff --git a/.prettierrc b/.prettierrc index dd651a512..2b98ddbe1 100644 --- a/.prettierrc +++ b/.prettierrc @@ -4,5 +4,9 @@ "tabWidth": 2, "useTabs": false, "printWidth": 120, - "trailingComma": "none" + "trailingComma": "none", + "plugins": ["prettier-plugin-embed", "prettier-plugin-sql"], + "embeddedSqlTags": ["sql", "db.sql", "this.db.sql"], + "language": "postgresql", + "keywordCase": "upper" } diff --git a/libs/lib-postgres/CHANGELOG.md b/libs/lib-postgres/CHANGELOG.md new file mode 100644 index 000000000..3a699be57 --- /dev/null +++ b/libs/lib-postgres/CHANGELOG.md @@ -0,0 +1 @@ +# @powersync/lib-service-postgres diff --git a/libs/lib-postgres/LICENSE b/libs/lib-postgres/LICENSE new file mode 100644 index 000000000..c8efd46cc --- /dev/null +++ b/libs/lib-postgres/LICENSE @@ -0,0 +1,67 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2023-2024 Journey Mobile, Inc. + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under these Terms and Conditions, as indicated by our inclusion of these Terms and Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, Redistribution and Trademark clauses below, we hereby grant you the right to use, copy, modify, create derivative works, publicly perform, publicly display and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use means making the Software available to others in a commercial product or service that: + +1. substitutes for the Software; +2. substitutes for any other product or service we offer using the Software that exists as of the date we make the Software available; or +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; +2. for non-commercial education; +3. for non-commercial research; and +4. in connection with professional services that you provide to a licensee using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our patents, the license grant above includes a license under our patents. If you make a claim against any party that the Software infringes or contributes to the infringement of any patent, then your patent license to the Software ends immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of the Software. +If you redistribute any copies, modifications or derivatives of the Software, you must include a copy of or a link to these Terms and Conditions and not remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of the Software, you have no right under these Terms and Conditions to use our trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under the Apache License, Version 2.0 that is effective on the second anniversary of the date we make the Software available. On or after that date, you may use the Software under the Apache License, Version 2.0, in which case the following will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/libs/lib-postgres/README.md b/libs/lib-postgres/README.md new file mode 100644 index 000000000..1cf2cf1de --- /dev/null +++ b/libs/lib-postgres/README.md @@ -0,0 +1,3 @@ +# PowerSync Service Postgres + +Library for common Postgres logic used in the PowerSync service. diff --git a/libs/lib-postgres/package.json b/libs/lib-postgres/package.json new file mode 100644 index 000000000..a62cee473 --- /dev/null +++ b/libs/lib-postgres/package.json @@ -0,0 +1,43 @@ +{ + "name": "@powersync/lib-service-postgres", + "repository": "https://github.com/powersync-ja/powersync-service", + "types": "dist/index.d.ts", + "version": "0.0.0", + "main": "dist/index.js", + "license": "FSL-1.1-Apache-2.0", + "type": "module", + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "tsc -b", + "build:tests": "tsc -b test/tsconfig.json", + "clean": "rm -rf ./dist && tsc -b --clean", + "test": "vitest" + }, + "exports": { + ".": { + "import": "./dist/index.js", + "require": "./dist/index.js", + "default": "./dist/index.js" + }, + "./types": { + "import": "./dist/types/types.js", + "require": "./dist/types/types.js", + "default": "./dist/types/types.js" + } + }, + "dependencies": { + "@powersync/lib-services-framework": "workspace:*", + "@powersync/service-jpgwire": "workspace:*", + "@powersync/service-sync-rules": "workspace:*", + "@powersync/service-types": "workspace:*", + "p-defer": "^4.0.1", + "ts-codec": "^1.3.0", + "uri-js": "^4.4.1", + "uuid": "^9.0.1" + }, + "devDependencies": { + "@types/uuid": "^9.0.4" + } +} diff --git a/libs/lib-postgres/src/db/connection/AbstractPostgresConnection.ts b/libs/lib-postgres/src/db/connection/AbstractPostgresConnection.ts new file mode 100644 index 000000000..3484ac81c --- /dev/null +++ b/libs/lib-postgres/src/db/connection/AbstractPostgresConnection.ts @@ -0,0 +1,109 @@ +import * as framework from '@powersync/lib-services-framework'; +import * as pgwire from '@powersync/service-jpgwire'; +import * as t from 'ts-codec'; + +export type DecodedSQLQueryExecutor> = { + first: () => Promise | null>; + rows: () => Promise[]>; +}; + +export abstract class AbstractPostgresConnection< + Listener extends framework.DisposableListener = framework.DisposableListener +> extends framework.DisposableObserver { + protected abstract baseConnection: pgwire.PgClient; + + stream(...args: pgwire.Statement[]): AsyncIterableIterator { + return this.baseConnection.stream(...args); + } + + query(script: string, options?: pgwire.PgSimpleQueryOptions): Promise; + query(...args: pgwire.Statement[]): Promise; + query(...args: any[]): Promise { + return this.baseConnection.query(...args); + } + + /** + * Template string helper which can be used to execute template SQL strings. + */ + sql(strings: TemplateStringsArray, ...params: pgwire.StatementParam[]) { + const { statement, params: queryParams } = sql(strings, ...params); + + const rows = (): Promise => + this.queryRows({ + statement, + params: queryParams + }); + + const first = async (): Promise => { + const [f] = await rows(); + return f; + }; + + return { + execute: () => + this.query({ + statement, + params + }), + rows, + first, + decoded: >(codec: T): DecodedSQLQueryExecutor => { + return { + first: async () => { + const result = await first(); + return result && codec.decode(result); + }, + rows: async () => { + const results = await rows(); + return results.map((r) => { + return codec.decode(r); + }); + } + }; + } + }; + } + + queryRows(script: string, options?: pgwire.PgSimpleQueryOptions): Promise; + queryRows(...args: pgwire.Statement[] | [...pgwire.Statement[], pgwire.PgExtendedQueryOptions]): Promise; + async queryRows(...args: any[]) { + return pgwire.pgwireRows(await this.query(...args)); + } + + async *streamRows(...args: pgwire.Statement[]): AsyncIterableIterator { + let columns: Array = []; + + for await (const chunk of this.stream(...args)) { + if (chunk.tag == 'RowDescription') { + columns = chunk.payload.map((c, index) => { + return c.name as keyof T; + }); + continue; + } + + if (!chunk.rows.length) { + continue; + } + + yield chunk.rows.map((row) => { + let q: Partial = {}; + for (const [index, c] of columns.entries()) { + q[c] = row[index]; + } + return q as T; + }); + } + } +} + +/** + * Template string helper function which generates PGWire statements. + */ +export const sql = (strings: TemplateStringsArray, ...params: pgwire.StatementParam[]): pgwire.Statement => { + const paramPlaceholders = new Array(params.length).fill('').map((value, index) => `$${index + 1}`); + const joinedQueryStatement = strings.map((query, index) => `${query} ${paramPlaceholders[index] ?? ''}`).join(' '); + return { + statement: joinedQueryStatement, + params + }; +}; diff --git a/libs/lib-postgres/src/db/connection/ConnectionSlot.ts b/libs/lib-postgres/src/db/connection/ConnectionSlot.ts new file mode 100644 index 000000000..b2b543b6c --- /dev/null +++ b/libs/lib-postgres/src/db/connection/ConnectionSlot.ts @@ -0,0 +1,165 @@ +import * as framework from '@powersync/lib-services-framework'; +import * as pgwire from '@powersync/service-jpgwire'; + +export interface NotificationListener extends framework.DisposableListener { + notification?: (payload: pgwire.PgNotification) => void; +} + +export interface ConnectionSlotListener extends NotificationListener { + connectionAvailable?: () => void; + connectionError?: (exception: any) => void; + connectionCreated?: (connection: pgwire.PgConnection) => Promise; +} + +export type ConnectionLease = { + connection: pgwire.PgConnection; + release: () => void; +}; + +export type ConnectionSlotOptions = { + config: pgwire.NormalizedConnectionConfig; + notificationChannels?: string[]; +}; + +export const MAX_CONNECTION_ATTEMPTS = 5; + +export class ConnectionSlot extends framework.DisposableObserver { + isAvailable: boolean; + isPoking: boolean; + + closed: boolean; + + protected connection: pgwire.PgConnection | null; + protected connectingPromise: Promise | null; + + constructor(protected options: ConnectionSlotOptions) { + super(); + this.isAvailable = false; + this.connection = null; + this.isPoking = false; + this.connectingPromise = null; + this.closed = false; + } + + get isConnected() { + return !!this.connection; + } + + protected async connect() { + this.connectingPromise = pgwire.connectPgWire(this.options.config, { type: 'standard' }); + const connection = await this.connectingPromise; + this.connectingPromise = null; + await this.iterateAsyncListeners(async (l) => l.connectionCreated?.(connection)); + if (this.hasNotificationListener()) { + await this.configureConnectionNotifications(connection); + } + return connection; + } + + async [Symbol.asyncDispose]() { + this.closed = true; + const connection = this.connection ?? (await this.connectingPromise); + await connection?.end(); + return super[Symbol.dispose](); + } + + protected async configureConnectionNotifications(connection: pgwire.PgConnection) { + if (connection.onnotification == this.handleNotification || this.closed == true) { + // Already configured + return; + } + + connection.onnotification = this.handleNotification; + + for (const channelName of this.options.notificationChannels ?? []) { + await connection.query({ + statement: `LISTEN ${channelName}` + }); + } + } + + registerListener(listener: Partial): () => void { + const dispose = super.registerListener(listener); + if (this.connection && this.hasNotificationListener()) { + this.configureConnectionNotifications(this.connection); + } + return () => { + dispose(); + if (this.connection && !this.hasNotificationListener()) { + this.connection.onnotification = () => {}; + } + }; + } + + protected handleNotification = (payload: pgwire.PgNotification) => { + if (!this.options.notificationChannels?.includes(payload.channel)) { + return; + } + this.iterateListeners((l) => l.notification?.(payload)); + }; + + protected hasNotificationListener() { + return !!Object.values(this.listeners).find((l) => !!l.notification); + } + + /** + * Test the connection if it can be reached. + */ + async poke() { + if (this.isPoking || (this.isConnected && this.isAvailable == false) || this.closed) { + return; + } + this.isPoking = true; + for (let retryCounter = 0; retryCounter <= MAX_CONNECTION_ATTEMPTS; retryCounter++) { + try { + const connection = this.connection ?? (await this.connect()); + + await connection.query({ + statement: 'SELECT 1' + }); + + if (!this.connection) { + this.connection = connection; + this.setAvailable(); + } else if (this.isAvailable) { + this.iterateListeners((cb) => cb.connectionAvailable?.()); + } + + // Connection is alive and healthy + break; + } catch (ex) { + // Should be valid for all cases + this.isAvailable = false; + if (this.connection) { + this.connection.onnotification = () => {}; + this.connection.destroy(); + this.connection = null; + } + if (retryCounter >= MAX_CONNECTION_ATTEMPTS) { + this.iterateListeners((cb) => cb.connectionError?.(ex)); + } + } + } + this.isPoking = false; + } + + protected setAvailable() { + this.isAvailable = true; + this.iterateListeners((l) => l.connectionAvailable?.()); + } + + lock(): ConnectionLease | null { + if (!this.isAvailable || !this.connection || this.closed) { + return null; + } + + this.isAvailable = false; + + return { + connection: this.connection, + release: () => { + this.setAvailable(); + } + }; + } +} diff --git a/libs/lib-postgres/src/db/connection/DatabaseClient.ts b/libs/lib-postgres/src/db/connection/DatabaseClient.ts new file mode 100644 index 000000000..1fbd048c9 --- /dev/null +++ b/libs/lib-postgres/src/db/connection/DatabaseClient.ts @@ -0,0 +1,261 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; +import * as pgwire from '@powersync/service-jpgwire'; +import pDefer, { DeferredPromise } from 'p-defer'; +import { AbstractPostgresConnection, sql } from './AbstractPostgresConnection.js'; +import { ConnectionLease, ConnectionSlot, NotificationListener } from './ConnectionSlot.js'; +import { WrappedConnection } from './WrappedConnection.js'; + +export type DatabaseClientOptions = { + config: lib_postgres.NormalizedBasePostgresConnectionConfig; + /** + * Optional schema which will be used as the default search path + */ + schema?: string; + /** + * Notification channels to listen to. + */ + notificationChannels?: string[]; +}; + +export type DatabaseClientListener = NotificationListener & { + connectionCreated?: (connection: pgwire.PgConnection) => Promise; +}; + +export const TRANSACTION_CONNECTION_COUNT = 5; + +/** + * This provides access to Postgres via the PGWire library. + * A connection pool is used for individual query executions while + * a custom pool of connections is available for transactions or other operations + * which require being executed on the same connection. + */ +export class DatabaseClient extends AbstractPostgresConnection { + closed: boolean; + + protected pool: pgwire.PgClient; + protected connections: ConnectionSlot[]; + + protected initialized: Promise; + protected queue: DeferredPromise[]; + + constructor(protected options: DatabaseClientOptions) { + super(); + this.closed = false; + this.pool = pgwire.connectPgWirePool(options.config); + this.connections = Array.from({ length: TRANSACTION_CONNECTION_COUNT }, () => { + const slot = new ConnectionSlot({ config: options.config, notificationChannels: options.notificationChannels }); + slot.registerListener({ + connectionAvailable: () => this.processConnectionQueue(), + connectionError: (ex) => this.handleConnectionError(ex), + connectionCreated: (connection) => this.iterateAsyncListeners(async (l) => l.connectionCreated?.(connection)) + }); + return slot; + }); + this.queue = []; + this.initialized = this.initialize(); + } + + protected get baseConnection() { + return this.pool; + } + + protected get schemaStatement() { + const { schema } = this.options; + if (!schema) { + return; + } + return { + statement: `SET search_path TO ${schema};` + }; + } + + registerListener(listener: Partial): () => void { + let disposeNotification: (() => void) | null = null; + if ('notification' in listener) { + // Pass this on to the first connection slot + // It will only actively listen on the connection once a listener has been registered + disposeNotification = this.connections[0].registerListener({ + notification: listener.notification + }); + this.pokeSlots(); + + delete listener['notification']; + } + + const superDispose = super.registerListener(listener); + return () => { + disposeNotification?.(); + superDispose(); + }; + } + + query(script: string, options?: pgwire.PgSimpleQueryOptions): Promise; + query(...args: pgwire.Statement[]): Promise; + async query(...args: any[]): Promise { + await this.initialized; + /** + * There is no direct way to set the default schema with pgwire. + * This hack uses multiple statements in order to always ensure the + * appropriate connection (in the pool) uses the correct schema. + */ + const { schemaStatement } = this; + if (typeof args[0] == 'object' && schemaStatement) { + args.unshift(schemaStatement); + } else if (typeof args[0] == 'string' && schemaStatement) { + args[0] = `${schemaStatement.statement}; ${args[0]}`; + } + + // Retry pool queries. Note that we can't retry queries in a transaction + // since a failed query will end the transaction. + return lib_postgres.retriedQuery(this.pool, ...args); + } + + async *stream(...args: pgwire.Statement[]): AsyncIterableIterator { + await this.initialized; + const { schemaStatement } = this; + if (schemaStatement) { + args.unshift(schemaStatement); + } + yield* super.stream(...args); + } + + async lockConnection(callback: (db: WrappedConnection) => Promise): Promise { + const { connection, release } = await this.requestConnection(); + + await this.setSchema(connection); + + try { + return await callback(new WrappedConnection(connection)); + } finally { + release(); + } + } + + async transaction(tx: (db: WrappedConnection) => Promise): Promise { + return this.lockConnection(async (db) => { + try { + await db.query(sql`BEGIN`); + const result = await tx(db); + await db.query(sql`COMMIT`); + return result; + } catch (ex) { + await db.query(sql`ROLLBACK`); + throw ex; + } + }); + } + + /** + * Use the `powersync` schema as the default when resolving table names + */ + protected async setSchema(client: pgwire.PgClient) { + const { schemaStatement } = this; + if (!schemaStatement) { + return; + } + await client.query(schemaStatement); + } + + protected async initialize() { + const { schema } = this.options; + if (schema) { + // First check if it exists + const exists = await this.pool.query(sql` + SELECT + schema_name + FROM + information_schema.schemata + WHERE + schema_name = ${{ type: 'varchar', value: schema }}; + `); + + if (exists.rows.length) { + return; + } + // Create the schema if it doesn't exist + await this.pool.query({ statement: `CREATE SCHEMA IF NOT EXISTS ${this.options.schema}` }); + } + } + + protected async requestConnection(): Promise { + if (this.closed) { + throw new Error('Database client is closed'); + } + + await this.initialized; + + // Queue the operation + const deferred = pDefer(); + this.queue.push(deferred); + + this.pokeSlots(); + + return deferred.promise; + } + + protected pokeSlots() { + // Poke the slots to check if they are alive + for (const slot of this.connections) { + // No need to await this. Errors are reported asynchronously + slot.poke(); + } + } + + protected leaseConnectionSlot(): ConnectionLease | null { + const availableSlots = this.connections.filter((s) => s.isAvailable); + for (const slot of availableSlots) { + const lease = slot.lock(); + if (lease) { + return lease; + } + // Possibly some contention detected, keep trying + } + return null; + } + + protected processConnectionQueue() { + if (this.closed && this.queue.length) { + for (const q of this.queue) { + q.reject(new Error('Database has closed while waiting for a connection')); + } + this.queue = []; + } + + if (this.queue.length) { + const lease = this.leaseConnectionSlot(); + if (lease) { + const deferred = this.queue.shift()!; + deferred.resolve(lease); + } + } + } + + /** + * Reports connection errors which might occur from bad configuration or + * a server which is no longer available. + * This fails all pending requests. + */ + protected handleConnectionError(exception: any) { + for (const q of this.queue) { + q.reject(exception); + } + this.queue = []; + } + + async [Symbol.asyncDispose]() { + await this.initialized; + this.closed = true; + + for (const c of this.connections) { + await c[Symbol.asyncDispose](); + } + + await this.pool.end(); + + // Reject all remaining items + for (const q of this.queue) { + q.reject(new Error(`Database is disposed`)); + } + this.queue = []; + } +} diff --git a/libs/lib-postgres/src/db/connection/WrappedConnection.ts b/libs/lib-postgres/src/db/connection/WrappedConnection.ts new file mode 100644 index 000000000..9bd5c6a67 --- /dev/null +++ b/libs/lib-postgres/src/db/connection/WrappedConnection.ts @@ -0,0 +1,11 @@ +import * as pgwire from '@powersync/service-jpgwire'; +import { AbstractPostgresConnection } from './AbstractPostgresConnection.js'; + +/** + * Provides helper functionality to transaction contexts given an existing PGWire connection + */ +export class WrappedConnection extends AbstractPostgresConnection { + constructor(protected baseConnection: pgwire.PgConnection) { + super(); + } +} diff --git a/libs/lib-postgres/src/db/db-index.ts b/libs/lib-postgres/src/db/db-index.ts new file mode 100644 index 000000000..8b6017e74 --- /dev/null +++ b/libs/lib-postgres/src/db/db-index.ts @@ -0,0 +1,4 @@ +export * from './connection/AbstractPostgresConnection.js'; +export * from './connection/ConnectionSlot.js'; +export * from './connection/DatabaseClient.js'; +export * from './connection/WrappedConnection.js'; diff --git a/libs/lib-postgres/src/index.ts b/libs/lib-postgres/src/index.ts new file mode 100644 index 000000000..0b47d2dd8 --- /dev/null +++ b/libs/lib-postgres/src/index.ts @@ -0,0 +1,11 @@ +export * from './db/db-index.js'; +export * as db from './db/db-index.js'; + +export * from './locks/locks-index.js'; +export * as locks from './locks/locks-index.js'; + +export * from './types/types.js'; +export * as types from './types/types.js'; + +export * from './utils/utils-index.js'; +export * as utils from './utils/utils-index.js'; diff --git a/libs/lib-postgres/src/locks/PostgresLockManager.ts b/libs/lib-postgres/src/locks/PostgresLockManager.ts new file mode 100644 index 000000000..4f01f5156 --- /dev/null +++ b/libs/lib-postgres/src/locks/PostgresLockManager.ts @@ -0,0 +1,128 @@ +import * as framework from '@powersync/lib-services-framework'; +import { v4 as uuidv4 } from 'uuid'; +import { DatabaseClient, sql } from '../db/db-index.js'; + +const DEFAULT_LOCK_TIMEOUT = 60_000; // 1 minute + +export interface PostgresLockManagerParams extends framework.locks.LockManagerParams { + db: DatabaseClient; +} + +export class PostgresLockManager extends framework.locks.AbstractLockManager { + constructor(protected params: PostgresLockManagerParams) { + super(params); + } + + protected get db() { + return this.params.db; + } + + get timeout() { + return this.params.timeout ?? DEFAULT_LOCK_TIMEOUT; + } + + get name() { + return this.params.name; + } + + async init() { + /** + * Locks are required for migrations, which means this table can't be + * created inside a migration. This ensures the locks table is present. + */ + await this.db.query(sql` + CREATE TABLE IF NOT EXISTS locks ( + name TEXT PRIMARY KEY, + lock_id UUID NOT NULL, + ts TIMESTAMPTZ NOT NULL + ); + `); + } + + protected async acquireHandle(): Promise { + const id = await this._acquireId(); + if (!id) { + return null; + } + return { + refresh: () => this.refreshHandle(id), + release: () => this.releaseHandle(id) + }; + } + + protected async _acquireId(): Promise { + const now = new Date(); + const nowISO = now.toISOString(); + const expiredTs = new Date(now.getTime() - this.timeout).toISOString(); + const lockId = uuidv4(); + + try { + // Attempt to acquire or refresh the lock + const res = await this.db.queryRows<{ lock_id: string }>(sql` + INSERT INTO + locks (name, lock_id, ts) + VALUES + ( + ${{ type: 'varchar', value: this.name }}, + ${{ type: 'uuid', value: lockId }}, + ${{ type: 1184, value: nowISO }} + ) + ON CONFLICT (name) DO UPDATE + SET + lock_id = CASE + WHEN locks.ts <= ${{ type: 1184, value: expiredTs }} THEN ${{ type: 'uuid', value: lockId }} + ELSE locks.lock_id + END, + ts = CASE + WHEN locks.ts <= ${{ type: 1184, value: expiredTs }} THEN ${{ + type: 1184, + value: nowISO + }} + ELSE locks.ts + END + RETURNING + lock_id; + `); + + if (res.length == 0 || res[0].lock_id !== lockId) { + // Lock is active and could not be acquired + return null; + } + + return lockId; + } catch (err) { + console.error('Error acquiring lock:', err); + throw err; + } + } + + protected async refreshHandle(lockId: string) { + const res = await this.db.query(sql` + UPDATE locks + SET + ts = ${{ type: 1184, value: new Date().toISOString() }} + WHERE + lock_id = ${{ type: 'uuid', value: lockId }} + RETURNING + lock_id; + `); + + if (res.rows.length === 0) { + throw new Error('Lock not found, could not refresh'); + } + } + + protected async releaseHandle(lockId: string) { + const res = await this.db.query(sql` + DELETE FROM locks + WHERE + lock_id = ${{ type: 'uuid', value: lockId }} + RETURNING + lock_id; + `); + + if (res.rows.length == 0) { + throw new Error('Lock not found, could not release'); + } + } +} diff --git a/libs/lib-postgres/src/locks/locks-index.ts b/libs/lib-postgres/src/locks/locks-index.ts new file mode 100644 index 000000000..06f0e730e --- /dev/null +++ b/libs/lib-postgres/src/locks/locks-index.ts @@ -0,0 +1 @@ +export * from './PostgresLockManager.js'; diff --git a/libs/lib-postgres/src/types/types.ts b/libs/lib-postgres/src/types/types.ts new file mode 100644 index 000000000..3d147e10b --- /dev/null +++ b/libs/lib-postgres/src/types/types.ts @@ -0,0 +1,149 @@ +import * as service_types from '@powersync/service-types'; +import * as t from 'ts-codec'; +import * as urijs from 'uri-js'; + +export interface NormalizedBasePostgresConnectionConfig { + id: string; + tag: string; + + hostname: string; + port: number; + database: string; + + username: string; + password: string; + + sslmode: 'verify-full' | 'verify-ca' | 'disable'; + cacert: string | undefined; + + client_certificate: string | undefined; + client_private_key: string | undefined; +} + +export const POSTGRES_CONNECTION_TYPE = 'postgresql' as const; + +export const BasePostgresConnectionConfig = t.object({ + /** Unique identifier for the connection - optional when a single connection is present. */ + id: t.string.optional(), + /** Additional meta tag for connection */ + tag: t.string.optional(), + type: t.literal(POSTGRES_CONNECTION_TYPE), + uri: t.string.optional(), + hostname: t.string.optional(), + port: service_types.configFile.portCodec.optional(), + username: t.string.optional(), + password: t.string.optional(), + database: t.string.optional(), + + /** Defaults to verify-full */ + sslmode: t.literal('verify-full').or(t.literal('verify-ca')).or(t.literal('disable')).optional(), + /** Required for verify-ca, optional for verify-full */ + cacert: t.string.optional(), + + client_certificate: t.string.optional(), + client_private_key: t.string.optional(), + + /** Expose database credentials */ + demo_database: t.boolean.optional(), + + /** + * Prefix for the slot name. Defaults to "powersync_" + */ + slot_name_prefix: t.string.optional() +}); + +export type BasePostgresConnectionConfig = t.Encoded; +export type BasePostgresConnectionConfigDecoded = t.Decoded; + +/** + * Validate and normalize connection options. + * + * Returns destructured options. + */ +export function normalizeConnectionConfig(options: BasePostgresConnectionConfigDecoded) { + let uri: urijs.URIComponents; + if (options.uri) { + uri = urijs.parse(options.uri); + if (uri.scheme != 'postgresql' && uri.scheme != 'postgres') { + `Invalid URI - protocol must be postgresql, got ${uri.scheme}`; + } else if (uri.scheme != 'postgresql') { + uri.scheme = 'postgresql'; + } + } else { + uri = urijs.parse('postgresql:///'); + } + + const hostname = options.hostname ?? uri.host ?? ''; + const port = validatePort(options.port ?? uri.port ?? 5432); + + const database = options.database ?? uri.path?.substring(1) ?? ''; + + const [uri_username, uri_password] = (uri.userinfo ?? '').split(':'); + + const username = options.username ?? uri_username ?? ''; + const password = options.password ?? uri_password ?? ''; + + const sslmode = options.sslmode ?? 'verify-full'; // Configuration not supported via URI + const cacert = options.cacert; + + if (sslmode == 'verify-ca' && cacert == null) { + throw new Error('Explicit cacert is required for sslmode=verify-ca'); + } + + if (hostname == '') { + throw new Error(`hostname required`); + } + + if (username == '') { + throw new Error(`username required`); + } + + if (password == '') { + throw new Error(`password required`); + } + + if (database == '') { + throw new Error(`database required`); + } + + return { + id: options.id ?? 'default', + tag: options.tag ?? 'default', + + hostname, + port, + database, + + username, + password, + sslmode, + cacert, + + client_certificate: options.client_certificate ?? undefined, + client_private_key: options.client_private_key ?? undefined + } satisfies NormalizedBasePostgresConnectionConfig; +} + +/** + * Check whether the port is in a "safe" range. + * + * We do not support connecting to "privileged" ports. + */ +export function validatePort(port: string | number): number { + if (typeof port == 'string') { + port = parseInt(port); + } + if (port < 1024) { + throw new Error(`Port ${port} not supported`); + } + return port; +} + +/** + * Construct a postgres URI, without username, password or ssl options. + * + * Only contains hostname, port, database. + */ +export function baseUri(options: NormalizedBasePostgresConnectionConfig) { + return `postgresql://${options.hostname}:${options.port}/${options.database}`; +} diff --git a/libs/lib-postgres/src/utils/pgwire_utils.ts b/libs/lib-postgres/src/utils/pgwire_utils.ts new file mode 100644 index 000000000..230fef7b8 --- /dev/null +++ b/libs/lib-postgres/src/utils/pgwire_utils.ts @@ -0,0 +1,49 @@ +// Adapted from https://github.com/kagis/pgwire/blob/0dc927f9f8990a903f238737326e53ba1c8d094f/mod.js#L2218 + +import * as pgwire from '@powersync/service-jpgwire'; +import { SqliteJsonValue } from '@powersync/service-sync-rules'; + +import { logger } from '@powersync/lib-services-framework'; + +export function escapeIdentifier(identifier: string) { + return `"${identifier.replace(/"/g, '""').replace(/\./g, '"."')}"`; +} + +export function autoParameter(arg: SqliteJsonValue | boolean): pgwire.StatementParam { + if (arg == null) { + return { type: 'varchar', value: null }; + } else if (typeof arg == 'string') { + return { type: 'varchar', value: arg }; + } else if (typeof arg == 'number') { + if (Number.isInteger(arg)) { + return { type: 'int8', value: arg }; + } else { + return { type: 'float8', value: arg }; + } + } else if (typeof arg == 'boolean') { + return { type: 'bool', value: arg }; + } else if (typeof arg == 'bigint') { + return { type: 'int8', value: arg }; + } else { + throw new Error(`Unsupported query parameter: ${typeof arg}`); + } +} + +export async function retriedQuery(db: pgwire.PgClient, ...statements: pgwire.Statement[]): Promise; +export async function retriedQuery(db: pgwire.PgClient, query: string): Promise; + +/** + * Retry a simple query - up to 2 attempts total. + */ +export async function retriedQuery(db: pgwire.PgClient, ...args: any[]) { + for (let tries = 2; ; tries--) { + try { + return await db.query(...args); + } catch (e) { + if (tries == 1) { + throw e; + } + logger.warn('Query error, retrying', e); + } + } +} diff --git a/libs/lib-postgres/src/utils/utils-index.ts b/libs/lib-postgres/src/utils/utils-index.ts new file mode 100644 index 000000000..5bfe85120 --- /dev/null +++ b/libs/lib-postgres/src/utils/utils-index.ts @@ -0,0 +1 @@ +export * from './pgwire_utils.js'; diff --git a/libs/lib-postgres/test/src/config.test.ts b/libs/lib-postgres/test/src/config.test.ts new file mode 100644 index 000000000..35f84d220 --- /dev/null +++ b/libs/lib-postgres/test/src/config.test.ts @@ -0,0 +1,12 @@ +import { describe, expect, test } from 'vitest'; +import { normalizeConnectionConfig } from '../../src/types/types.js'; + +describe('config', () => { + test('Should resolve database', () => { + const normalized = normalizeConnectionConfig({ + type: 'postgresql', + uri: 'postgresql://postgres:postgres@localhost:4321/powersync_test' + }); + expect(normalized.database).equals('powersync_test'); + }); +}); diff --git a/libs/lib-postgres/test/tsconfig.json b/libs/lib-postgres/test/tsconfig.json new file mode 100644 index 000000000..4ce408172 --- /dev/null +++ b/libs/lib-postgres/test/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "baseUrl": "./", + "noEmit": true, + "esModuleInterop": true, + "skipLibCheck": true, + "sourceMap": true, + "paths": {} + }, + "include": ["src"], + "references": [ + { + "path": "../" + } + ] +} diff --git a/libs/lib-postgres/tsconfig.json b/libs/lib-postgres/tsconfig.json new file mode 100644 index 000000000..a0ae425c6 --- /dev/null +++ b/libs/lib-postgres/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "esModuleInterop": true, + "skipLibCheck": true, + "sourceMap": true + }, + "include": ["src"], + "references": [] +} diff --git a/libs/lib-postgres/vitest.config.ts b/libs/lib-postgres/vitest.config.ts new file mode 100644 index 000000000..94ede10e2 --- /dev/null +++ b/libs/lib-postgres/vitest.config.ts @@ -0,0 +1,3 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({}); diff --git a/libs/lib-services/src/migrations/AbstractMigrationAgent.ts b/libs/lib-services/src/migrations/AbstractMigrationAgent.ts index bfb465501..3be96acf3 100644 --- a/libs/lib-services/src/migrations/AbstractMigrationAgent.ts +++ b/libs/lib-services/src/migrations/AbstractMigrationAgent.ts @@ -102,6 +102,7 @@ export abstract class AbstractMigrationAgent { return migration.name === params.state!.last_run; }); @@ -112,8 +113,33 @@ export abstract class AbstractMigrationAgent b.timestamp.getTime() - a.timestamp.getTime()) + .find((log) => log.name == last_run); + + // There should be a log entry for this + if (!lastLogEntry) { + throw new Error(`Could not find last migration log entry for ${last_run}`); + } + + // If we are migrating up: + // If the last run was an up migration: + // Then we want to start at the next migration index + // If after a previous Down migration + // Then we need to start at the current migration index + + // If we are migrating down: + // If the previous migration was a down migration + // Then we need to start at the next index + // If the previous migration was an up migration + // Then we want to include the last run (up) migration + if ( + (params.direction === defs.Direction.Up && lastLogEntry.direction == defs.Direction.Up) || + (params.direction == defs.Direction.Down && lastLogEntry.direction == defs.Direction.Down) + ) { index += 1; } } diff --git a/libs/lib-services/src/utils/DisposableObserver.ts b/libs/lib-services/src/utils/DisposableObserver.ts index 1440d57e7..194ed6955 100644 --- a/libs/lib-services/src/utils/DisposableObserver.ts +++ b/libs/lib-services/src/utils/DisposableObserver.ts @@ -7,7 +7,7 @@ export interface DisposableListener { disposed: () => void; } -export interface DisposableObserverClient extends ObserverClient, Disposable { +export interface ManagedObserverClient extends ObserverClient { /** * Registers a listener that is automatically disposed when the parent is disposed. * This is useful for disposing nested listeners. @@ -15,6 +15,11 @@ export interface DisposableObserverClient extends registerManagedListener: (parent: DisposableObserverClient, cb: Partial) => () => void; } +export interface DisposableObserverClient extends ManagedObserverClient, Disposable {} +export interface AsyncDisposableObserverClient + extends ManagedObserverClient, + AsyncDisposable {} + export class DisposableObserver extends BaseObserver implements DisposableObserverClient diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 9cbf67738..87f577eca 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -62,6 +62,10 @@ export class MongoBucketStorage this.slot_name_prefix = options.slot_name_prefix; } + async [Symbol.asyncDispose]() { + super[Symbol.dispose](); + } + getInstance(options: storage.PersistedSyncRulesContent): MongoSyncBucketStorage { let { id, slot_name } = options; if ((typeof id as any) == 'bigint') { @@ -106,7 +110,6 @@ export class MongoBucketStorage // In both the below cases, we create a new sync rules instance. // The current one will continue erroring until the next one has finished processing. - // TODO: Update if (next != null && next.slot_name == slot_name) { // We need to redo the "next" sync rules await this.updateSyncRules({ diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoStorageProvider.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoStorageProvider.ts index a147bfe08..cfa010b26 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoStorageProvider.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoStorageProvider.ts @@ -23,13 +23,16 @@ export class MongoStorageProvider implements storage.BucketStorageProvider { const client = lib_mongo.db.createMongoClient(decodedConfig); const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database }); - + const factory = new MongoBucketStorage(database, { + // TODO currently need the entire resolved config due to this + slot_name_prefix: resolvedConfig.slot_name_prefix + }); return { - storage: new MongoBucketStorage(database, { - // TODO currently need the entire resolved config due to this - slot_name_prefix: resolvedConfig.slot_name_prefix - }), - shutDown: () => client.close(), + storage: factory, + shutDown: async () => { + await factory[Symbol.asyncDispose](); + await client.close(); + }, tearDown: () => { logger.info(`Tearing down storage: ${database.db.namespace}...`); return database.db.dropDatabase(); diff --git a/modules/module-mongodb-storage/test/src/__snapshots__/storage.test.ts.snap b/modules/module-mongodb-storage/test/src/__snapshots__/storage.test.ts.snap new file mode 100644 index 000000000..7c072c0a1 --- /dev/null +++ b/modules/module-mongodb-storage/test/src/__snapshots__/storage.test.ts.snap @@ -0,0 +1,9 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`Mongo Sync Bucket Storage > empty storage metrics 1`] = ` +{ + "operations_size_bytes": 0, + "parameters_size_bytes": 0, + "replication_size_bytes": 0, +} +`; diff --git a/modules/module-mongodb-storage/test/src/migrations.test.ts b/modules/module-mongodb-storage/test/src/migrations.test.ts new file mode 100644 index 000000000..e4728fa04 --- /dev/null +++ b/modules/module-mongodb-storage/test/src/migrations.test.ts @@ -0,0 +1,10 @@ +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { MongoMigrationAgent } from '../../src/migrations/MongoMigrationAgent.js'; +import { env } from './env.js'; + +const MIGRATION_AGENT_FACTORY = () => { + return new MongoMigrationAgent({ type: 'mongodb', uri: env.MONGO_TEST_URL }); +}; + +describe('Mongo Migrations Store', () => register.registerMigrationTests(MIGRATION_AGENT_FACTORY)); diff --git a/modules/module-mongodb-storage/test/src/setup.ts b/modules/module-mongodb-storage/test/src/setup.ts index debe66011..43946a1aa 100644 --- a/modules/module-mongodb-storage/test/src/setup.ts +++ b/modules/module-mongodb-storage/test/src/setup.ts @@ -1,9 +1,13 @@ import { container } from '@powersync/lib-services-framework'; import { test_utils } from '@powersync/service-core-tests'; -import { beforeAll } from 'vitest'; +import { beforeAll, beforeEach } from 'vitest'; beforeAll(async () => { // Executes for every test file container.registerDefaults(); await test_utils.initMetrics(); }); + +beforeEach(async () => { + await test_utils.resetMetrics(); +}); diff --git a/modules/module-mongodb/package.json b/modules/module-mongodb/package.json index 400c7b713..ef30d3c5b 100644 --- a/modules/module-mongodb/package.json +++ b/modules/module-mongodb/package.json @@ -41,6 +41,7 @@ "devDependencies": { "@types/uuid": "^9.0.4", "@powersync/service-core-tests": "workspace:*", - "@powersync/service-module-mongodb-storage": "workspace:*" + "@powersync/service-module-mongodb-storage": "workspace:*", + "@powersync/service-module-postgres-storage": "workspace:*" } } diff --git a/modules/module-mongodb/test/src/change_stream.test.ts b/modules/module-mongodb/test/src/change_stream.test.ts index 8001dd1fc..a4c461887 100644 --- a/modules/module-mongodb/test/src/change_stream.test.ts +++ b/modules/module-mongodb/test/src/change_stream.test.ts @@ -8,7 +8,8 @@ import { test_utils } from '@powersync/service-core-tests'; import { PostImagesOption } from '@module/types/types.js'; import { ChangeStreamTestContext } from './change_stream_utils.js'; -import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; +import { env } from './env.js'; +import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js'; const BASIC_SYNC_RULES = ` bucket_definitions: @@ -17,10 +18,14 @@ bucket_definitions: - SELECT _id as id, description FROM "test_data" `; -describe('change stream - mongodb', { timeout: 20_000 }, function () { +describe.skipIf(!env.TEST_MONGO_STORAGE)('change stream - mongodb', { timeout: 20_000 }, function () { defineChangeStreamTests(INITIALIZED_MONGO_STORAGE_FACTORY); }); +describe.skipIf(!env.TEST_POSTGRES_STORAGE)('change stream - postgres', { timeout: 20_000 }, function () { + defineChangeStreamTests(INITIALIZED_POSTGRES_STORAGE_FACTORY); +}); + function defineChangeStreamTests(factory: storage.TestStorageFactory) { test('replicating basic values', async () => { await using context = await ChangeStreamTestContext.open(factory); diff --git a/modules/module-mongodb/test/src/change_stream_utils.ts b/modules/module-mongodb/test/src/change_stream_utils.ts index 7390c5a1c..6d21ee817 100644 --- a/modules/module-mongodb/test/src/change_stream_utils.ts +++ b/modules/module-mongodb/test/src/change_stream_utils.ts @@ -38,6 +38,7 @@ export class ChangeStreamTestContext { this.abortController.abort(); await this.streamPromise?.catch((e) => e); await this.connectionManager.destroy(); + await this.factory[Symbol.asyncDispose](); } async [Symbol.asyncDispose]() { @@ -157,7 +158,6 @@ export async function getClientCheckpoint( if (cp.lsn && cp.lsn >= lsn) { return cp.checkpoint; } - await new Promise((resolve) => setTimeout(resolve, 30)); } diff --git a/modules/module-mongodb/test/src/env.ts b/modules/module-mongodb/test/src/env.ts index 7bfe03857..ad0f171ec 100644 --- a/modules/module-mongodb/test/src/env.ts +++ b/modules/module-mongodb/test/src/env.ts @@ -3,6 +3,9 @@ import { utils } from '@powersync/lib-services-framework'; export const env = utils.collectEnvironmentVariables({ MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'), MONGO_TEST_DATA_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test_data'), + PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5431/powersync_storage_test'), CI: utils.type.boolean.default('false'), - SLOW_TESTS: utils.type.boolean.default('false') + SLOW_TESTS: utils.type.boolean.default('false'), + TEST_MONGO_STORAGE: utils.type.boolean.default('true'), + TEST_POSTGRES_STORAGE: utils.type.boolean.default('true') }); diff --git a/modules/module-mongodb/test/src/setup.ts b/modules/module-mongodb/test/src/setup.ts index fe127d8a9..c89537d5d 100644 --- a/modules/module-mongodb/test/src/setup.ts +++ b/modules/module-mongodb/test/src/setup.ts @@ -1,5 +1,6 @@ import { container } from '@powersync/lib-services-framework'; import { test_utils } from '@powersync/service-core-tests'; +import { beforeEach } from 'node:test'; import { beforeAll } from 'vitest'; beforeAll(async () => { @@ -8,3 +9,7 @@ beforeAll(async () => { await test_utils.initMetrics(); }); + +beforeEach(async () => { + await test_utils.resetMetrics(); +}); diff --git a/modules/module-mongodb/test/src/slow_tests.test.ts b/modules/module-mongodb/test/src/slow_tests.test.ts index 429561119..7e365b15c 100644 --- a/modules/module-mongodb/test/src/slow_tests.test.ts +++ b/modules/module-mongodb/test/src/slow_tests.test.ts @@ -6,9 +6,9 @@ import { storage } from '@powersync/service-core'; import { ChangeStreamTestContext, setSnapshotHistorySeconds } from './change_stream_utils.js'; import { env } from './env.js'; -import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; +import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js'; -describe('change stream slow tests - mongodb', { timeout: 60_000 }, function () { +describe.skipIf(!env.TEST_MONGO_STORAGE)('change stream slow tests - mongodb', { timeout: 60_000 }, function () { if (env.CI || env.SLOW_TESTS) { defineSlowTests(INITIALIZED_MONGO_STORAGE_FACTORY); } else { @@ -17,6 +17,15 @@ describe('change stream slow tests - mongodb', { timeout: 60_000 }, function () } }); +describe.skipIf(!env.TEST_POSTGRES_STORAGE)('change stream slow tests - postgres', { timeout: 60_000 }, function () { + if (env.CI || env.SLOW_TESTS) { + defineSlowTests(INITIALIZED_POSTGRES_STORAGE_FACTORY); + } else { + // Need something in this file. + test('no-op', () => {}); + } +}); + function defineSlowTests(factory: storage.TestStorageFactory) { test('replicating snapshot with lots of data', async () => { await using context = await ChangeStreamTestContext.open(factory); diff --git a/modules/module-mongodb/test/src/util.ts b/modules/module-mongodb/test/src/util.ts index 95cb9001d..aaa566370 100644 --- a/modules/module-mongodb/test/src/util.ts +++ b/modules/module-mongodb/test/src/util.ts @@ -1,5 +1,6 @@ import { mongo } from '@powersync/lib-service-mongodb'; import * as mongo_storage from '@powersync/service-module-mongodb-storage'; +import * as postgres_storage from '@powersync/service-module-postgres-storage'; import * as types from '@module/types/types.js'; import { env } from './env.js'; @@ -16,6 +17,10 @@ export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.MongoTestStorageF isCI: env.CI }); +export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.PostgresTestStorageFactoryGenerator({ + url: env.PG_STORAGE_TEST_URL +}); + export async function clearTestDb(db: mongo.Db) { await db.dropDatabase(); } diff --git a/modules/module-mysql/package.json b/modules/module-mysql/package.json index 35ed59d4b..61c4ed4f0 100644 --- a/modules/module-mysql/package.json +++ b/modules/module-mysql/package.json @@ -46,6 +46,7 @@ "@types/async": "^3.2.24", "@types/uuid": "^9.0.4", "@powersync/service-core-tests": "workspace:*", - "@powersync/service-module-mongodb-storage": "workspace:*" + "@powersync/service-module-mongodb-storage": "workspace:*", + "@powersync/service-module-postgres-storage": "workspace:*" } } diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index bf1b24a75..a41e285ce 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -6,12 +6,12 @@ import { ColumnDescriptor, framework, getUuidReplicaIdentityBson, Metrics, stora import mysql, { FieldPacket } from 'mysql2'; import { BinLogEvent, StartOptions, TableMapEntry } from '@powersync/mysql-zongji'; +import mysqlPromise from 'mysql2/promise'; import * as common from '../common/common-index.js'; -import * as zongji_utils from './zongji/zongji-utils.js'; -import { MySQLConnectionManager } from './MySQLConnectionManager.js'; import { isBinlogStillAvailable, ReplicatedGTID, toColumnDescriptors } from '../common/common-index.js'; -import mysqlPromise from 'mysql2/promise'; import { createRandomServerId, escapeMysqlTableName } from '../utils/mysql-utils.js'; +import { MySQLConnectionManager } from './MySQLConnectionManager.js'; +import * as zongji_utils from './zongji/zongji-utils.js'; export interface BinLogStreamOptions { connections: MySQLConnectionManager; @@ -75,8 +75,19 @@ export class BinLogStream { } get connectionId() { + const { connectionId } = this.connections; // Default to 1 if not set - return this.connections.connectionId ? Number.parseInt(this.connections.connectionId) : 1; + if (!connectionId) { + return 1; + } + /** + * This is often `"default"` (string) which will parse to `NaN` + */ + const parsed = Number.parseInt(connectionId); + if (isNaN(parsed)) { + return 1; + } + return parsed; } get stopped() { diff --git a/modules/module-mysql/test/src/BinLogStream.test.ts b/modules/module-mysql/test/src/BinLogStream.test.ts index 7295935ad..2ed560980 100644 --- a/modules/module-mysql/test/src/BinLogStream.test.ts +++ b/modules/module-mysql/test/src/BinLogStream.test.ts @@ -3,7 +3,8 @@ import { putOp, removeOp } from '@powersync/service-core-tests'; import { v4 as uuid } from 'uuid'; import { describe, expect, test } from 'vitest'; import { BinlogStreamTestContext } from './BinlogStreamUtils.js'; -import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; +import { env } from './env.js'; +import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js'; const BASIC_SYNC_RULES = ` bucket_definitions: @@ -12,13 +13,13 @@ bucket_definitions: - SELECT id, description FROM "test_data" `; -describe( - ' Binlog stream - mongodb', - function () { - defineBinlogStreamTests(INITIALIZED_MONGO_STORAGE_FACTORY); - }, - { timeout: 20_000 } -); +describe.skipIf(!env.TEST_MONGO_STORAGE)(' Binlog stream - mongodb', { timeout: 20_000 }, function () { + defineBinlogStreamTests(INITIALIZED_MONGO_STORAGE_FACTORY); +}); + +describe.skipIf(!env.TEST_POSTGRES_STORAGE)(' Binlog stream - postgres', { timeout: 20_000 }, function () { + defineBinlogStreamTests(INITIALIZED_POSTGRES_STORAGE_FACTORY); +}); function defineBinlogStreamTests(factory: storage.TestStorageFactory) { test('Replicate basic values', async () => { diff --git a/modules/module-mysql/test/src/BinlogStreamUtils.ts b/modules/module-mysql/test/src/BinlogStreamUtils.ts index be6c3064a..5f7062960 100644 --- a/modules/module-mysql/test/src/BinlogStreamUtils.ts +++ b/modules/module-mysql/test/src/BinlogStreamUtils.ts @@ -49,6 +49,7 @@ export class BinlogStreamTestContext { this.abortController.abort(); await this.streamPromise; await this.connectionManager.end(); + await this.factory[Symbol.asyncDispose](); } [Symbol.asyncDispose]() { @@ -104,6 +105,7 @@ export class BinlogStreamTestContext { async replicateSnapshot() { await this.binlogStream.initReplication(); + await this.storage!.autoActivate(); this.replicationDone = true; } diff --git a/modules/module-mysql/test/src/env.ts b/modules/module-mysql/test/src/env.ts index 53ecef648..063745e4a 100644 --- a/modules/module-mysql/test/src/env.ts +++ b/modules/module-mysql/test/src/env.ts @@ -3,6 +3,9 @@ import { utils } from '@powersync/lib-services-framework'; export const env = utils.collectEnvironmentVariables({ MYSQL_TEST_URI: utils.type.string.default('mysql://root:mypassword@localhost:3306/mydatabase'), MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'), + PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5431/powersync_storage_test'), CI: utils.type.boolean.default('false'), - SLOW_TESTS: utils.type.boolean.default('false') + SLOW_TESTS: utils.type.boolean.default('false'), + TEST_MONGO_STORAGE: utils.type.boolean.default('true'), + TEST_POSTGRES_STORAGE: utils.type.boolean.default('true') }); diff --git a/modules/module-mysql/test/src/setup.ts b/modules/module-mysql/test/src/setup.ts index debe66011..43946a1aa 100644 --- a/modules/module-mysql/test/src/setup.ts +++ b/modules/module-mysql/test/src/setup.ts @@ -1,9 +1,13 @@ import { container } from '@powersync/lib-services-framework'; import { test_utils } from '@powersync/service-core-tests'; -import { beforeAll } from 'vitest'; +import { beforeAll, beforeEach } from 'vitest'; beforeAll(async () => { // Executes for every test file container.registerDefaults(); await test_utils.initMetrics(); }); + +beforeEach(async () => { + await test_utils.resetMetrics(); +}); diff --git a/modules/module-mysql/test/src/util.ts b/modules/module-mysql/test/src/util.ts index 597f03b17..ffb797032 100644 --- a/modules/module-mysql/test/src/util.ts +++ b/modules/module-mysql/test/src/util.ts @@ -1,6 +1,7 @@ import * as types from '@module/types/types.js'; import { getMySQLVersion, isVersionAtLeast } from '@module/utils/mysql-utils.js'; import * as mongo_storage from '@powersync/service-module-mongodb-storage'; +import * as postgres_storage from '@powersync/service-module-postgres-storage'; import mysqlPromise from 'mysql2/promise'; import { env } from './env.js'; @@ -16,6 +17,10 @@ export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.MongoTestStorageF isCI: env.CI }); +export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.PostgresTestStorageFactoryGenerator({ + url: env.PG_STORAGE_TEST_URL +}); + export async function clearTestDb(connection: mysqlPromise.Connection) { const version = await getMySQLVersion(connection); if (isVersionAtLeast(version, '8.4.0')) { diff --git a/modules/module-postgres-storage/CHANGELOG.md b/modules/module-postgres-storage/CHANGELOG.md new file mode 100644 index 000000000..def897232 --- /dev/null +++ b/modules/module-postgres-storage/CHANGELOG.md @@ -0,0 +1 @@ +# @powersync/service-module-postgres-storage diff --git a/modules/module-postgres-storage/LICENSE b/modules/module-postgres-storage/LICENSE new file mode 100644 index 000000000..c8efd46cc --- /dev/null +++ b/modules/module-postgres-storage/LICENSE @@ -0,0 +1,67 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2023-2024 Journey Mobile, Inc. + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under these Terms and Conditions, as indicated by our inclusion of these Terms and Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, Redistribution and Trademark clauses below, we hereby grant you the right to use, copy, modify, create derivative works, publicly perform, publicly display and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use means making the Software available to others in a commercial product or service that: + +1. substitutes for the Software; +2. substitutes for any other product or service we offer using the Software that exists as of the date we make the Software available; or +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; +2. for non-commercial education; +3. for non-commercial research; and +4. in connection with professional services that you provide to a licensee using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our patents, the license grant above includes a license under our patents. If you make a claim against any party that the Software infringes or contributes to the infringement of any patent, then your patent license to the Software ends immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of the Software. +If you redistribute any copies, modifications or derivatives of the Software, you must include a copy of or a link to these Terms and Conditions and not remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of the Software, you have no right under these Terms and Conditions to use our trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under the Apache License, Version 2.0 that is effective on the second anniversary of the date we make the Software available. On or after that date, you may use the Software under the Apache License, Version 2.0, in which case the following will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/modules/module-postgres-storage/README.md b/modules/module-postgres-storage/README.md new file mode 100644 index 000000000..f53bc947e --- /dev/null +++ b/modules/module-postgres-storage/README.md @@ -0,0 +1,67 @@ +# Postgres Sync Bucket Storage + +This module provides a `BucketStorageProvider` which uses a Postgres database for persistence. + +## Beta + +This feature is currently in a beta release. See [here](https://docs.powersync.com/resources/feature-status#feature-status) for more details. + +## Configuration + +Postgres storage can be enabled by selecting the appropriate storage `type` and providing connection details for a Postgres server. + +The storage connection configuration supports the same fields as the Postgres replication connection configuration. + +A sample YAML configuration could look like + +```yaml +replication: + # Specify database connection details + # Note only 1 connection is currently supported + # Multiple connection support is on the roadmap + connections: + - type: postgresql + uri: !env PS_DATA_SOURCE_URI + +# Connection settings for sync bucket storage +storage: + type: postgresql + # This accepts the same parameters as a Postgres replication source connection + uri: !env PS_STORAGE_SOURCE_URI +``` + +**IMPORTANT**: +A separate Postgres server is currently required for replication connections (if using Postgres for replication) and storage. Using the same server might cause unexpected results. + +### Connection credentials + +The Postgres bucket storage implementation requires write access to the provided Postgres database. The module will create a `powersync` schema in the provided database which will contain all the tables and data used for bucket storage. Ensure that the provided credentials specified in the `uri` or `username`, `password` configuration fields have the appropriate write access. + +A sample user could be created with the following + +If a `powersync` schema should be created manually + +```sql +CREATE USER powersync_storage_user +WITH + PASSWORD 'secure_password'; + +CREATE SCHEMA IF NOT EXISTS powersync AUTHORIZATION powersync_storage_user; + +GRANT CONNECT ON DATABASE postgres TO powersync_storage_user; + +GRANT USAGE ON SCHEMA powersync TO powersync_storage_user; + +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA powersync TO powersync_storage_user; +``` + +If the PowerSync service should create a `powersync` schema + +```sql +CREATE USER powersync_storage_user +WITH + PASSWORD 'secure_password'; + +-- The user should only have access to the schema it created +GRANT CREATE ON DATABASE postgres TO powersync_storage_user; +``` diff --git a/modules/module-postgres-storage/package.json b/modules/module-postgres-storage/package.json new file mode 100644 index 000000000..441c2f22a --- /dev/null +++ b/modules/module-postgres-storage/package.json @@ -0,0 +1,50 @@ +{ + "name": "@powersync/service-module-postgres-storage", + "repository": "https://github.com/powersync-ja/powersync-service", + "types": "dist/@types/index.d.ts", + "version": "0.0.1", + "main": "dist/index.js", + "type": "module", + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "tsc -b", + "build:tests": "tsc -b test/tsconfig.json", + "clean": "rm -rf ./lib && tsc -b --clean", + "test": "vitest" + }, + "exports": { + ".": { + "import": "./dist/index.js", + "require": "./dist/index.js", + "default": "./dist/index.js", + "types": "./dist/@types/index.d.ts" + }, + "./types": { + "import": "./dist/types/types.js", + "require": "./dist/types/types.js", + "default": "./dist/types/types.js", + "types": "./dist/@types/index.d.ts" + } + }, + "dependencies": { + "@powersync/lib-services-framework": "workspace:*", + "@powersync/lib-service-postgres": "workspace:*", + "@powersync/service-core": "workspace:*", + "@powersync/service-core-tests": "workspace:*", + "@powersync/service-jpgwire": "workspace:*", + "@powersync/service-jsonbig": "^0.17.10", + "@powersync/service-sync-rules": "workspace:*", + "@powersync/service-types": "workspace:*", + "ix": "^5.0.0", + "lru-cache": "^10.2.2", + "p-defer": "^4.0.1", + "ts-codec": "^1.3.0", + "uuid": "^9.0.1" + }, + "devDependencies": { + "@types/uuid": "^9.0.4", + "typescript": "^5.2.2" + } +} diff --git a/modules/module-postgres-storage/src/index.ts b/modules/module-postgres-storage/src/index.ts new file mode 100644 index 000000000..3665b4e09 --- /dev/null +++ b/modules/module-postgres-storage/src/index.ts @@ -0,0 +1,10 @@ +export * from './module/PostgresStorageModule.js'; + +export * from './migrations/PostgresMigrationAgent.js'; + +export * from './utils/utils-index.js'; +export * as utils from './utils/utils-index.js'; + +export * from './storage/storage-index.js'; +export * as storage from './storage/storage-index.js'; +export * from './types/types.js'; diff --git a/modules/module-postgres-storage/src/migrations/PostgresMigrationAgent.ts b/modules/module-postgres-storage/src/migrations/PostgresMigrationAgent.ts new file mode 100644 index 000000000..ba6831d06 --- /dev/null +++ b/modules/module-postgres-storage/src/migrations/PostgresMigrationAgent.ts @@ -0,0 +1,46 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; +import * as framework from '@powersync/lib-services-framework'; +import { migrations } from '@powersync/service-core'; +import * as path from 'path'; +import { fileURLToPath } from 'url'; + +import { normalizePostgresStorageConfig, PostgresStorageConfigDecoded } from '../types/types.js'; + +import { STORAGE_SCHEMA_NAME } from '../utils/db.js'; +import { PostgresMigrationStore } from './PostgresMigrationStore.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +const MIGRATIONS_DIR = path.join(__dirname, 'scripts'); + +export class PostgresMigrationAgent extends migrations.AbstractPowerSyncMigrationAgent { + store: framework.MigrationStore; + locks: framework.LockManager; + + protected db: lib_postgres.DatabaseClient; + + constructor(config: PostgresStorageConfigDecoded) { + super(); + + this.db = new lib_postgres.DatabaseClient({ + config: normalizePostgresStorageConfig(config), + schema: STORAGE_SCHEMA_NAME + }); + this.store = new PostgresMigrationStore({ + db: this.db + }); + this.locks = new lib_postgres.PostgresLockManager({ + name: 'migrations', + db: this.db + }); + } + + getInternalScriptsDir(): string { + return MIGRATIONS_DIR; + } + + async [Symbol.asyncDispose](): Promise { + await this.db[Symbol.asyncDispose](); + } +} diff --git a/modules/module-postgres-storage/src/migrations/PostgresMigrationStore.ts b/modules/module-postgres-storage/src/migrations/PostgresMigrationStore.ts new file mode 100644 index 000000000..11b1d68c7 --- /dev/null +++ b/modules/module-postgres-storage/src/migrations/PostgresMigrationStore.ts @@ -0,0 +1,70 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; +import { migrations } from '@powersync/lib-services-framework'; +import { models } from '../types/types.js'; +import { sql } from '../utils/db.js'; + +export type PostgresMigrationStoreOptions = { + db: lib_postgres.DatabaseClient; +}; + +export class PostgresMigrationStore implements migrations.MigrationStore { + constructor(protected options: PostgresMigrationStoreOptions) {} + + protected get db() { + return this.options.db; + } + + async init() { + await this.db.query(sql` + CREATE TABLE IF NOT EXISTS migrations ( + id SERIAL PRIMARY KEY, + last_run TEXT, + LOG JSONB NOT NULL + ); + `); + } + + async clear() { + await this.db.query(sql`DELETE FROM migrations;`); + } + + async load(): Promise { + const res = await this.db.sql` + SELECT + last_run, + LOG + FROM + migrations + LIMIT + 1 + ` + .decoded(models.Migration) + .first(); + + if (!res) { + return; + } + + return { + last_run: res.last_run, + log: res.log + }; + } + + async save(state: migrations.MigrationState): Promise { + await this.db.query(sql` + INSERT INTO + migrations (id, last_run, LOG) + VALUES + ( + 1, + ${{ type: 'varchar', value: state.last_run }}, + ${{ type: 'jsonb', value: state.log }} + ) + ON CONFLICT (id) DO UPDATE + SET + last_run = EXCLUDED.last_run, + LOG = EXCLUDED.log; + `); + } +} diff --git a/modules/module-postgres-storage/src/migrations/migration-utils.ts b/modules/module-postgres-storage/src/migrations/migration-utils.ts new file mode 100644 index 000000000..be50c98cc --- /dev/null +++ b/modules/module-postgres-storage/src/migrations/migration-utils.ts @@ -0,0 +1,14 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; +import { configFile } from '@powersync/service-types'; +import { isPostgresStorageConfig, normalizePostgresStorageConfig, PostgresStorageConfig } from '../types/types.js'; +import { STORAGE_SCHEMA_NAME } from '../utils/db.js'; + +export const openMigrationDB = (config: configFile.BaseStorageConfig) => { + if (!isPostgresStorageConfig(config)) { + throw new Error(`Input storage configuration is not for Postgres`); + } + return new lib_postgres.DatabaseClient({ + config: normalizePostgresStorageConfig(PostgresStorageConfig.decode(config)), + schema: STORAGE_SCHEMA_NAME + }); +}; diff --git a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts new file mode 100644 index 000000000..0cd26784e --- /dev/null +++ b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts @@ -0,0 +1,141 @@ +import { migrations } from '@powersync/service-core'; + +import { dropTables } from '../../utils/db.js'; +import { openMigrationDB } from '../migration-utils.js'; + +export const up: migrations.PowerSyncMigrationFunction = async (context) => { + const { + service_context: { configuration } + } = context; + await using client = openMigrationDB(configuration.storage); + + /** + * Request an explicit connection which will automatically set the search + * path to the powersync schema + */ + await client.transaction(async (db) => { + await db.sql` + CREATE SEQUENCE op_id_sequence AS int8 START + WITH + 1 + `.execute(); + + await db.sql` + CREATE SEQUENCE sync_rules_id_sequence AS int START + WITH + 1 + `.execute(); + + await db.sql` + CREATE TABLE bucket_data ( + group_id integer NOT NULL, + bucket_name TEXT NOT NULL, + op_id bigint NOT NULL, + CONSTRAINT unique_id PRIMARY KEY (group_id, bucket_name, op_id), + op text NOT NULL, + source_table TEXT, + source_key bytea, + table_name TEXT, + row_id TEXT, + checksum bigint NOT NULL, + data TEXT, + target_op bigint + ) + `.execute(); + + await db.sql`CREATE TABLE instance (id TEXT PRIMARY KEY) `.execute(); + + await db.sql` + CREATE TABLE sync_rules ( + id INTEGER PRIMARY KEY, + state TEXT NOT NULL, + snapshot_done BOOLEAN NOT NULL DEFAULT FALSE, + last_checkpoint BIGINT, + last_checkpoint_lsn TEXT, + no_checkpoint_before TEXT, + slot_name TEXT, + last_checkpoint_ts TIMESTAMP WITH TIME ZONE, + last_keepalive_ts TIMESTAMP WITH TIME ZONE, + keepalive_op BIGINT, + last_fatal_error TEXT, + content TEXT NOT NULL + ); + `.execute(); + + await db.sql` + CREATE TABLE bucket_parameters ( + id BIGINT DEFAULT nextval('op_id_sequence') PRIMARY KEY, + group_id integer NOT NULL, + source_table TEXT NOT NULL, + source_key bytea NOT NULL, + lookup bytea NOT NULL, + --- Stored as text which is stringified with JSONBig + --- BigInts are not standard JSON, storing as JSONB seems risky + bucket_parameters text NOT NULL + ); + `.execute(); + + await db.sql` + CREATE INDEX bucket_parameters_lookup_index ON bucket_parameters (group_id ASC, lookup ASC, id DESC) + `.execute(); + + await db.sql` + CREATE INDEX bucket_parameters_source_index ON bucket_parameters (group_id, source_table, source_key) + `.execute(); + + await db.sql` + CREATE TABLE current_data ( + group_id integer NOT NULL, + source_table TEXT NOT NULL, + source_key bytea NOT NULL, + CONSTRAINT unique_current_data_id PRIMARY KEY (group_id, source_table, source_key), + buckets jsonb NOT NULL, + data bytea NOT NULL, + lookups bytea[] NOT NULL + ); + `.execute(); + + await db.sql` + CREATE TABLE source_tables ( + --- This is currently a TEXT column to make the (shared) tests easier to integrate + --- we could improve this if necessary + id TEXT PRIMARY KEY, + group_id integer NOT NULL, + connection_id integer NOT NULL, + relation_id jsonb, + schema_name text NOT NULL, + table_name text NOT NULL, + replica_id_columns jsonb, + snapshot_done BOOLEAN NOT NULL DEFAULT FALSE + ) + `.execute(); + + await db.sql`CREATE INDEX source_table_lookup ON source_tables (group_id, table_name)`.execute(); + + await db.sql` + CREATE TABLE write_checkpoints ( + user_id text PRIMARY KEY, + lsns jsonb NOT NULL, + write_checkpoint BIGINT NOT NULL + ) + `.execute(); + + await db.sql` + CREATE TABLE custom_write_checkpoints ( + user_id text NOT NULL, + write_checkpoint BIGINT NOT NULL, + sync_rules_id integer NOT NULL, + CONSTRAINT unique_user_sync PRIMARY KEY (user_id, sync_rules_id) + ); + `.execute(); + }); +}; + +export const down: migrations.PowerSyncMigrationFunction = async (context) => { + const { + service_context: { configuration } + } = context; + await using client = openMigrationDB(configuration.storage); + + await dropTables(client); +}; diff --git a/modules/module-postgres-storage/src/module/PostgresStorageModule.ts b/modules/module-postgres-storage/src/module/PostgresStorageModule.ts new file mode 100644 index 000000000..90cbbb430 --- /dev/null +++ b/modules/module-postgres-storage/src/module/PostgresStorageModule.ts @@ -0,0 +1,30 @@ +import { modules, system } from '@powersync/service-core'; + +import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js'; +import { PostgresStorageProvider } from '../storage/PostgresStorageProvider.js'; +import { isPostgresStorageConfig, PostgresStorageConfig } from '../types/types.js'; + +export class PostgresStorageModule extends modules.AbstractModule { + constructor() { + super({ + name: 'Postgres Bucket Storage' + }); + } + + async initialize(context: system.ServiceContextContainer): Promise { + const { storageEngine } = context; + + // Register the ability to use Postgres as a BucketStorage + storageEngine.registerProvider(new PostgresStorageProvider()); + + if (isPostgresStorageConfig(context.configuration.storage)) { + context.migrations.registerMigrationAgent( + new PostgresMigrationAgent(PostgresStorageConfig.decode(context.configuration.storage)) + ); + } + } + + async teardown(): Promise { + // Teardown for this module is implemented in the storage engine + } +} diff --git a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts new file mode 100644 index 000000000..e2e6c4540 --- /dev/null +++ b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts @@ -0,0 +1,496 @@ +import * as framework from '@powersync/lib-services-framework'; +import { storage, sync, utils } from '@powersync/service-core'; +import * as pg_wire from '@powersync/service-jpgwire'; +import * as sync_rules from '@powersync/service-sync-rules'; +import crypto from 'crypto'; +import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js'; +import { LRUCache } from 'lru-cache/min'; +import * as timers from 'timers/promises'; +import * as uuid from 'uuid'; + +import * as lib_postgres from '@powersync/lib-service-postgres'; +import { models, NormalizedPostgresStorageConfig } from '../types/types.js'; + +import { NOTIFICATION_CHANNEL, STORAGE_SCHEMA_NAME } from '../utils/db.js'; +import { notifySyncRulesUpdate } from './batch/PostgresBucketBatch.js'; +import { PostgresSyncRulesStorage } from './PostgresSyncRulesStorage.js'; +import { PostgresPersistedSyncRulesContent } from './sync-rules/PostgresPersistedSyncRulesContent.js'; + +export type PostgresBucketStorageOptions = { + config: NormalizedPostgresStorageConfig; + slot_name_prefix: string; +}; + +export class PostgresBucketStorageFactory + extends framework.DisposableObserver + implements storage.BucketStorageFactory +{ + readonly db: lib_postgres.DatabaseClient; + public readonly slot_name_prefix: string; + + private sharedIterator = new sync.BroadcastIterable((signal) => this.watchActiveCheckpoint(signal)); + + private readonly storageCache = new LRUCache({ + max: 3, + fetchMethod: async (id) => { + const syncRulesRow = await this.db.sql` + SELECT + * + FROM + sync_rules + WHERE + id = ${{ value: id, type: 'int4' }} + ` + .decoded(models.SyncRules) + .first(); + if (syncRulesRow == null) { + // Deleted in the meantime? + return undefined; + } + const rules = new PostgresPersistedSyncRulesContent(this.db, syncRulesRow); + return this.getInstance(rules); + }, + dispose: (storage) => { + storage[Symbol.dispose](); + } + }); + + constructor(protected options: PostgresBucketStorageOptions) { + super(); + this.db = new lib_postgres.DatabaseClient({ + config: options.config, + schema: STORAGE_SCHEMA_NAME, + notificationChannels: [NOTIFICATION_CHANNEL] + }); + this.slot_name_prefix = options.slot_name_prefix; + + this.db.registerListener({ + connectionCreated: async (connection) => this.prepareStatements(connection) + }); + } + + async [Symbol.asyncDispose]() { + super[Symbol.dispose](); + await this.db[Symbol.asyncDispose](); + } + + async prepareStatements(connection: pg_wire.PgConnection) { + // It should be possible to prepare statements for some common operations here. + // This has not been implemented yet. + } + + getInstance(syncRules: storage.PersistedSyncRulesContent): storage.SyncRulesBucketStorage { + const storage = new PostgresSyncRulesStorage({ + factory: this, + db: this.db, + sync_rules: syncRules, + batchLimits: this.options.config.batch_limits + }); + this.iterateListeners((cb) => cb.syncStorageCreated?.(storage)); + storage.registerListener({ + batchStarted: (batch) => { + // This nested listener will be automatically disposed when the storage is disposed + batch.registerManagedListener(storage, { + replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload)) + }); + } + }); + return storage; + } + + async getStorageMetrics(): Promise { + const active_sync_rules = await this.getActiveSyncRules({ defaultSchema: 'public' }); + if (active_sync_rules == null) { + return { + operations_size_bytes: 0, + parameters_size_bytes: 0, + replication_size_bytes: 0 + }; + } + + const sizes = await this.db.sql` + SELECT + pg_total_relation_size('current_data') AS current_size_bytes, + pg_total_relation_size('bucket_parameters') AS parameter_size_bytes, + pg_total_relation_size('bucket_data') AS operations_size_bytes; + `.first<{ current_size_bytes: bigint; parameter_size_bytes: bigint; operations_size_bytes: bigint }>(); + + return { + operations_size_bytes: Number(sizes!.operations_size_bytes), + parameters_size_bytes: Number(sizes!.parameter_size_bytes), + replication_size_bytes: Number(sizes!.current_size_bytes) + }; + } + + async getPowerSyncInstanceId(): Promise { + const instanceRow = await this.db.sql` + SELECT + id + FROM + instance + ` + .decoded(models.Instance) + .first(); + if (instanceRow) { + return instanceRow.id; + } + const lockManager = new lib_postgres.PostgresLockManager({ + db: this.db, + name: `instance-id-insertion-lock` + }); + await lockManager.lock(async () => { + await this.db.sql` + INSERT INTO + instance (id) + VALUES + (${{ type: 'varchar', value: uuid.v4() }}) + `.execute(); + }); + const newInstanceRow = await this.db.sql` + SELECT + id + FROM + instance + ` + .decoded(models.Instance) + .first(); + return newInstanceRow!.id; + } + + // TODO possibly share implementation in abstract class + async configureSyncRules( + sync_rules: string, + options?: { lock?: boolean } + ): Promise<{ + updated: boolean; + persisted_sync_rules?: storage.PersistedSyncRulesContent; + lock?: storage.ReplicationLock; + }> { + const next = await this.getNextSyncRulesContent(); + const active = await this.getActiveSyncRulesContent(); + + if (next?.sync_rules_content == sync_rules) { + framework.logger.info('Sync rules from configuration unchanged'); + return { updated: false }; + } else if (next == null && active?.sync_rules_content == sync_rules) { + framework.logger.info('Sync rules from configuration unchanged'); + return { updated: false }; + } else { + framework.logger.info('Sync rules updated from configuration'); + const persisted_sync_rules = await this.updateSyncRules({ + content: sync_rules, + lock: options?.lock + }); + return { updated: true, persisted_sync_rules, lock: persisted_sync_rules.current_lock ?? undefined }; + } + } + + async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise { + // TODO some shared implementation for this might be nice + // Parse and validate before applying any changes + sync_rules.SqlSyncRules.fromYaml(options.content, { + // No schema-based validation at this point + schema: undefined, + defaultSchema: 'not_applicable', // Not needed for validation + throwOnError: true + }); + + return this.db.transaction(async (db) => { + await db.sql` + UPDATE sync_rules + SET + state = ${{ type: 'varchar', value: storage.SyncRuleState.STOP }} + WHERE + state = ${{ type: 'varchar', value: storage.SyncRuleState.PROCESSING }} + `.execute(); + + const newSyncRulesRow = await db.sql` + WITH + next_id AS ( + SELECT + nextval('sync_rules_id_sequence') AS id + ) + INSERT INTO + sync_rules (id, content, state, slot_name) + VALUES + ( + ( + SELECT + id + FROM + next_id + ), + ${{ type: 'varchar', value: options.content }}, + ${{ type: 'varchar', value: storage.SyncRuleState.PROCESSING }}, + CONCAT( + ${{ type: 'varchar', value: this.slot_name_prefix }}, + ( + SELECT + id + FROM + next_id + ), + '_', + ${{ type: 'varchar', value: crypto.randomBytes(2).toString('hex') }} + ) + ) + RETURNING + * + ` + .decoded(models.SyncRules) + .first(); + + await notifySyncRulesUpdate(this.db, newSyncRulesRow!); + + return new PostgresPersistedSyncRulesContent(this.db, newSyncRulesRow!); + }); + } + + async slotRemoved(slot_name: string): Promise { + const next = await this.getNextSyncRulesContent(); + const active = await this.getActiveSyncRulesContent(); + + // In both the below cases, we create a new sync rules instance. + // The current one will continue erroring until the next one has finished processing. + if (next != null && next.slot_name == slot_name) { + // We need to redo the "next" sync rules + await this.updateSyncRules({ + content: next.sync_rules_content + }); + // Pro-actively stop replicating + await this.db.sql` + UPDATE sync_rules + SET + state = ${{ value: storage.SyncRuleState.STOP, type: 'varchar' }} + WHERE + id = ${{ value: next.id, type: 'int4' }} + AND state = ${{ value: storage.SyncRuleState.PROCESSING, type: 'varchar' }} + `.execute(); + } else if (next == null && active?.slot_name == slot_name) { + // Slot removed for "active" sync rules, while there is no "next" one. + await this.updateSyncRules({ + content: active.sync_rules_content + }); + + // Pro-actively stop replicating + await this.db.sql` + UPDATE sync_rules + SET + state = ${{ value: storage.SyncRuleState.STOP, type: 'varchar' }} + WHERE + id = ${{ value: active.id, type: 'int4' }} + AND state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }} + `.execute(); + } + } + + // TODO possibly share via abstract class + async getActiveSyncRules(options: storage.ParseSyncRulesOptions): Promise { + const content = await this.getActiveSyncRulesContent(); + return content?.parsed(options) ?? null; + } + + async getActiveSyncRulesContent(): Promise { + const activeRow = await this.db.sql` + SELECT + * + FROM + sync_rules + WHERE + state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }} + ORDER BY + id DESC + LIMIT + 1 + ` + .decoded(models.SyncRules) + .first(); + if (!activeRow) { + return null; + } + + return new PostgresPersistedSyncRulesContent(this.db, activeRow); + } + + // TODO possibly share via abstract class + async getNextSyncRules(options: storage.ParseSyncRulesOptions): Promise { + const content = await this.getNextSyncRulesContent(); + return content?.parsed(options) ?? null; + } + + async getNextSyncRulesContent(): Promise { + const nextRow = await this.db.sql` + SELECT + * + FROM + sync_rules + WHERE + state = ${{ value: storage.SyncRuleState.PROCESSING, type: 'varchar' }} + ORDER BY + id DESC + LIMIT + 1 + ` + .decoded(models.SyncRules) + .first(); + if (!nextRow) { + return null; + } + + return new PostgresPersistedSyncRulesContent(this.db, nextRow); + } + + async getReplicatingSyncRules(): Promise { + const rows = await this.db.sql` + SELECT + * + FROM + sync_rules + WHERE + state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }} + OR state = ${{ value: storage.SyncRuleState.PROCESSING, type: 'varchar' }} + ` + .decoded(models.SyncRules) + .rows(); + + return rows.map((row) => new PostgresPersistedSyncRulesContent(this.db, row)); + } + + async getStoppedSyncRules(): Promise { + const rows = await this.db.sql` + SELECT + * + FROM + sync_rules + WHERE + state = ${{ value: storage.SyncRuleState.STOP, type: 'varchar' }} + ` + .decoded(models.SyncRules) + .rows(); + + return rows.map((row) => new PostgresPersistedSyncRulesContent(this.db, row)); + } + + async getActiveCheckpoint(): Promise { + const activeCheckpoint = await this.db.sql` + SELECT + id, + last_checkpoint, + last_checkpoint_lsn + FROM + sync_rules + WHERE + state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }} + ORDER BY + id DESC + LIMIT + 1 + ` + .decoded(models.ActiveCheckpoint) + .first(); + + return this.makeActiveCheckpoint(activeCheckpoint); + } + + async *watchWriteCheckpoint(user_id: string, signal: AbortSignal): AsyncIterable { + let lastCheckpoint: utils.OpId | null = null; + let lastWriteCheckpoint: bigint | null = null; + + const iter = wrapWithAbort(this.sharedIterator, signal); + for await (const cp of iter) { + const { checkpoint, lsn } = cp; + + // lsn changes are not important by itself. + // What is important is: + // 1. checkpoint (op_id) changes. + // 2. write checkpoint changes for the specific user + const bucketStorage = await cp.getBucketStorage(); + if (!bucketStorage) { + continue; + } + + const lsnFilters: Record = lsn ? { 1: lsn } : {}; + + const currentWriteCheckpoint = await bucketStorage.lastWriteCheckpoint({ + user_id, + heads: { + ...lsnFilters + } + }); + + if (currentWriteCheckpoint == lastWriteCheckpoint && checkpoint == lastCheckpoint) { + // No change - wait for next one + // In some cases, many LSNs may be produced in a short time. + // Add a delay to throttle the write checkpoint lookup a bit. + await timers.setTimeout(20 + 10 * Math.random()); + continue; + } + + lastWriteCheckpoint = currentWriteCheckpoint; + lastCheckpoint = checkpoint; + + yield { base: cp, writeCheckpoint: currentWriteCheckpoint }; + } + } + + protected async *watchActiveCheckpoint(signal: AbortSignal): AsyncIterable { + const doc = await this.db.sql` + SELECT + id, + last_checkpoint, + last_checkpoint_lsn + FROM + sync_rules + WHERE + state = ${{ type: 'varchar', value: storage.SyncRuleState.ACTIVE }} + LIMIT + 1 + ` + .decoded(models.ActiveCheckpoint) + .first(); + + const sink = new sync.LastValueSink(undefined); + + const disposeListener = this.db.registerListener({ + notification: (notification) => sink.next(notification.payload) + }); + + signal.addEventListener('aborted', async () => { + disposeListener(); + sink.complete(); + }); + + yield this.makeActiveCheckpoint(doc); + + let lastOp: storage.ActiveCheckpoint | null = null; + for await (const payload of sink.withSignal(signal)) { + if (signal.aborted) { + return; + } + + const notification = models.ActiveCheckpointNotification.decode(payload); + const activeCheckpoint = this.makeActiveCheckpoint(notification.active_checkpoint); + + if (lastOp == null || activeCheckpoint.lsn != lastOp.lsn || activeCheckpoint.checkpoint != lastOp.checkpoint) { + lastOp = activeCheckpoint; + yield activeCheckpoint; + } + } + } + + private makeActiveCheckpoint(row: models.ActiveCheckpointDecoded | null) { + return { + checkpoint: utils.timestampToOpId(row?.last_checkpoint ?? 0n), + lsn: row?.last_checkpoint_lsn ?? null, + hasSyncRules() { + return row != null; + }, + getBucketStorage: async () => { + if (row == null) { + return null; + } + return (await this.storageCache.fetch(Number(row.id))) ?? null; + } + } satisfies storage.ActiveCheckpoint; + } +} diff --git a/modules/module-postgres-storage/src/storage/PostgresCompactor.ts b/modules/module-postgres-storage/src/storage/PostgresCompactor.ts new file mode 100644 index 000000000..b81ba5f78 --- /dev/null +++ b/modules/module-postgres-storage/src/storage/PostgresCompactor.ts @@ -0,0 +1,366 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; +import { logger } from '@powersync/lib-services-framework'; +import { storage, utils } from '@powersync/service-core'; +import * as pgwire from '@powersync/service-jpgwire'; +import * as t from 'ts-codec'; +import { BIGINT_MAX } from '../types/codecs.js'; +import { models } from '../types/types.js'; +import { sql } from '../utils/db.js'; +import { pick } from '../utils/ts-codec.js'; +import { encodedCacheKey } from './batch/OperationBatch.js'; + +interface CurrentBucketState { + /** Bucket name */ + bucket: string; + /** + * Rows seen in the bucket, with the last op_id of each. + */ + seen: Map; + /** + * Estimated memory usage of the seen Map. + */ + trackingSize: number; + + /** + * Last (lowest) seen op_id that is not a PUT. + */ + lastNotPut: bigint | null; + + /** + * Number of REMOVE/MOVE operations seen since lastNotPut. + */ + opsSincePut: number; +} + +/** + * Additional options, primarily for testing. + */ +export interface PostgresCompactOptions extends storage.CompactOptions { + /** Minimum of 2 */ + clearBatchLimit?: number; + /** Minimum of 1 */ + moveBatchLimit?: number; + /** Minimum of 1 */ + moveBatchQueryLimit?: number; +} + +const DEFAULT_CLEAR_BATCH_LIMIT = 5000; +const DEFAULT_MOVE_BATCH_LIMIT = 2000; +const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000; + +/** This default is primarily for tests. */ +const DEFAULT_MEMORY_LIMIT_MB = 64; + +export class PostgresCompactor { + private updates: pgwire.Statement[] = []; + + private idLimitBytes: number; + private moveBatchLimit: number; + private moveBatchQueryLimit: number; + private clearBatchLimit: number; + private maxOpId: bigint | undefined; + private buckets: string[] | undefined; + + constructor( + private db: lib_postgres.DatabaseClient, + private group_id: number, + options?: PostgresCompactOptions + ) { + this.idLimitBytes = (options?.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024; + this.moveBatchLimit = options?.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT; + this.moveBatchQueryLimit = options?.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT; + this.clearBatchLimit = options?.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT; + this.maxOpId = options?.maxOpId; + this.buckets = options?.compactBuckets; + } + + /** + * Compact buckets by converting operations into MOVE and/or CLEAR operations. + * + * See /docs/compacting-operations.md for details. + */ + async compact() { + if (this.buckets) { + for (let bucket of this.buckets) { + // We can make this more efficient later on by iterating + // through the buckets in a single query. + // That makes batching more tricky, so we leave for later. + await this.compactInternal(bucket); + } + } else { + await this.compactInternal(undefined); + } + } + + async compactInternal(bucket: string | undefined) { + const idLimitBytes = this.idLimitBytes; + + let currentState: CurrentBucketState | null = null; + + let bucketLower: string | null = null; + let bucketUpper: string | null = null; + + if (bucket?.includes('[')) { + // Exact bucket name + bucketLower = bucket; + bucketUpper = bucket; + } else if (bucket) { + // Bucket definition name + bucketLower = `${bucket}[`; + bucketUpper = `${bucket}[\uFFFF`; + } + + let upperOpIdLimit = BIGINT_MAX; + + while (true) { + const batch = await this.db.sql` + SELECT + op, + op_id, + source_table, + table_name, + row_id, + source_key, + bucket_name + FROM + bucket_data + WHERE + group_id = ${{ type: 'int4', value: this.group_id }} + AND bucket_name LIKE COALESCE(${{ type: 'varchar', value: bucketLower }}, '%') + AND op_id < ${{ type: 'int8', value: upperOpIdLimit }} + ORDER BY + bucket_name, + op_id DESC + LIMIT + ${{ type: 'int4', value: this.moveBatchQueryLimit }} + ` + .decoded( + pick(models.BucketData, ['op', 'source_table', 'table_name', 'source_key', 'row_id', 'op_id', 'bucket_name']) + ) + .rows(); + + if (batch.length == 0) { + // We've reached the end + break; + } + + // Set upperBound for the next batch + upperOpIdLimit = batch[batch.length - 1].op_id; + + for (const doc of batch) { + if (currentState == null || doc.bucket_name != currentState.bucket) { + if (currentState != null && currentState.lastNotPut != null && currentState.opsSincePut >= 1) { + // Important to flush before clearBucket() + await this.flush(); + logger.info( + `Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations` + ); + + const bucket = currentState.bucket; + const clearOp = currentState.lastNotPut; + // Free memory before clearing bucket + currentState = null; + await this.clearBucket(bucket, clearOp); + } + currentState = { + bucket: doc.bucket_name, + seen: new Map(), + trackingSize: 0, + lastNotPut: null, + opsSincePut: 0 + }; + } + + if (this.maxOpId != null && doc.op_id > this.maxOpId) { + continue; + } + + let isPersistentPut = doc.op == 'PUT'; + + if (doc.op == 'REMOVE' || doc.op == 'PUT') { + const key = `${doc.table_name}/${doc.row_id}/${encodedCacheKey(doc.source_table!, doc.source_key!)}`; + const targetOp = currentState.seen.get(utils.flatstr(key)); + if (targetOp) { + // Will convert to MOVE, so don't count as PUT + isPersistentPut = false; + + this.updates.push(sql` + UPDATE bucket_data + SET + op = 'MOVE', + target_op = ${{ type: 'int8', value: targetOp }}, + table_name = NULL, + row_id = NULL, + data = NULL, + source_table = NULL, + source_key = NULL + WHERE + group_id = ${{ type: 'int4', value: this.group_id }} + AND bucket_name = ${{ type: 'varchar', value: doc.bucket_name }} + AND op_id = ${{ type: 'int8', value: doc.op_id }} + `); + } else { + if (currentState.trackingSize >= idLimitBytes) { + // Reached memory limit. + // Keep the highest seen values in this case. + } else { + // flatstr reduces the memory usage by flattening the string + currentState.seen.set(utils.flatstr(key), doc.op_id); + // length + 16 for the string + // 24 for the bigint + // 50 for map overhead + // 50 for additional overhead + currentState.trackingSize += key.length + 140; + } + } + } + + if (isPersistentPut) { + currentState.lastNotPut = null; + currentState.opsSincePut = 0; + } else if (doc.op != 'CLEAR') { + if (currentState.lastNotPut == null) { + currentState.lastNotPut = doc.op_id; + } + currentState.opsSincePut += 1; + } + + if (this.updates.length >= this.moveBatchLimit) { + await this.flush(); + } + } + } + + await this.flush(); + currentState?.seen.clear(); + if (currentState?.lastNotPut != null && currentState?.opsSincePut > 1) { + logger.info( + `Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations` + ); + const bucket = currentState.bucket; + const clearOp = currentState.lastNotPut; + // Free memory before clearing bucket + currentState = null; + await this.clearBucket(bucket, clearOp); + } + } + + private async flush() { + if (this.updates.length > 0) { + logger.info(`Compacting ${this.updates.length} ops`); + await this.db.query(...this.updates); + this.updates = []; + } + } + + /** + * Perform a CLEAR compact for a bucket. + * + * @param bucket bucket name + * @param op op_id of the last non-PUT operation, which will be converted to CLEAR. + */ + private async clearBucket(bucket: string, op: bigint) { + /** + * This entire method could be implemented as a Postgres function, but this might make debugging + * a bit more challenging. + */ + let done = false; + while (!done) { + await this.db.lockConnection(async (db) => { + /** + * Start a transaction where each read returns the state at the start of the transaction,. + * Similar to the MongoDB readConcern: { level: 'snapshot' } mode. + */ + await db.sql`BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ`.execute(); + + try { + let checksum = 0; + let lastOpId: bigint | null = null; + let targetOp: bigint | null = null; + let gotAnOp = false; + + const codec = pick(models.BucketData, ['op', 'source_table', 'source_key', 'op_id', 'checksum', 'target_op']); + for await (const operations of db.streamRows>(sql` + SELECT + source_table, + source_key, + op, + op_id, + checksum, + target_op + FROM + bucket_data + WHERE + group_id = ${{ type: 'int4', value: this.group_id }} + AND bucket_name = ${{ type: 'varchar', value: bucket }} + AND op_id <= ${{ type: 'int8', value: op }} + ORDER BY + op_id + LIMIT + ${{ type: 'int4', value: this.clearBatchLimit }} + `)) { + const decodedOps = operations.map((o) => codec.decode(o)); + for (const op of decodedOps) { + if ([models.OpType.MOVE, models.OpType.REMOVE, models.OpType.CLEAR].includes(op.op)) { + checksum = utils.addChecksums(checksum, Number(op.checksum)); + lastOpId = op.op_id; + if (op.op != models.OpType.CLEAR) { + gotAnOp = true; + } + if (op.target_op != null) { + if (targetOp == null || op.target_op > targetOp) { + targetOp = op.target_op; + } + } + } else { + throw new Error(`Unexpected ${op.op} operation at ${this.group_id}:${bucket}:${op.op_id}`); + } + } + } + + if (!gotAnOp) { + await db.sql`COMMIT`.execute(); + done = true; + return; + } + + logger.info(`Flushing CLEAR at ${lastOpId}`); + + await db.sql` + DELETE FROM bucket_data + WHERE + group_id = ${{ type: 'int4', value: this.group_id }} + AND bucket_name = ${{ type: 'varchar', value: bucket }} + AND op_id <= ${{ type: 'int8', value: lastOpId }} + `.execute(); + + await db.sql` + INSERT INTO + bucket_data ( + group_id, + bucket_name, + op_id, + op, + checksum, + target_op + ) + VALUES + ( + ${{ type: 'int4', value: this.group_id }}, + ${{ type: 'varchar', value: bucket }}, + ${{ type: 'int8', value: lastOpId }}, + ${{ type: 'varchar', value: models.OpType.CLEAR }}, + ${{ type: 'int8', value: checksum }}, + ${{ type: 'int8', value: targetOp }} + ) + `.execute(); + + await db.sql`COMMIT`.execute(); + } catch (ex) { + await db.sql`ROLLBACK`.execute(); + throw ex; + } + }); + } + } +} diff --git a/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts b/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts new file mode 100644 index 000000000..7125ec6a8 --- /dev/null +++ b/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts @@ -0,0 +1,42 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; +import { logger } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; + +import { isPostgresStorageConfig, normalizePostgresStorageConfig, PostgresStorageConfig } from '../types/types.js'; +import { dropTables } from '../utils/db.js'; +import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js'; + +export class PostgresStorageProvider implements storage.BucketStorageProvider { + get type() { + return lib_postgres.POSTGRES_CONNECTION_TYPE; + } + + async getStorage(options: storage.GetStorageOptions): Promise { + const { resolvedConfig } = options; + + const { storage } = resolvedConfig; + if (!isPostgresStorageConfig(storage)) { + // This should not be reached since the generation should be managed externally. + throw new Error( + `Cannot create Postgres bucket storage with provided config ${storage.type} !== ${lib_postgres.POSTGRES_CONNECTION_TYPE}` + ); + } + + const decodedConfig = PostgresStorageConfig.decode(storage); + const normalizedConfig = normalizePostgresStorageConfig(decodedConfig); + const storageFactory = new PostgresBucketStorageFactory({ + config: normalizedConfig, + slot_name_prefix: options.resolvedConfig.slot_name_prefix + }); + return { + storage: storageFactory, + shutDown: async () => storageFactory.db[Symbol.asyncDispose](), + tearDown: async () => { + logger.info(`Tearing down Postgres storage: ${normalizedConfig.database}...`); + await dropTables(storageFactory.db); + await storageFactory.db[Symbol.asyncDispose](); + return true; + } + } satisfies storage.ActiveStorage; + } +} diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts new file mode 100644 index 000000000..6dc929ef9 --- /dev/null +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -0,0 +1,666 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; +import { DisposableObserver } from '@powersync/lib-services-framework'; +import { storage, utils } from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; +import * as sync_rules from '@powersync/service-sync-rules'; +import * as uuid from 'uuid'; +import { BIGINT_MAX } from '../types/codecs.js'; +import { models, RequiredOperationBatchLimits } from '../types/types.js'; +import { replicaIdToSubkey } from '../utils/bson.js'; +import { mapOpEntry } from '../utils/bucket-data.js'; + +import { StatementParam } from '@powersync/service-jpgwire'; +import { StoredRelationId } from '../types/models/SourceTable.js'; +import { pick } from '../utils/ts-codec.js'; +import { PostgresBucketBatch } from './batch/PostgresBucketBatch.js'; +import { PostgresWriteCheckpointAPI } from './checkpoints/PostgresWriteCheckpointAPI.js'; +import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js'; +import { PostgresCompactor } from './PostgresCompactor.js'; + +export type PostgresSyncRulesStorageOptions = { + factory: PostgresBucketStorageFactory; + db: lib_postgres.DatabaseClient; + sync_rules: storage.PersistedSyncRulesContent; + write_checkpoint_mode?: storage.WriteCheckpointMode; + batchLimits: RequiredOperationBatchLimits; +}; + +export class PostgresSyncRulesStorage + extends DisposableObserver + implements storage.SyncRulesBucketStorage +{ + public readonly group_id: number; + public readonly sync_rules: storage.PersistedSyncRulesContent; + public readonly slot_name: string; + public readonly factory: PostgresBucketStorageFactory; + + protected db: lib_postgres.DatabaseClient; + protected writeCheckpointAPI: PostgresWriteCheckpointAPI; + + // TODO we might be able to share this in an abstract class + private parsedSyncRulesCache: { parsed: sync_rules.SqlSyncRules; options: storage.ParseSyncRulesOptions } | undefined; + private checksumCache = new storage.ChecksumCache({ + fetchChecksums: (batch) => { + return this.getChecksumsInternal(batch); + } + }); + + constructor(protected options: PostgresSyncRulesStorageOptions) { + super(); + this.group_id = options.sync_rules.id; + this.db = options.db; + this.sync_rules = options.sync_rules; + this.slot_name = options.sync_rules.slot_name; + this.factory = options.factory; + + this.writeCheckpointAPI = new PostgresWriteCheckpointAPI({ + db: this.db, + mode: options.write_checkpoint_mode ?? storage.WriteCheckpointMode.MANAGED + }); + } + + get writeCheckpointMode(): storage.WriteCheckpointMode { + return this.writeCheckpointAPI.writeCheckpointMode; + } + + // TODO we might be able to share this in an abstract class + getParsedSyncRules(options: storage.ParseSyncRulesOptions): sync_rules.SqlSyncRules { + const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {}; + /** + * Check if the cached sync rules, if present, had the same options. + * Parse sync rules if the options are different or if there is no cached value. + */ + if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) { + this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).sync_rules, options }; + } + + return this.parsedSyncRulesCache!.parsed; + } + + async reportError(e: any): Promise { + const message = String(e.message ?? 'Replication failure'); + await this.db.sql` + UPDATE sync_rules + SET + last_fatal_error = ${{ type: 'varchar', value: message }} + WHERE + id = ${{ type: 'int4', value: this.group_id }}; + `.execute(); + } + + compact(options?: storage.CompactOptions): Promise { + return new PostgresCompactor(this.db, this.group_id, options).compact(); + } + + batchCreateCustomWriteCheckpoints(checkpoints: storage.BatchedCustomWriteCheckpointOptions[]): Promise { + return this.writeCheckpointAPI.batchCreateCustomWriteCheckpoints( + checkpoints.map((c) => ({ ...c, sync_rules_id: this.group_id })) + ); + } + + createCustomWriteCheckpoint(checkpoint: storage.BatchedCustomWriteCheckpointOptions): Promise { + return this.writeCheckpointAPI.createCustomWriteCheckpoint({ + ...checkpoint, + sync_rules_id: this.group_id + }); + } + + lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise { + return this.writeCheckpointAPI.lastWriteCheckpoint({ + ...filters, + sync_rules_id: this.group_id + }); + } + + setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void { + return this.writeCheckpointAPI.setWriteCheckpointMode(mode); + } + + createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise { + return this.writeCheckpointAPI.createManagedWriteCheckpoint(checkpoint); + } + + async getCheckpoint(): Promise { + const checkpointRow = await this.db.sql` + SELECT + last_checkpoint, + last_checkpoint_lsn + FROM + sync_rules + WHERE + id = ${{ type: 'int4', value: this.group_id }} + ` + .decoded(pick(models.SyncRules, ['last_checkpoint', 'last_checkpoint_lsn'])) + .first(); + + return { + checkpoint: utils.timestampToOpId(checkpointRow?.last_checkpoint ?? 0n), + lsn: checkpointRow?.last_checkpoint_lsn ?? null + }; + } + + async resolveTable(options: storage.ResolveTableOptions): Promise { + const { group_id, connection_id, connection_tag, entity_descriptor } = options; + + const { schema, name: table, objectId, replicationColumns } = entity_descriptor; + + const columns = replicationColumns.map((column) => ({ + name: column.name, + type: column.type, + // The PGWire returns this as a BigInt. We want to store this as JSONB + type_oid: typeof column.typeId !== 'undefined' ? Number(column.typeId) : column.typeId + })); + return this.db.transaction(async (db) => { + let sourceTableRow = await db.sql` + SELECT + * + FROM + source_tables + WHERE + group_id = ${{ type: 'int4', value: group_id }} + AND connection_id = ${{ type: 'int4', value: connection_id }} + AND relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }} + AND schema_name = ${{ type: 'varchar', value: schema }} + AND table_name = ${{ type: 'varchar', value: table }} + AND replica_id_columns = ${{ type: 'jsonb', value: columns }} + ` + .decoded(models.SourceTable) + .first(); + + if (sourceTableRow == null) { + const row = await db.sql` + INSERT INTO + source_tables ( + id, + group_id, + connection_id, + relation_id, + schema_name, + table_name, + replica_id_columns + ) + VALUES + ( + ${{ type: 'varchar', value: uuid.v4() }}, + ${{ type: 'int4', value: group_id }}, + ${{ type: 'int4', value: connection_id }}, + --- The objectId can be string | number, we store it as jsonb value + ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}, + ${{ type: 'varchar', value: schema }}, + ${{ type: 'varchar', value: table }}, + ${{ type: 'jsonb', value: columns }} + ) + RETURNING + * + ` + .decoded(models.SourceTable) + .first(); + sourceTableRow = row; + } + + const sourceTable = new storage.SourceTable( + sourceTableRow!.id, + connection_tag, + objectId, + schema, + table, + replicationColumns, + sourceTableRow!.snapshot_done ?? true + ); + sourceTable.syncEvent = options.sync_rules.tableTriggersEvent(sourceTable); + sourceTable.syncData = options.sync_rules.tableSyncsData(sourceTable); + sourceTable.syncParameters = options.sync_rules.tableSyncsParameters(sourceTable); + + const truncatedTables = await db.sql` + SELECT + * + FROM + source_tables + WHERE + group_id = ${{ type: 'int4', value: group_id }} + AND connection_id = ${{ type: 'int4', value: connection_id }} + AND id != ${{ type: 'varchar', value: sourceTableRow!.id }} + AND ( + relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }} + OR ( + schema_name = ${{ type: 'varchar', value: schema }} + AND table_name = ${{ type: 'varchar', value: table }} + ) + ) + ` + .decoded(models.SourceTable) + .rows(); + + return { + table: sourceTable, + dropTables: truncatedTables.map( + (doc) => + new storage.SourceTable( + doc.id, + connection_tag, + doc.relation_id?.object_id ?? 0, + doc.schema_name, + doc.table_name, + doc.replica_id_columns?.map((c) => ({ + name: c.name, + typeOid: c.typeId, + type: c.type + })) ?? [], + doc.snapshot_done ?? true + ) + ) + }; + }); + } + + async startBatch( + options: storage.StartBatchOptions, + callback: (batch: storage.BucketStorageBatch) => Promise + ): Promise { + const syncRules = await this.db.sql` + SELECT + last_checkpoint_lsn, + no_checkpoint_before, + keepalive_op + FROM + sync_rules + WHERE + id = ${{ type: 'int4', value: this.group_id }} + ` + .decoded(pick(models.SyncRules, ['last_checkpoint_lsn', 'no_checkpoint_before', 'keepalive_op'])) + .first(); + + const checkpoint_lsn = syncRules?.last_checkpoint_lsn ?? null; + + await using batch = new PostgresBucketBatch({ + db: this.db, + sync_rules: this.sync_rules.parsed(options).sync_rules, + group_id: this.group_id, + slot_name: this.slot_name, + last_checkpoint_lsn: checkpoint_lsn, + keep_alive_op: syncRules?.keepalive_op, + no_checkpoint_before_lsn: syncRules?.no_checkpoint_before ?? options.zeroLSN, + store_current_data: options.storeCurrentData, + skip_existing_rows: options.skipExistingRows ?? false, + batch_limits: this.options.batchLimits + }); + this.iterateListeners((cb) => cb.batchStarted?.(batch)); + + await callback(batch); + await batch.flush(); + if (batch.last_flushed_op) { + return { flushed_op: String(batch.last_flushed_op) }; + } else { + return null; + } + } + + async getParameterSets( + checkpoint: utils.OpId, + lookups: sync_rules.SqliteJsonValue[][] + ): Promise { + const rows = await this.db.sql` + SELECT DISTINCT + ON (lookup, source_table, source_key) lookup, + source_table, + source_key, + id, + bucket_parameters + FROM + bucket_parameters + WHERE + group_id = ${{ type: 'int4', value: this.group_id }} + AND lookup = ANY ( + SELECT + decode((FILTER ->> 0)::text, 'hex') -- Decode the hex string to bytea + FROM + jsonb_array_elements(${{ + type: 'jsonb', + value: lookups.map((l) => storage.serializeLookupBuffer(l).toString('hex')) + }}) AS FILTER + ) + AND id <= ${{ type: 'int8', value: BigInt(checkpoint) }} + ORDER BY + lookup, + source_table, + source_key, + id DESC + ` + .decoded(pick(models.BucketParameters, ['bucket_parameters'])) + .rows(); + + const groupedParameters = rows.map((row) => { + return JSONBig.parse(row.bucket_parameters) as sync_rules.SqliteJsonRow; + }); + return groupedParameters.flat(); + } + + async *getBucketDataBatch( + checkpoint: utils.OpId, + dataBuckets: Map, + options?: storage.BucketDataBatchOptions + ): AsyncIterable { + if (dataBuckets.size == 0) { + return; + } + + const end = checkpoint ?? BIGINT_MAX; + const filters = Array.from(dataBuckets.entries()).map(([name, start]) => ({ + bucket_name: name, + start: start + })); + + const rowLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT; + const sizeLimit = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES; + + let batchSize = 0; + let currentBatch: utils.SyncBucketData | null = null; + let targetOp: bigint | null = null; + let rowCount = 0; + + /** + * It is possible to perform this query with JSONB join. e.g. + * ```sql + * WITH + * filter_data AS ( + * SELECT + * FILTER ->> 'bucket_name' AS bucket_name, + * (FILTER ->> 'start')::BIGINT AS start_op_id + * FROM + * jsonb_array_elements($1::jsonb) AS FILTER + * ) + * SELECT + * b.*, + * octet_length(b.data) AS data_size + * FROM + * bucket_data b + * JOIN filter_data f ON b.bucket_name = f.bucket_name + * AND b.op_id > f.start_op_id + * AND b.op_id <= $2 + * WHERE + * b.group_id = $3 + * ORDER BY + * b.bucket_name ASC, + * b.op_id ASC + * LIMIT + * $4; + * ``` + * Which might be better for large volumes of buckets, but in testing the JSON method + * was significantly slower than the method below. Syncing 2.5 million rows in a single + * bucket takes 2 minutes and 11 seconds with the method below. With the JSON method + * 1 million rows were only synced before a 5 minute timeout. + */ + for await (const rows of this.db.streamRows({ + statement: ` + SELECT + * + FROM + bucket_data + WHERE + group_id = $1 + and op_id <= $2 + and ( + ${filters.map((f, index) => `(bucket_name = $${index * 2 + 4} and op_id > $${index * 2 + 5})`).join(' OR ')} + ) + ORDER BY + bucket_name ASC, + op_id ASC + LIMIT + $3;`, + params: [ + { type: 'int4', value: this.group_id }, + { type: 'int8', value: end }, + { type: 'int4', value: rowLimit + 1 }, + ...filters.flatMap((f) => [ + { type: 'varchar' as const, value: f.bucket_name }, + { type: 'int8' as const, value: f.start } satisfies StatementParam + ]) + ] + })) { + const decodedRows = rows.map((r) => models.BucketData.decode(r as any)); + + for (const row of decodedRows) { + const { bucket_name } = row; + const rowSize = row.data ? row.data.length : 0; + + if ( + currentBatch == null || + currentBatch.bucket != bucket_name || + batchSize >= sizeLimit || + (currentBatch?.data.length && batchSize + rowSize > sizeLimit) || + currentBatch.data.length >= rowLimit + ) { + let start: string | undefined = undefined; + if (currentBatch != null) { + if (currentBatch.bucket == bucket_name) { + currentBatch.has_more = true; + } + + const yieldBatch = currentBatch; + start = currentBatch.after; + currentBatch = null; + batchSize = 0; + yield { batch: yieldBatch, targetOp: targetOp }; + targetOp = null; + if (rowCount >= rowLimit) { + // We've yielded all the requested rows + break; + } + } + + start ??= dataBuckets.get(bucket_name); + if (start == null) { + throw new Error(`data for unexpected bucket: ${bucket_name}`); + } + currentBatch = { + bucket: bucket_name, + after: start, + has_more: false, + data: [], + next_after: start + }; + targetOp = null; + } + + const entry = mapOpEntry(row); + + if (row.source_table && row.source_key) { + entry.subkey = replicaIdToSubkey(row.source_table, storage.deserializeReplicaId(row.source_key)); + } + + if (row.target_op != null) { + // MOVE, CLEAR + const rowTargetOp = row.target_op; + if (targetOp == null || rowTargetOp > targetOp) { + targetOp = rowTargetOp; + } + } + + currentBatch.data.push(entry); + currentBatch.next_after = entry.op_id; + + batchSize += rowSize; + + // Manually track the total rows yielded + rowCount++; + } + } + + if (currentBatch != null) { + const yieldBatch = currentBatch; + currentBatch = null; + yield { batch: yieldBatch, targetOp: targetOp }; + targetOp = null; + } + } + + async getChecksums(checkpoint: utils.OpId, buckets: string[]): Promise { + return this.checksumCache.getChecksumMap(checkpoint, buckets); + } + + async terminate(options?: storage.TerminateOptions) { + if (!options || options?.clearStorage) { + await this.clear(); + } + await this.db.sql` + UPDATE sync_rules + SET + state = ${{ type: 'varchar', value: storage.SyncRuleState.TERMINATED }}, + snapshot_done = ${{ type: 'bool', value: false }} + WHERE + id = ${{ type: 'int4', value: this.group_id }} + `.execute(); + } + + async getStatus(): Promise { + const syncRulesRow = await this.db.sql` + SELECT + snapshot_done, + last_checkpoint_lsn, + state + FROM + sync_rules + WHERE + id = ${{ type: 'int4', value: this.group_id }} + ` + .decoded(pick(models.SyncRules, ['snapshot_done', 'last_checkpoint_lsn', 'state'])) + .first(); + + if (syncRulesRow == null) { + throw new Error('Cannot find sync rules status'); + } + + return { + snapshot_done: syncRulesRow.snapshot_done, + active: syncRulesRow.state == storage.SyncRuleState.ACTIVE, + checkpoint_lsn: syncRulesRow.last_checkpoint_lsn ?? null + }; + } + + async clear(): Promise { + await this.db.sql` + UPDATE sync_rules + SET + snapshot_done = FALSE, + last_checkpoint_lsn = NULL, + last_checkpoint = NULL, + no_checkpoint_before = NULL + WHERE + id = ${{ type: 'int4', value: this.group_id }} + `.execute(); + + await this.db.sql` + DELETE FROM bucket_data + WHERE + group_id = ${{ type: 'int4', value: this.group_id }} + `.execute(); + + await this.db.sql` + DELETE FROM bucket_parameters + WHERE + group_id = ${{ type: 'int4', value: this.group_id }} + `.execute(); + + await this.db.sql` + DELETE FROM current_data + WHERE + group_id = ${{ type: 'int4', value: this.group_id }} + `.execute(); + + await this.db.sql` + DELETE FROM source_tables + WHERE + group_id = ${{ type: 'int4', value: this.group_id }} + `.execute(); + } + + async autoActivate(): Promise { + await this.db.transaction(async (db) => { + const syncRulesRow = await db.sql` + SELECT + state + FROM + sync_rules + WHERE + id = ${{ type: 'int4', value: this.group_id }} + ` + .decoded(pick(models.SyncRules, ['state'])) + .first(); + + if (syncRulesRow && syncRulesRow.state == storage.SyncRuleState.PROCESSING) { + await db.sql` + UPDATE sync_rules + SET + state = ${{ type: 'varchar', value: storage.SyncRuleState.ACTIVE }} + WHERE + id = ${{ type: 'int4', value: this.group_id }} + `.execute(); + } + + await db.sql` + UPDATE sync_rules + SET + state = ${{ type: 'varchar', value: storage.SyncRuleState.STOP }} + WHERE + state = ${{ type: 'varchar', value: storage.SyncRuleState.ACTIVE }} + AND id != ${{ type: 'int4', value: this.group_id }} + `.execute(); + }); + } + + private async getChecksumsInternal(batch: storage.FetchPartialBucketChecksum[]): Promise { + if (batch.length == 0) { + return new Map(); + } + + const rangedBatch = batch.map((b) => ({ + ...b, + start: b.start ?? 0 + })); + + const results = await this.db.sql` + WITH + filter_data AS ( + SELECT + FILTER ->> 'bucket' AS bucket_name, + (FILTER ->> 'start')::BIGINT AS start_op_id, + (FILTER ->> 'end')::BIGINT AS end_op_id + FROM + jsonb_array_elements(${{ type: 'jsonb', value: rangedBatch }}::jsonb) AS FILTER + ) + SELECT + b.bucket_name AS bucket, + SUM(b.checksum) AS checksum_total, + COUNT(*) AS total, + MAX( + CASE + WHEN b.op = 'CLEAR' THEN 1 + ELSE 0 + END + ) AS has_clear_op + FROM + bucket_data b + JOIN filter_data f ON b.bucket_name = f.bucket_name + AND b.op_id > f.start_op_id + AND b.op_id <= f.end_op_id + WHERE + b.group_id = ${{ type: 'int4', value: this.group_id }} + GROUP BY + b.bucket_name; + `.rows<{ bucket: string; checksum_total: bigint; total: bigint; has_clear_op: number }>(); + + return new Map( + results.map((doc) => { + return [ + doc.bucket, + { + bucket: doc.bucket, + partialCount: Number(doc.total), + partialChecksum: Number(BigInt(doc.checksum_total) & 0xffffffffn) & 0xffffffff, + isFullChecksum: doc.has_clear_op == 1 + } satisfies storage.PartialChecksum + ]; + }) + ); + } +} diff --git a/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts b/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts new file mode 100644 index 000000000..1739beeba --- /dev/null +++ b/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts @@ -0,0 +1,61 @@ +import { framework, PowerSyncMigrationManager, ServiceContext, TestStorageOptions } from '@powersync/service-core'; +import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js'; +import { normalizePostgresStorageConfig, PostgresStorageConfigDecoded } from '../types/types.js'; +import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js'; + +export type PostgresTestStorageOptions = { + url: string; + /** + * Vitest can cause issues when loading .ts files for migrations. + * This allows for providing a custom PostgresMigrationAgent. + */ + migrationAgent?: (config: PostgresStorageConfigDecoded) => PostgresMigrationAgent; +}; + +export const PostgresTestStorageFactoryGenerator = (factoryOptions: PostgresTestStorageOptions) => { + return async (options?: TestStorageOptions) => { + try { + const migrationManager: PowerSyncMigrationManager = new framework.MigrationManager(); + + const BASE_CONFIG = { + type: 'postgresql' as const, + uri: factoryOptions.url, + sslmode: 'disable' as const + }; + + const TEST_CONNECTION_OPTIONS = normalizePostgresStorageConfig(BASE_CONFIG); + + await using migrationAgent = factoryOptions.migrationAgent + ? factoryOptions.migrationAgent(BASE_CONFIG) + : new PostgresMigrationAgent(BASE_CONFIG); + migrationManager.registerMigrationAgent(migrationAgent); + + const mockServiceContext = { configuration: { storage: BASE_CONFIG } } as unknown as ServiceContext; + + if (!options?.doNotClear) { + await migrationManager.migrate({ + direction: framework.migrations.Direction.Down, + migrationContext: { + service_context: mockServiceContext + } + }); + } + + await migrationManager.migrate({ + direction: framework.migrations.Direction.Up, + migrationContext: { + service_context: mockServiceContext + } + }); + + return new PostgresBucketStorageFactory({ + config: TEST_CONNECTION_OPTIONS, + slot_name_prefix: 'test_' + }); + } catch (ex) { + // Vitest does not display these errors nicely when using the `await using` syntx + console.error(ex, ex.cause); + throw ex; + } + }; +}; diff --git a/modules/module-postgres-storage/src/storage/batch/OperationBatch.ts b/modules/module-postgres-storage/src/storage/batch/OperationBatch.ts new file mode 100644 index 000000000..2b91fab68 --- /dev/null +++ b/modules/module-postgres-storage/src/storage/batch/OperationBatch.ts @@ -0,0 +1,101 @@ +/** + * TODO share this implementation better in the core package. + * There are some subtle differences in this implementation. + */ + +import { storage, utils } from '@powersync/service-core'; +import { RequiredOperationBatchLimits } from '../../types/types.js'; + +/** + * Batch of input operations. + * + * We accumulate operations up to MAX_RECORD_BATCH_SIZE, + * then further split into sub-batches if MAX_CURRENT_DATA_BATCH_SIZE is exceeded. + */ +export class OperationBatch { + batch: RecordOperation[] = []; + currentSize: number = 0; + + readonly maxBatchCount: number; + readonly maxRecordSize: number; + readonly maxCurrentDataBatchSize: number; + + get length() { + return this.batch.length; + } + + constructor(protected options: RequiredOperationBatchLimits) { + this.maxBatchCount = options.max_record_count; + this.maxRecordSize = options.max_estimated_size; + this.maxCurrentDataBatchSize = options.max_current_data_batch_size; + } + + push(op: RecordOperation) { + this.batch.push(op); + this.currentSize += op.estimatedSize; + } + + shouldFlush() { + return this.batch.length >= this.maxBatchCount || this.currentSize > this.maxCurrentDataBatchSize; + } + + /** + * + * @param sizes Map of source key to estimated size of the current_data document, or undefined if current_data is not persisted. + * + */ + *batched(sizes: Map | undefined): Generator { + if (sizes == null) { + yield this.batch; + return; + } + let currentBatch: RecordOperation[] = []; + let currentBatchSize = 0; + for (let op of this.batch) { + const key = op.internalBeforeKey; + const size = sizes.get(key) ?? 0; + if (currentBatchSize + size > this.maxCurrentDataBatchSize && currentBatch.length > 0) { + yield currentBatch; + currentBatch = []; + currentBatchSize = 0; + } + currentBatchSize += size; + currentBatch.push(op); + } + if (currentBatch.length > 0) { + yield currentBatch; + } + } +} + +export class RecordOperation { + public readonly afterId: storage.ReplicaId | null; + public readonly beforeId: storage.ReplicaId; + public readonly internalBeforeKey: string; + public readonly internalAfterKey: string | null; + public readonly estimatedSize: number; + + constructor(public readonly record: storage.SaveOptions) { + const afterId = record.afterReplicaId ?? null; + const beforeId = record.beforeReplicaId ?? record.afterReplicaId; + this.afterId = afterId; + this.beforeId = beforeId; + this.internalBeforeKey = cacheKey(record.sourceTable.id, beforeId); + this.internalAfterKey = afterId ? cacheKey(record.sourceTable.id, afterId) : null; + this.estimatedSize = utils.estimateRowSize(record.before) + utils.estimateRowSize(record.after); + } +} + +/** + * In-memory cache key - must not be persisted. + */ +export function cacheKey(sourceTableId: string, id: storage.ReplicaId) { + return encodedCacheKey(sourceTableId, storage.serializeReplicaId(id)); +} + +/** + * Calculates a cache key for a stored ReplicaId. This is usually stored as a bytea/Buffer. + */ +export function encodedCacheKey(sourceTableId: string, storedKey: Buffer) { + return `${sourceTableId}.${storedKey.toString('base64')}`; +} diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts new file mode 100644 index 000000000..d7596b2bc --- /dev/null +++ b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts @@ -0,0 +1,885 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; +import { container, DisposableObserver, errors, logger } from '@powersync/lib-services-framework'; +import { storage, utils } from '@powersync/service-core'; +import * as sync_rules from '@powersync/service-sync-rules'; +import * as timers from 'timers/promises'; +import * as t from 'ts-codec'; +import { CurrentBucket, CurrentData, CurrentDataDecoded } from '../../types/models/CurrentData.js'; +import { models, RequiredOperationBatchLimits } from '../../types/types.js'; +import { NOTIFICATION_CHANNEL, sql } from '../../utils/db.js'; +import { pick } from '../../utils/ts-codec.js'; +import { batchCreateCustomWriteCheckpoints } from '../checkpoints/PostgresWriteCheckpointAPI.js'; +import { cacheKey, encodedCacheKey, OperationBatch, RecordOperation } from './OperationBatch.js'; +import { PostgresPersistedBatch } from './PostgresPersistedBatch.js'; + +export interface PostgresBucketBatchOptions { + db: lib_postgres.DatabaseClient; + sync_rules: sync_rules.SqlSyncRules; + group_id: number; + slot_name: string; + last_checkpoint_lsn: string | null; + no_checkpoint_before_lsn: string; + store_current_data: boolean; + keep_alive_op?: bigint | null; + /** + * Set to true for initial replication. + */ + skip_existing_rows: boolean; + batch_limits: RequiredOperationBatchLimits; +} + +/** + * Intermediate type which helps for only watching the active sync rules + * via the Postgres NOTIFY protocol. + */ +const StatefulCheckpoint = models.ActiveCheckpoint.and(t.object({ state: t.Enum(storage.SyncRuleState) })); +type StatefulCheckpointDecoded = t.Decoded; + +/** + * 15MB. Currently matches MongoDB. + * This could be increased in future. + */ +const MAX_ROW_SIZE = 15 * 1024 * 1024; + +export class PostgresBucketBatch + extends DisposableObserver + implements storage.BucketStorageBatch +{ + public last_flushed_op: bigint | null = null; + + protected db: lib_postgres.DatabaseClient; + protected group_id: number; + protected last_checkpoint_lsn: string | null; + protected no_checkpoint_before_lsn: string; + + protected persisted_op: bigint | null; + + protected write_checkpoint_batch: storage.CustomWriteCheckpointOptions[]; + protected readonly sync_rules: sync_rules.SqlSyncRules; + protected batch: OperationBatch | null; + private lastWaitingLogThrottled = 0; + + constructor(protected options: PostgresBucketBatchOptions) { + super(); + this.db = options.db; + this.group_id = options.group_id; + this.last_checkpoint_lsn = options.last_checkpoint_lsn; + this.no_checkpoint_before_lsn = options.no_checkpoint_before_lsn; + this.write_checkpoint_batch = []; + this.sync_rules = options.sync_rules; + this.batch = null; + this.persisted_op = null; + if (options.keep_alive_op) { + this.persisted_op = options.keep_alive_op; + } + } + + get lastCheckpointLsn() { + return this.last_checkpoint_lsn; + } + + async save(record: storage.SaveOptions): Promise { + // TODO maybe share with abstract class + const { after, afterReplicaId, before, beforeReplicaId, sourceTable, tag } = record; + for (const event of this.getTableEvents(sourceTable)) { + this.iterateListeners((cb) => + cb.replicationEvent?.({ + batch: this, + table: sourceTable, + data: { + op: tag, + after: after && utils.isCompleteRow(this.options.store_current_data, after) ? after : undefined, + before: before && utils.isCompleteRow(this.options.store_current_data, before) ? before : undefined + }, + event + }) + ); + } + /** + * Return if the table is just an event table + */ + if (!sourceTable.syncData && !sourceTable.syncParameters) { + return null; + } + + logger.debug(`Saving ${record.tag}:${record.before?.id}/${record.after?.id}`); + + this.batch ??= new OperationBatch(this.options.batch_limits); + this.batch.push(new RecordOperation(record)); + + if (this.batch.shouldFlush()) { + const r = await this.flush(); + // HACK: Give other streams a chance to also flush + await timers.setTimeout(5); + return r; + } + return null; + } + + async truncate(sourceTables: storage.SourceTable[]): Promise { + await this.flush(); + + let last_op: bigint | null = null; + for (let table of sourceTables) { + last_op = await this.truncateSingle(table); + } + + if (last_op) { + this.persisted_op = last_op; + } + + return { + flushed_op: String(last_op!) + }; + } + + protected async truncateSingle(sourceTable: storage.SourceTable) { + // To avoid too large transactions, we limit the amount of data we delete per transaction. + // Since we don't use the record data here, we don't have explicit size limits per batch. + const BATCH_LIMIT = 2000; + let lastBatchCount = BATCH_LIMIT; + let processedCount = 0; + const codec = pick(models.CurrentData, ['buckets', 'lookups', 'source_key']); + + while (lastBatchCount == BATCH_LIMIT) { + lastBatchCount = 0; + await this.withReplicationTransaction(async (db) => { + const persistedBatch = new PostgresPersistedBatch({ + group_id: this.group_id, + ...this.options.batch_limits + }); + + for await (const rows of db.streamRows>(sql` + SELECT + buckets, + lookups, + source_key + FROM + current_data + WHERE + group_id = ${{ type: 'int4', value: this.group_id }} + AND source_table = ${{ type: 'varchar', value: sourceTable.id }} + LIMIT + ${{ type: 'int4', value: BATCH_LIMIT }} + `)) { + lastBatchCount += rows.length; + processedCount += rows.length; + + const decodedRows = rows.map((row) => codec.decode(row)); + for (const value of decodedRows) { + persistedBatch.saveBucketData({ + before_buckets: value.buckets, + evaluated: [], + table: sourceTable, + source_key: value.source_key + }); + persistedBatch.saveParameterData({ + existing_lookups: value.lookups, + evaluated: [], + table: sourceTable, + source_key: value.source_key + }); + persistedBatch.deleteCurrentData({ + // This is serialized since we got it from a DB query + serialized_source_key: value.source_key, + source_table_id: sourceTable.id + }); + } + } + await persistedBatch.flush(db); + }); + } + if (processedCount == 0) { + // The op sequence should not have progressed + return null; + } + + const currentSequence = await this.db.sql` + SELECT + LAST_VALUE AS value + FROM + op_id_sequence; + `.first<{ value: bigint }>(); + return currentSequence!.value; + } + + async drop(sourceTables: storage.SourceTable[]): Promise { + await this.truncate(sourceTables); + const result = await this.flush(); + + await this.db.transaction(async (db) => { + for (const table of sourceTables) { + await db.sql` + DELETE FROM source_tables + WHERE + id = ${{ type: 'varchar', value: table.id }} + `.execute(); + } + }); + return result; + } + + async flush(): Promise { + let result: storage.FlushedResult | null = null; + // One flush may be split over multiple transactions. + // Each flushInner() is one transaction. + while (this.batch != null) { + let r = await this.flushInner(); + if (r) { + result = r; + } + } + await batchCreateCustomWriteCheckpoints(this.db, this.write_checkpoint_batch); + this.write_checkpoint_batch = []; + return result; + } + + private async flushInner(): Promise { + const batch = this.batch; + if (batch == null) { + return null; + } + + let resumeBatch: OperationBatch | null = null; + + const lastOp = await this.withReplicationTransaction(async (db) => { + resumeBatch = await this.replicateBatch(db, batch); + + const sequence = await db.sql` + SELECT + LAST_VALUE AS value + FROM + op_id_sequence; + `.first<{ value: bigint }>(); + return sequence!.value; + }); + + // null if done, set if we need another flush + this.batch = resumeBatch; + + if (lastOp == null) { + throw new Error('Unexpected last_op == null'); + } + + this.persisted_op = lastOp; + this.last_flushed_op = lastOp; + return { flushed_op: String(lastOp) }; + } + + async commit(lsn: string): Promise { + await this.flush(); + + if (this.last_checkpoint_lsn != null && lsn < this.last_checkpoint_lsn) { + // When re-applying transactions, don't create a new checkpoint until + // we are past the last transaction. + logger.info(`Re-applied transaction ${lsn} - skipping checkpoint`); + return false; + } + + if (lsn < this.no_checkpoint_before_lsn) { + if (Date.now() - this.lastWaitingLogThrottled > 5_000) { + logger.info( + `Waiting until ${this.no_checkpoint_before_lsn} before creating checkpoint, currently at ${lsn}. Persisted op: ${this.persisted_op}` + ); + this.lastWaitingLogThrottled = Date.now(); + } + + // Edge case: During initial replication, we have a no_checkpoint_before_lsn set, + // and don't actually commit the snapshot. + // The first commit can happen from an implicit keepalive message. + // That needs the persisted_op to get an accurate checkpoint, so + // we persist that in keepalive_op. + + await this.db.sql` + UPDATE sync_rules + SET + keepalive_op = ${{ type: 'int8', value: this.persisted_op }} + WHERE + id = ${{ type: 'int4', value: this.group_id }} + `.execute(); + + return false; + } + const now = new Date().toISOString(); + const update: Partial = { + last_checkpoint_lsn: lsn, + last_checkpoint_ts: now, + last_keepalive_ts: now, + snapshot_done: true, + last_fatal_error: null, + keepalive_op: null + }; + + if (this.persisted_op != null) { + update.last_checkpoint = this.persisted_op.toString(); + } + + const doc = await this.db.sql` + UPDATE sync_rules + SET + keepalive_op = ${{ type: 'int8', value: update.keepalive_op }}, + last_fatal_error = ${{ type: 'varchar', value: update.last_fatal_error }}, + snapshot_done = ${{ type: 'bool', value: update.snapshot_done }}, + last_keepalive_ts = ${{ type: 1184, value: update.last_keepalive_ts }}, + last_checkpoint = COALESCE( + ${{ type: 'int8', value: update.last_checkpoint }}, + last_checkpoint + ), + last_checkpoint_ts = ${{ type: 1184, value: update.last_checkpoint_ts }}, + last_checkpoint_lsn = ${{ type: 'varchar', value: update.last_checkpoint_lsn }} + WHERE + id = ${{ type: 'int4', value: this.group_id }} + RETURNING + id, + state, + last_checkpoint, + last_checkpoint_lsn + ` + .decoded(StatefulCheckpoint) + .first(); + + await notifySyncRulesUpdate(this.db, doc!); + + this.persisted_op = null; + this.last_checkpoint_lsn = lsn; + return true; + } + + async keepalive(lsn: string): Promise { + if (this.last_checkpoint_lsn != null && lsn <= this.last_checkpoint_lsn) { + // No-op + return false; + } + + if (lsn < this.no_checkpoint_before_lsn) { + return false; + } + + if (this.persisted_op != null) { + // The commit may have been skipped due to "no_checkpoint_before_lsn". + // Apply it now if relevant + logger.info(`Commit due to keepalive at ${lsn} / ${this.persisted_op}`); + return await this.commit(lsn); + } + + const updated = await this.db.sql` + UPDATE sync_rules + SET + snapshot_done = ${{ type: 'bool', value: true }}, + last_checkpoint_lsn = ${{ type: 'varchar', value: lsn }}, + last_fatal_error = ${{ type: 'varchar', value: null }}, + last_keepalive_ts = ${{ type: 1184, value: new Date().toISOString() }} + WHERE + id = ${{ type: 'int4', value: this.group_id }} + RETURNING + id, + state, + last_checkpoint, + last_checkpoint_lsn + ` + .decoded(StatefulCheckpoint) + .first(); + + await notifySyncRulesUpdate(this.db, updated!); + + this.last_checkpoint_lsn = lsn; + return true; + } + + async markSnapshotDone( + tables: storage.SourceTable[], + no_checkpoint_before_lsn: string + ): Promise { + const ids = tables.map((table) => table.id.toString()); + + await this.db.transaction(async (db) => { + await db.sql` + UPDATE source_tables + SET + snapshot_done = ${{ type: 'bool', value: true }} + WHERE + id IN ( + SELECT + (value ->> 0)::TEXT + FROM + jsonb_array_elements(${{ type: 'jsonb', value: ids }}) AS value + ); + `.execute(); + + if (no_checkpoint_before_lsn > this.no_checkpoint_before_lsn) { + this.no_checkpoint_before_lsn = no_checkpoint_before_lsn; + + await db.sql` + UPDATE sync_rules + SET + no_checkpoint_before = ${{ type: 'varchar', value: no_checkpoint_before_lsn }}, + last_keepalive_ts = ${{ type: 1184, value: new Date().toISOString() }} + WHERE + id = ${{ type: 'int4', value: this.group_id }} + `.execute(); + } + }); + return tables.map((table) => { + const copy = new storage.SourceTable( + table.id, + table.connectionTag, + table.objectId, + table.schema, + table.table, + table.replicaIdColumns, + table.snapshotComplete + ); + copy.syncData = table.syncData; + copy.syncParameters = table.syncParameters; + return copy; + }); + } + + addCustomWriteCheckpoint(checkpoint: storage.BatchedCustomWriteCheckpointOptions): void { + this.write_checkpoint_batch.push({ + ...checkpoint, + sync_rules_id: this.group_id + }); + } + + protected async replicateBatch(db: lib_postgres.WrappedConnection, batch: OperationBatch) { + let sizes: Map | undefined = undefined; + if (this.options.store_current_data && !this.options.skip_existing_rows) { + // We skip this step if we don't store current_data, since the sizes will + // always be small in that case. + + // With skipExistingRows, we don't load the full documents into memory, + // so we can also skip the size lookup step. + + // Find sizes of current_data documents, to assist in intelligent batching without + // exceeding memory limits. + const sizeLookups = batch.batch.map((r) => { + return { + source_table: r.record.sourceTable.id.toString(), + /** + * Encode to hex in order to pass a jsonb + */ + source_key: storage.serializeReplicaId(r.beforeId).toString('hex') + }; + }); + + sizes = new Map(); + + for await (const rows of db.streamRows<{ + source_table: string; + source_key: storage.ReplicaId; + data_size: number; + }>(lib_postgres.sql` + WITH + filter_data AS ( + SELECT + decode(FILTER ->> 'source_key', 'hex') AS source_key, -- Decoding from hex to bytea + (FILTER ->> 'source_table') AS source_table_id + FROM + jsonb_array_elements(${{ type: 'jsonb', value: sizeLookups }}::jsonb) AS FILTER + ) + SELECT + pg_column_size(c.data) AS data_size, + c.source_table, + c.source_key + FROM + current_data c + JOIN filter_data f ON c.source_table = f.source_table_id + AND c.source_key = f.source_key + WHERE + c.group_id = ${{ type: 'int4', value: this.group_id }} + `)) { + for (const row of rows) { + const key = cacheKey(row.source_table, row.source_key); + sizes.set(key, row.data_size); + } + } + } + + // If set, we need to start a new transaction with this batch. + let resumeBatch: OperationBatch | null = null; + + // Now batch according to the sizes + // This is a single batch if storeCurrentData == false + for await (const b of batch.batched(sizes)) { + if (resumeBatch) { + // These operations need to be completed in a new transaction. + for (let op of b) { + resumeBatch.push(op); + } + continue; + } + + const lookups = b.map((r) => { + return { + source_table: r.record.sourceTable.id, + source_key: storage.serializeReplicaId(r.beforeId).toString('hex') + }; + }); + + const current_data_lookup = new Map(); + for await (const currentDataRows of db.streamRows({ + statement: /* sql */ ` + WITH + filter_data AS ( + SELECT + decode(FILTER ->> 'source_key', 'hex') AS source_key, -- Decoding from hex to bytea + (FILTER ->> 'source_table') AS source_table_id + FROM + jsonb_array_elements($1::jsonb) AS FILTER + ) + SELECT + --- With skipExistingRows, we only need to know whether or not the row exists. + ${this.options.skip_existing_rows ? `c.source_table, c.source_key` : 'c.*'} + FROM + current_data c + JOIN filter_data f ON c.source_table = f.source_table_id + AND c.source_key = f.source_key + WHERE + c.group_id = $2 + `, + params: [ + { + type: 'jsonb', + value: lookups + }, + { + type: 'int8', + value: this.group_id + } + ] + })) { + for (const row of currentDataRows) { + const decoded = this.options.skip_existing_rows + ? pick(CurrentData, ['source_key', 'source_table']).decode(row) + : CurrentData.decode(row); + current_data_lookup.set( + encodedCacheKey(decoded.source_table, decoded.source_key), + decoded as CurrentDataDecoded + ); + } + } + + let persistedBatch: PostgresPersistedBatch | null = new PostgresPersistedBatch({ + group_id: this.group_id, + ...this.options.batch_limits + }); + + for (const op of b) { + // These operations need to be completed in a new transaction + if (resumeBatch) { + resumeBatch.push(op); + continue; + } + + const currentData = current_data_lookup.get(op.internalBeforeKey) ?? null; + if (currentData != null) { + // If it will be used again later, it will be set again using nextData below + current_data_lookup.delete(op.internalBeforeKey); + } + const nextData = await this.saveOperation(persistedBatch!, op, currentData); + if (nextData != null) { + // Update our current_data and size cache + current_data_lookup.set(op.internalAfterKey!, nextData); + sizes?.set(op.internalAfterKey!, nextData.data.byteLength); + } + + if (persistedBatch!.shouldFlushTransaction()) { + await persistedBatch!.flush(db); + // The operations stored in this batch will be processed in the `resumeBatch` + persistedBatch = null; + // Return the remaining entries for the next resume transaction + resumeBatch = new OperationBatch(this.options.batch_limits); + } + } + + if (persistedBatch) { + /** + * The operations were less than the max size if here. Flush now. + * `persistedBatch` will be `null` if the operations should be flushed in a new transaction. + */ + await persistedBatch.flush(db); + } + } + return resumeBatch; + } + + protected async saveOperation( + persistedBatch: PostgresPersistedBatch, + operation: RecordOperation, + currentData?: CurrentDataDecoded | null + ) { + const record = operation.record; + // We store bytea colums for source keys + const beforeId = operation.beforeId; + const afterId = operation.afterId; + let after = record.after; + const sourceTable = record.sourceTable; + + let existingBuckets: CurrentBucket[] = []; + let newBuckets: CurrentBucket[] = []; + let existingLookups: Buffer[] = []; + let newLookups: Buffer[] = []; + + if (this.options.skip_existing_rows) { + if (record.tag == storage.SaveOperationTag.INSERT) { + if (currentData != null) { + // Initial replication, and we already have the record. + // This may be a different version of the record, but streaming replication + // will take care of that. + // Skip the insert here. + return null; + } + } else { + throw new Error(`${record.tag} not supported with skipExistingRows: true`); + } + } + + if (record.tag == storage.SaveOperationTag.UPDATE) { + const result = currentData; + if (result == null) { + // Not an error if we re-apply a transaction + existingBuckets = []; + existingLookups = []; + // Log to help with debugging if there was a consistency issue + if (this.options.store_current_data) { + logger.warn( + `Cannot find previous record for update on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}` + ); + } + } else { + existingBuckets = result.buckets; + existingLookups = result.lookups; + if (this.options.store_current_data) { + const data = storage.deserializeBson(result.data) as sync_rules.SqliteRow; + after = storage.mergeToast(after!, data); + } + } + } else if (record.tag == storage.SaveOperationTag.DELETE) { + const result = currentData; + if (result == null) { + // Not an error if we re-apply a transaction + existingBuckets = []; + existingLookups = []; + // Log to help with debugging if there was a consistency issue + if (this.options.store_current_data) { + logger.warn( + `Cannot find previous record for delete on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}` + ); + } + } else { + existingBuckets = result.buckets; + existingLookups = result.lookups; + } + } + + let afterData: Buffer | undefined; + if (afterId != null && !this.options.store_current_data) { + afterData = storage.serializeBson({}); + } else if (afterId != null) { + try { + afterData = storage.serializeBson(after); + if (afterData!.byteLength > MAX_ROW_SIZE) { + throw new Error(`Row too large: ${afterData?.byteLength}`); + } + } catch (e) { + // Replace with empty values, equivalent to TOAST values + after = Object.fromEntries( + Object.entries(after!).map(([key, value]) => { + return [key, undefined]; + }) + ); + afterData = storage.serializeBson(after); + + container.reporter.captureMessage( + `Data too big on ${record.sourceTable.qualifiedName}.${record.after?.id}: ${e.message}`, + { + level: errors.ErrorSeverity.WARNING, + metadata: { + replication_slot: this.options.slot_name, + table: record.sourceTable.qualifiedName + } + } + ); + } + } + + // 2. Save bucket data + if (beforeId != null && (afterId == null || !storage.replicaIdEquals(beforeId, afterId))) { + // Source ID updated + if (sourceTable.syncData) { + // Delete old record + persistedBatch.saveBucketData({ + source_key: beforeId, + table: sourceTable, + before_buckets: existingBuckets, + evaluated: [] + }); + // Clear this, so we don't also try to REMOVE for the new id + existingBuckets = []; + } + + if (sourceTable.syncParameters) { + // Delete old parameters + persistedBatch.saveParameterData({ + source_key: beforeId, + table: sourceTable, + evaluated: [], + existing_lookups: existingLookups + }); + existingLookups = []; + } + } + + // If we re-apply a transaction, we can end up with a partial row. + // + // We may end up with toasted values, which means the record is not quite valid. + // However, it will be valid by the end of the transaction. + // + // In this case, we don't save the op, but we do save the current data. + if (afterId && after && utils.isCompleteRow(this.options.store_current_data, after)) { + // Insert or update + if (sourceTable.syncData) { + const { results: evaluated, errors: syncErrors } = this.sync_rules.evaluateRowWithErrors({ + record: after, + sourceTable + }); + + for (const error of syncErrors) { + container.reporter.captureMessage( + `Failed to evaluate data query on ${record.sourceTable.qualifiedName}.${record.after?.id}: ${error.error}`, + { + level: errors.ErrorSeverity.WARNING, + metadata: { + replication_slot: this.options.slot_name, + table: record.sourceTable.qualifiedName + } + } + ); + logger.error( + `Failed to evaluate data query on ${record.sourceTable.qualifiedName}.${record.after?.id}: ${error.error}` + ); + } + + // Save new one + persistedBatch.saveBucketData({ + source_key: afterId, + evaluated, + table: sourceTable, + before_buckets: existingBuckets + }); + + newBuckets = evaluated.map((e) => { + return { + bucket: e.bucket, + table: e.table, + id: e.id + }; + }); + } + + if (sourceTable.syncParameters) { + // Parameters + const { results: paramEvaluated, errors: paramErrors } = this.sync_rules.evaluateParameterRowWithErrors( + sourceTable, + after + ); + + for (let error of paramErrors) { + container.reporter.captureMessage( + `Failed to evaluate parameter query on ${record.sourceTable.qualifiedName}.${record.after?.id}: ${error.error}`, + { + level: errors.ErrorSeverity.WARNING, + metadata: { + replication_slot: this.options.slot_name, + table: record.sourceTable.qualifiedName + } + } + ); + logger.error( + `Failed to evaluate parameter query on ${record.sourceTable.qualifiedName}.${after.id}: ${error.error}` + ); + } + + persistedBatch.saveParameterData({ + source_key: afterId, + table: sourceTable, + evaluated: paramEvaluated, + existing_lookups: existingLookups + }); + + newLookups = paramEvaluated.map((p) => { + return storage.serializeLookupBuffer(p.lookup); + }); + } + } + + let result: CurrentDataDecoded | null = null; + + // 5. TOAST: Update current data and bucket list. + if (afterId) { + // Insert or update + result = { + source_key: afterId, + group_id: this.group_id, + data: afterData!, + source_table: sourceTable.id, + buckets: newBuckets, + lookups: newLookups + }; + persistedBatch.upsertCurrentData(result); + } + + if (afterId == null || !storage.replicaIdEquals(beforeId, afterId)) { + // Either a delete (afterId == null), or replaced the old replication id + persistedBatch.deleteCurrentData({ + source_table_id: record.sourceTable.id, + source_key: beforeId! + }); + } + + return result; + } + + /** + * Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable} + * TODO maybe share this with an abstract class + */ + protected getTableEvents(table: storage.SourceTable): sync_rules.SqlEventDescriptor[] { + return this.sync_rules.event_descriptors.filter((evt) => + [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table)) + ); + } + + protected async withReplicationTransaction( + callback: (tx: lib_postgres.WrappedConnection) => Promise + ): Promise { + try { + return await this.db.transaction(async (db) => { + return await callback(db); + }); + } finally { + await this.db.sql` + UPDATE sync_rules + SET + last_keepalive_ts = ${{ type: 1184, value: new Date().toISOString() }} + WHERE + id = ${{ type: 'int4', value: this.group_id }} + `.execute(); + } + } +} + +/** + * Uses Postgres' NOTIFY functionality to update different processes when the + * active checkpoint has been updated. + */ +export const notifySyncRulesUpdate = async (db: lib_postgres.DatabaseClient, update: StatefulCheckpointDecoded) => { + if (update.state != storage.SyncRuleState.ACTIVE) { + return; + } + + await db.query({ + statement: `NOTIFY ${NOTIFICATION_CHANNEL}, '${models.ActiveCheckpointNotification.encode({ active_checkpoint: update })}'` + }); +}; diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts new file mode 100644 index 000000000..8dcb7eb0d --- /dev/null +++ b/modules/module-postgres-storage/src/storage/batch/PostgresPersistedBatch.ts @@ -0,0 +1,441 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; +import { logger } from '@powersync/lib-services-framework'; +import { storage, utils } from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; +import * as sync_rules from '@powersync/service-sync-rules'; +import { models, RequiredOperationBatchLimits } from '../../types/types.js'; +import { replicaIdToSubkey } from '../../utils/bson.js'; + +export type SaveBucketDataOptions = { + /** + * This value will be serialized into a BSON Byte array for storage + */ + source_key: storage.ReplicaId; + table: storage.SourceTable; + before_buckets: models.CurrentBucket[]; + evaluated: sync_rules.EvaluatedRow[]; +}; + +export type SaveParameterDataOptions = { + source_key: storage.ReplicaId; + table: storage.SourceTable; + evaluated: sync_rules.EvaluatedParameters[]; + existing_lookups: Buffer[]; +}; + +export type DeleteCurrentDataOptions = { + source_table_id: bigint; + /** + * ReplicaID which needs to be serialized in order to be queried + * or inserted into the DB + */ + source_key?: storage.ReplicaId; + /** + * Optionally provide the serialized source key directly + */ + serialized_source_key?: Buffer; +}; + +export type PostgresPersistedBatchOptions = RequiredOperationBatchLimits & { + group_id: number; +}; + +export class PostgresPersistedBatch { + group_id: number; + + /** + * Very rough estimate of current operations size in bytes + */ + currentSize: number; + + readonly maxTransactionBatchSize: number; + readonly maxTransactionDocCount: number; + + /** + * Ordered set of bucket_data insert operation parameters + */ + protected bucketDataInserts: models.BucketData[]; + protected parameterDataInserts: models.BucketParameters[]; + protected currentDataDeletes: Pick[]; + /** + * This is stored as a map to avoid multiple inserts (or conflicts) for the same key + */ + protected currentDataInserts: Map; + + constructor(options: PostgresPersistedBatchOptions) { + this.group_id = options.group_id; + + this.maxTransactionBatchSize = options.max_estimated_size; + this.maxTransactionDocCount = options.max_record_count; + + this.bucketDataInserts = []; + this.parameterDataInserts = []; + this.currentDataDeletes = []; + this.currentDataInserts = new Map(); + this.currentSize = 0; + } + + saveBucketData(options: SaveBucketDataOptions) { + const remaining_buckets = new Map(); + for (const b of options.before_buckets) { + const key = currentBucketKey(b); + remaining_buckets.set(key, b); + } + + const dchecksum = utils.hashDelete(replicaIdToSubkey(options.table.id, options.source_key)); + + const serializedSourceKey = storage.serializeReplicaId(options.source_key); + const hexSourceKey = serializedSourceKey.toString('hex'); + + for (const k of options.evaluated) { + const key = currentBucketKey(k); + remaining_buckets.delete(key); + + const data = JSONBig.stringify(k.data); + const checksum = utils.hashData(k.table, k.id, data); + + this.bucketDataInserts.push({ + group_id: this.group_id, + bucket_name: k.bucket, + op: models.OpType.PUT, + source_table: options.table.id, + source_key: hexSourceKey, + table_name: k.table, + row_id: k.id, + checksum, + data, + op_id: 0, // Will use nextval of sequence + target_op: null + }); + + this.currentSize += k.bucket.length + data.length + hexSourceKey.length + 100; + } + + for (const bd of remaining_buckets.values()) { + // REMOVE operation + this.bucketDataInserts.push({ + group_id: this.group_id, + bucket_name: bd.bucket, + op: models.OpType.REMOVE, + source_table: options.table.id, + source_key: hexSourceKey, + table_name: bd.table, + row_id: bd.id, + checksum: dchecksum, + op_id: 0, // Will use nextval of sequence + target_op: null, + data: null + }); + this.currentSize += bd.bucket.length + hexSourceKey.length + 100; + } + } + + saveParameterData(options: SaveParameterDataOptions) { + // This is similar to saving bucket data. + // A key difference is that we don't need to keep the history intact. + // We do need to keep track of recent history though - enough that we can get consistent data for any specific checkpoint. + // Instead of storing per bucket id, we store per "lookup". + // A key difference is that we don't need to store or keep track of anything per-bucket - the entire record is + // either persisted or removed. + // We also don't need to keep history intact. + const { source_key, table, evaluated, existing_lookups } = options; + const serializedSourceKey = storage.serializeReplicaId(source_key); + const hexSourceKey = serializedSourceKey.toString('hex'); + const remaining_lookups = new Map(); + for (const l of existing_lookups) { + remaining_lookups.set(l.toString('base64'), l); + } + + // 1. Insert new entries + for (const result of evaluated) { + const binLookup = storage.serializeLookupBuffer(result.lookup); + const base64 = binLookup.toString('base64'); + remaining_lookups.delete(base64); + const hexLookup = binLookup.toString('hex'); + const serializedBucketParameters = JSONBig.stringify(result.bucket_parameters); + this.parameterDataInserts.push({ + group_id: this.group_id, + source_table: table.id, + source_key: hexSourceKey, + bucket_parameters: serializedBucketParameters, + id: 0, // auto incrementing id + lookup: hexLookup + }); + this.currentSize += hexLookup.length + serializedBucketParameters.length + hexSourceKey.length + 100; + } + + // 2. "REMOVE" entries for any lookup not touched. + for (const lookup of remaining_lookups.values()) { + const hexLookup = lookup.toString('hex'); + this.parameterDataInserts.push({ + group_id: this.group_id, + source_table: table.id, + source_key: hexSourceKey, + bucket_parameters: JSON.stringify([]), + id: 0, // auto incrementing id + lookup: hexLookup + }); + this.currentSize += hexLookup.length + hexSourceKey.length + 100; + } + } + + deleteCurrentData(options: DeleteCurrentDataOptions) { + const serializedReplicaId = options.serialized_source_key ?? storage.serializeReplicaId(options.source_key); + this.currentDataDeletes.push({ + group_id: this.group_id, + source_table: options.source_table_id.toString(), + source_key: serializedReplicaId.toString('hex') + }); + this.currentSize += serializedReplicaId.byteLength + 100; + } + + upsertCurrentData(options: models.CurrentDataDecoded) { + const { source_table, source_key, buckets } = options; + + const serializedReplicaId = storage.serializeReplicaId(source_key); + const hexReplicaId = serializedReplicaId.toString('hex'); + const serializedBuckets = JSONBig.stringify(options.buckets); + + /** + * Only track the last unique ID for this current_data record. + * Applying multiple items in the flush method could cause an + * " + * ON CONFLICT DO UPDATE command cannot affect row a second time + * " + * error. + */ + const key = `${this.group_id}-${source_table}-${hexReplicaId}`; + + this.currentDataInserts.set(key, { + group_id: this.group_id, + source_table: source_table, + source_key: hexReplicaId, + buckets: serializedBuckets, + data: options.data.toString('hex'), + lookups: options.lookups.map((l) => l.toString('hex')) + }); + + this.currentSize += + (options.data?.byteLength ?? 0) + + serializedReplicaId.byteLength + + buckets.length + + options.lookups.reduce((total, l) => { + return total + l.byteLength; + }, 0) + + 100; + } + + shouldFlushTransaction() { + return ( + this.currentSize >= this.maxTransactionBatchSize || + this.bucketDataInserts.length >= this.maxTransactionDocCount || + this.currentDataInserts.size >= this.maxTransactionDocCount || + this.currentDataDeletes.length >= this.maxTransactionDocCount || + this.parameterDataInserts.length >= this.maxTransactionDocCount + ); + } + + async flush(db: lib_postgres.WrappedConnection) { + logger.info( + `powersync_${this.group_id} Flushed ${this.bucketDataInserts.length} + ${this.parameterDataInserts.length} + ${ + this.currentDataInserts.size + this.currentDataDeletes.length + } updates, ${Math.round(this.currentSize / 1024)}kb.` + ); + + await this.flushBucketData(db); + await this.flushParameterData(db); + await this.flushCurrentData(db); + + this.bucketDataInserts = []; + this.parameterDataInserts = []; + this.currentDataDeletes = []; + this.currentDataInserts = new Map(); + this.currentSize = 0; + } + + protected async flushBucketData(db: lib_postgres.WrappedConnection) { + if (this.bucketDataInserts.length > 0) { + await db.sql` + WITH + parsed_data AS ( + SELECT + group_id, + bucket_name, + source_table, + decode(source_key, 'hex') AS source_key, -- Decode hex to bytea + table_name, + op, + row_id, + checksum, + data, + target_op + FROM + jsonb_to_recordset(${{ type: 'jsonb', value: this.bucketDataInserts }}::jsonb) AS t ( + group_id integer, + bucket_name text, + source_table text, + source_key text, -- Input as hex string + table_name text, + op text, + row_id text, + checksum bigint, + data text, + target_op bigint + ) + ) + INSERT INTO + bucket_data ( + group_id, + bucket_name, + op_id, + op, + source_table, + source_key, + table_name, + row_id, + checksum, + data, + target_op + ) + SELECT + group_id, + bucket_name, + nextval('op_id_sequence'), + op, + source_table, + source_key, -- Already decoded + table_name, + row_id, + checksum, + data, + target_op + FROM + parsed_data; + `.execute(); + } + } + + protected async flushParameterData(db: lib_postgres.WrappedConnection) { + if (this.parameterDataInserts.length > 0) { + await db.sql` + WITH + parsed_data AS ( + SELECT + group_id, + source_table, + decode(source_key, 'hex') AS source_key, -- Decode hex to bytea + decode(lookup, 'hex') AS lookup, -- Decode hex to bytea + bucket_parameters + FROM + jsonb_to_recordset(${{ type: 'jsonb', value: this.parameterDataInserts }}::jsonb) AS t ( + group_id integer, + source_table text, + source_key text, -- Input as hex string + lookup text, -- Input as hex string + bucket_parameters text -- Input as stringified JSON + ) + ) + INSERT INTO + bucket_parameters ( + group_id, + source_table, + source_key, + lookup, + bucket_parameters + ) + SELECT + group_id, + source_table, + source_key, -- Already decoded + lookup, -- Already decoded + bucket_parameters + FROM + parsed_data; + `.execute(); + } + } + + protected async flushCurrentData(db: lib_postgres.WrappedConnection) { + if (this.currentDataInserts.size > 0) { + await db.sql` + WITH + parsed_data AS ( + SELECT + group_id, + source_table, + decode(source_key, 'hex') AS source_key, -- Decode hex to bytea + buckets::jsonb AS buckets, + decode(data, 'hex') AS data, -- Decode hex to bytea + ARRAY( + SELECT + decode((value ->> 0)::TEXT, 'hex') + FROM + jsonb_array_elements(lookups::jsonb) AS value + ) AS lookups -- Decode array of hex strings to bytea[] + FROM + jsonb_to_recordset(${{ + type: 'jsonb', + value: Array.from(this.currentDataInserts.values()) + }}::jsonb) AS t ( + group_id integer, + source_table text, + source_key text, -- Input as hex string + buckets text, + data text, -- Input as hex string + lookups text -- Input as stringified JSONB array of hex strings + ) + ) + INSERT INTO + current_data ( + group_id, + source_table, + source_key, + buckets, + data, + lookups + ) + SELECT + group_id, + source_table, + source_key, -- Already decoded + buckets, + data, -- Already decoded + lookups -- Already decoded + FROM + parsed_data + ON CONFLICT (group_id, source_table, source_key) DO UPDATE + SET + buckets = EXCLUDED.buckets, + data = EXCLUDED.data, + lookups = EXCLUDED.lookups; + `.execute(); + } + + if (this.currentDataDeletes.length > 0) { + await db.sql` + WITH + conditions AS ( + SELECT + group_id, + source_table, + decode(source_key, 'hex') AS source_key -- Decode hex to bytea + FROM + jsonb_to_recordset(${{ type: 'jsonb', value: this.currentDataDeletes }}::jsonb) AS t ( + group_id integer, + source_table text, + source_key text -- Input as hex string + ) + ) + DELETE FROM current_data USING conditions + WHERE + current_data.group_id = conditions.group_id + AND current_data.source_table = conditions.source_table + AND current_data.source_key = conditions.source_key; + `.execute(); + } + } +} + +export function currentBucketKey(b: models.CurrentBucket) { + return `${b.bucket}/${b.table}/${b.id}`; +} diff --git a/modules/module-postgres-storage/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts b/modules/module-postgres-storage/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts new file mode 100644 index 000000000..a12c405bf --- /dev/null +++ b/modules/module-postgres-storage/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts @@ -0,0 +1,176 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; +import * as framework from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; +import { models } from '../../types/types.js'; + +export type PostgresCheckpointAPIOptions = { + db: lib_postgres.DatabaseClient; + mode: storage.WriteCheckpointMode; +}; + +export class PostgresWriteCheckpointAPI implements storage.WriteCheckpointAPI { + readonly db: lib_postgres.DatabaseClient; + private _mode: storage.WriteCheckpointMode; + + constructor(options: PostgresCheckpointAPIOptions) { + this.db = options.db; + this._mode = options.mode; + } + + get writeCheckpointMode() { + return this._mode; + } + + setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void { + this._mode = mode; + } + + async batchCreateCustomWriteCheckpoints(checkpoints: storage.CustomWriteCheckpointOptions[]): Promise { + return batchCreateCustomWriteCheckpoints(this.db, checkpoints); + } + + async createCustomWriteCheckpoint(options: storage.CustomWriteCheckpointOptions): Promise { + if (this.writeCheckpointMode !== storage.WriteCheckpointMode.CUSTOM) { + throw new framework.errors.ValidationError( + `Creating a custom Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"` + ); + } + + const { checkpoint, user_id, sync_rules_id } = options; + const row = await this.db.sql` + INSERT INTO + custom_write_checkpoints (user_id, write_checkpoint, sync_rules_id) + VALUES + ( + ${{ type: 'varchar', value: user_id }}, + ${{ type: 'int8', value: checkpoint }}, + ${{ type: 'int4', value: sync_rules_id }} + ) + ON CONFLICT DO UPDATE + SET + write_checkpoint = EXCLUDED.write_checkpoint + RETURNING + *; + ` + .decoded(models.CustomWriteCheckpoint) + .first(); + return row!.write_checkpoint; + } + + async createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise { + if (this.writeCheckpointMode !== storage.WriteCheckpointMode.MANAGED) { + throw new framework.errors.ValidationError( + `Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"` + ); + } + + const row = await this.db.sql` + INSERT INTO + write_checkpoints (user_id, lsns, write_checkpoint) + VALUES + ( + ${{ type: 'varchar', value: checkpoint.user_id }}, + ${{ type: 'jsonb', value: checkpoint.heads }}, + ${{ type: 'int8', value: 1 }} + ) + ON CONFLICT (user_id) DO UPDATE + SET + write_checkpoint = write_checkpoints.write_checkpoint + 1, + lsns = EXCLUDED.lsns + RETURNING + *; + ` + .decoded(models.WriteCheckpoint) + .first(); + return row!.write_checkpoint; + } + + async lastWriteCheckpoint(filters: storage.LastWriteCheckpointFilters): Promise { + switch (this.writeCheckpointMode) { + case storage.WriteCheckpointMode.CUSTOM: + if (false == 'sync_rules_id' in filters) { + throw new framework.errors.ValidationError(`Sync rules ID is required for custom Write Checkpoint filtering`); + } + return this.lastCustomWriteCheckpoint(filters as storage.CustomWriteCheckpointFilters); + case storage.WriteCheckpointMode.MANAGED: + if (false == 'heads' in filters) { + throw new framework.errors.ValidationError( + `Replication HEAD is required for managed Write Checkpoint filtering` + ); + } + return this.lastManagedWriteCheckpoint(filters as storage.ManagedWriteCheckpointFilters); + } + } + + protected async lastCustomWriteCheckpoint(filters: storage.CustomWriteCheckpointFilters) { + const { user_id, sync_rules_id } = filters; + const row = await this.db.sql` + SELECT + * + FROM + custom_write_checkpoints + WHERE + user_id = ${{ type: 'varchar', value: user_id }} + AND sync_rules_id = ${{ type: 'int4', value: sync_rules_id }} + ` + .decoded(models.CustomWriteCheckpoint) + .first(); + return row?.write_checkpoint ?? null; + } + + protected async lastManagedWriteCheckpoint(filters: storage.ManagedWriteCheckpointFilters) { + const { user_id, heads } = filters; + // TODO: support multiple heads when we need to support multiple connections + const lsn = heads['1']; + if (lsn == null) { + // Can happen if we haven't replicated anything yet. + return null; + } + const row = await this.db.sql` + SELECT + * + FROM + write_checkpoints + WHERE + user_id = ${{ type: 'varchar', value: user_id }} + AND lsns ->> '1' <= ${{ type: 'varchar', value: lsn }}; + ` + .decoded(models.WriteCheckpoint) + .first(); + return row?.write_checkpoint ?? null; + } +} + +export async function batchCreateCustomWriteCheckpoints( + db: lib_postgres.DatabaseClient, + checkpoints: storage.CustomWriteCheckpointOptions[] +): Promise { + if (!checkpoints.length) { + return; + } + + await db.sql` + WITH + json_data AS ( + SELECT + jsonb_array_elements(${{ type: 'jsonb', value: JSONBig.stringify(checkpoints) }}) AS + CHECKPOINT + ) + INSERT INTO + custom_write_checkpoints (user_id, write_checkpoint, sync_rules_id) + SELECT + CHECKPOINT ->> 'user_id'::varchar, + ( + CHECKPOINT ->> 'checkpoint' + )::int8, + ( + CHECKPOINT ->> 'sync_rules_id' + )::int4 + FROM + json_data + ON CONFLICT (user_id, sync_rules_id) DO UPDATE + SET + write_checkpoint = EXCLUDED.write_checkpoint; + `.execute(); +} diff --git a/modules/module-postgres-storage/src/storage/storage-index.ts b/modules/module-postgres-storage/src/storage/storage-index.ts new file mode 100644 index 000000000..b97b6a966 --- /dev/null +++ b/modules/module-postgres-storage/src/storage/storage-index.ts @@ -0,0 +1,5 @@ +export * from './PostgresBucketStorageFactory.js'; +export * from './PostgresCompactor.js'; +export * from './PostgresStorageProvider.js'; +export * from './PostgresSyncRulesStorage.js'; +export * from './PostgresTestStorageFactoryGenerator.js'; diff --git a/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts b/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts new file mode 100644 index 000000000..c3cf5da6e --- /dev/null +++ b/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts @@ -0,0 +1,67 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; +import { logger } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import { SqlSyncRules } from '@powersync/service-sync-rules'; + +import { models } from '../../types/types.js'; + +export class PostgresPersistedSyncRulesContent implements storage.PersistedSyncRulesContent { + public readonly slot_name: string; + + public readonly id: number; + public readonly sync_rules_content: string; + public readonly last_checkpoint_lsn: string | null; + public readonly last_fatal_error: string | null; + public readonly last_keepalive_ts: Date | null; + public readonly last_checkpoint_ts: Date | null; + current_lock: storage.ReplicationLock | null = null; + + constructor( + private db: lib_postgres.DatabaseClient, + row: models.SyncRulesDecoded + ) { + this.id = Number(row.id); + this.sync_rules_content = row.content; + this.last_checkpoint_lsn = row.last_checkpoint_lsn; + this.slot_name = row.slot_name; + this.last_fatal_error = row.last_fatal_error; + this.last_checkpoint_ts = row.last_checkpoint_ts ? new Date(row.last_checkpoint_ts) : null; + this.last_keepalive_ts = row.last_keepalive_ts ? new Date(row.last_keepalive_ts) : null; + } + + parsed(options: storage.ParseSyncRulesOptions): storage.PersistedSyncRules { + return { + id: this.id, + slot_name: this.slot_name, + sync_rules: SqlSyncRules.fromYaml(this.sync_rules_content, options) + }; + } + + async lock(): Promise { + const manager = new lib_postgres.PostgresLockManager({ + db: this.db, + name: `sync_rules_${this.id}_${this.slot_name}` + }); + const lockHandle = await manager.acquire(); + if (!lockHandle) { + throw new Error(`Sync rules: ${this.id} have been locked by another process for replication.`); + } + + const interval = setInterval(async () => { + try { + await lockHandle.refresh(); + } catch (e) { + logger.error('Failed to refresh lock', e); + clearInterval(interval); + } + }, 30_130); + + return (this.current_lock = { + sync_rules_id: this.id, + release: async () => { + clearInterval(interval); + return lockHandle.release(); + } + }); + } +} diff --git a/modules/module-postgres-storage/src/types/codecs.ts b/modules/module-postgres-storage/src/types/codecs.ts new file mode 100644 index 000000000..9f0824ee1 --- /dev/null +++ b/modules/module-postgres-storage/src/types/codecs.ts @@ -0,0 +1,136 @@ +import * as t from 'ts-codec'; + +export const BIGINT_MAX = BigInt('9223372036854775807'); + +/** + * The use of ts-codec: + * We currently use pgwire for Postgres queries. This library provides fine-grained control + * over parameter typings and efficient streaming of query responses. Additionally, configuring + * pgwire with default certificates allows us to use the same connection configuration process + * for both replication and storage libraries. + * + * Unfortunately, ORM driver support for pgwire is limited, so we rely on pure SQL queries in the + * absence of writing an ORM driver from scratch. + * + * [Opinion]: Writing pure SQL queries throughout a codebase can be daunting from a maintenance + * and debugging perspective. For example, row response types are often declared when performing a query: + * + * ```typescript + * const rows = await db.queryRows(`SELECT one, two FROM my_table`); + * ``` + * This type declaration suggests `rows` is an array of `MyRowType` objects, even though no validation + * is enforced. Adding a field to the `MyRowType` interface without updating the query could easily + * introduce subtle bugs. Similarly, type mismatches between SQL results and TypeScript interfaces, such as + * a `Date` field returned as a `string`, require manual conversion. + * + * `ts-codec` is not an ORM, but it simplifies working with pure SQL query responses in several ways: + * + * - **Validations**: The `decode` operation ensures that the returned row matches the expected object + * structure, throwing an error if it doesn't. + * - **Decoding Columns**: pgwire already decodes common SQLite types, but `ts-codec` adds an extra layer + * for JS-native values. For instance, `jsonb` columns are returned as `JsonContainer`/`string` and can + * be automatically parsed into objects. Similarly, fields like `group_id` are converted from `Bigint` + * to `Number` for easier use. + * - **Encoded Forms**: A single `ts-codec` type definition can infer both encoded and decoded forms. This + * is especially useful for persisted batch operations that rely on JSON query parameters for bulk inserts. + * Collections like `bucket_data`, `current_data`, and `bucket_parameters` use encoded/decoded types, making + * changes easier to manage and validate. While some manual encoding is done for intermediate values (e.g., + * size estimation), these types are validated with `ts-codec` to ensure consistency. + */ + +/** + * Wraps a codec which is encoded to a JSON string + */ +export const jsonb = (subCodec: t.Codec) => + t.codec( + 'jsonb', + (decoded: Decoded) => { + return JSON.stringify(subCodec.encode(decoded) as any); + }, + (encoded: string | { data: string }) => { + const s = typeof encoded == 'object' ? encoded.data : encoded; + return subCodec.decode(JSON.parse(s)); + } + ); + +/** + * Just performs a pure JSON.parse for the decoding step + */ +export const jsonb_raw = () => + t.codec( + 'jsonb_raw', + (decoded: Decoded) => { + return JSON.stringify(decoded); + }, + (encoded: string | { data: string }) => { + const s = typeof encoded == 'object' ? encoded.data : encoded; + return JSON.parse(s); + } + ); + +export const bigint = t.codec( + 'bigint', + (decoded: BigInt) => { + return decoded.toString(); + }, + (encoded: string | number) => { + return BigInt(encoded); + } +); + +export const uint8array = t.codec( + 'uint8array', + (d) => d, + (e) => e +); + +/** + * PGWire returns BYTEA values as Uint8Array instances. + * We also serialize to a hex string for bulk inserts. + */ +export const hexBuffer = t.codec( + 'hexBuffer', + (decoded: Buffer) => { + return decoded.toString('hex'); + }, + (encoded: string | Uint8Array) => { + if (encoded instanceof Uint8Array) { + return Buffer.from(encoded); + } + if (typeof encoded !== 'string') { + throw new Error(`Expected either a Buffer instance or hex encoded buffer string`); + } + return Buffer.from(encoded, 'hex'); + } +); + +/** + * PGWire returns INTEGER columns as a `bigint`. + * This does a decode operation to `number`. + */ +export const pgwire_number = t.codec( + 'pg_number', + (decoded: number) => decoded, + (encoded: bigint | number) => { + if (typeof encoded == 'number') { + return encoded; + } + if (typeof encoded !== 'bigint') { + throw new Error(`Expected either number or bigint for value`); + } + if (encoded > BigInt(Number.MAX_SAFE_INTEGER) || encoded < BigInt(Number.MIN_SAFE_INTEGER)) { + throw new RangeError('BigInt value is out of safe integer range for conversion to Number.'); + } + return Number(encoded); + } +); + +/** + * A codec which contains the same type on the input and output. + */ +export const IdentityCodec = () => + t.codec( + 'identity', + (encoded) => encoded, + (decoded) => decoded + ); diff --git a/modules/module-postgres-storage/src/types/models/ActiveCheckpoint.ts b/modules/module-postgres-storage/src/types/models/ActiveCheckpoint.ts new file mode 100644 index 000000000..f55ba29ed --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/ActiveCheckpoint.ts @@ -0,0 +1,15 @@ +import * as t from 'ts-codec'; +import { bigint, pgwire_number } from '../codecs.js'; + +/** + * Notification payload sent via Postgres' NOTIFY API. + * + */ +export const ActiveCheckpoint = t.object({ + id: pgwire_number, + last_checkpoint: t.Null.or(bigint), + last_checkpoint_lsn: t.Null.or(t.string) +}); + +export type ActiveCheckpoint = t.Encoded; +export type ActiveCheckpointDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/ActiveCheckpointNotification.ts b/modules/module-postgres-storage/src/types/models/ActiveCheckpointNotification.ts new file mode 100644 index 000000000..e2d49989b --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/ActiveCheckpointNotification.ts @@ -0,0 +1,14 @@ +import * as t from 'ts-codec'; +import { jsonb } from '../codecs.js'; +import { ActiveCheckpoint } from './ActiveCheckpoint.js'; + +export const ActiveCheckpointPayload = t.object({ + active_checkpoint: ActiveCheckpoint +}); + +export type ActiveCheckpointPayload = t.Encoded; +export type ActiveCheckpointPayloadDecoded = t.Decoded; + +export const ActiveCheckpointNotification = jsonb(ActiveCheckpointPayload); +export type ActiveCheckpointNotification = t.Encoded; +export type ActiveCheckpointNotificationDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/BucketData.ts b/modules/module-postgres-storage/src/types/models/BucketData.ts new file mode 100644 index 000000000..757e58165 --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/BucketData.ts @@ -0,0 +1,26 @@ +import * as t from 'ts-codec'; +import { bigint, hexBuffer, pgwire_number } from '../codecs.js'; + +export enum OpType { + PUT = 'PUT', + REMOVE = 'REMOVE', + MOVE = 'MOVE', + CLEAR = 'CLEAR' +} + +export const BucketData = t.object({ + group_id: pgwire_number, + bucket_name: t.string, + op_id: bigint, + op: t.Enum(OpType), + source_table: t.Null.or(t.string), + source_key: t.Null.or(hexBuffer), + table_name: t.string.or(t.Null), + row_id: t.string.or(t.Null), + checksum: bigint, + data: t.Null.or(t.string), + target_op: t.Null.or(bigint) +}); + +export type BucketData = t.Encoded; +export type BucketDataDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/BucketParameters.ts b/modules/module-postgres-storage/src/types/models/BucketParameters.ts new file mode 100644 index 000000000..e4744c895 --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/BucketParameters.ts @@ -0,0 +1,14 @@ +import * as t from 'ts-codec'; +import { bigint, hexBuffer, pgwire_number } from '../codecs.js'; + +export const BucketParameters = t.object({ + id: bigint, + group_id: pgwire_number, + source_table: t.string, + source_key: hexBuffer, + lookup: hexBuffer, + bucket_parameters: t.string +}); + +export type BucketParameters = t.Encoded; +export type BucketParametersDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/CurrentData.ts b/modules/module-postgres-storage/src/types/models/CurrentData.ts new file mode 100644 index 000000000..828d9a8c0 --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/CurrentData.ts @@ -0,0 +1,23 @@ +import * as t from 'ts-codec'; +import { hexBuffer, jsonb, pgwire_number } from '../codecs.js'; + +export const CurrentBucket = t.object({ + bucket: t.string, + table: t.string, + id: t.string +}); + +export type CurrentBucket = t.Encoded; +export type CurrentBucketDecoded = t.Decoded; + +export const CurrentData = t.object({ + buckets: jsonb(t.array(CurrentBucket)), + data: hexBuffer, + group_id: pgwire_number, + lookups: t.array(hexBuffer), + source_key: hexBuffer, + source_table: t.string +}); + +export type CurrentData = t.Encoded; +export type CurrentDataDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/Instance.ts b/modules/module-postgres-storage/src/types/models/Instance.ts new file mode 100644 index 000000000..c1e94e2db --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/Instance.ts @@ -0,0 +1,8 @@ +import * as t from 'ts-codec'; + +export const Instance = t.object({ + id: t.string +}); + +export type Instance = t.Encoded; +export type InstanceDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/Migration.ts b/modules/module-postgres-storage/src/types/models/Migration.ts new file mode 100644 index 000000000..3cf283c58 --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/Migration.ts @@ -0,0 +1,19 @@ +import { framework } from '@powersync/service-core'; +import * as t from 'ts-codec'; +import { jsonb } from '../codecs.js'; + +export const Migration = t.object({ + last_run: t.string, + log: jsonb( + t.array( + t.object({ + name: t.string, + direction: t.Enum(framework.migrations.Direction), + timestamp: framework.codecs.date + }) + ) + ) +}); + +export type Migration = t.Encoded; +export type MigrationDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/SourceTable.ts b/modules/module-postgres-storage/src/types/models/SourceTable.ts new file mode 100644 index 000000000..1673bc959 --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/SourceTable.ts @@ -0,0 +1,32 @@ +import * as t from 'ts-codec'; +import { bigint, jsonb, jsonb_raw, pgwire_number } from '../codecs.js'; + +export type StoredRelationId = { + object_id: string | number; +}; + +export const ColumnDescriptor = t.object({ + name: t.string, + /** + * The type of the column ie VARCHAR, INT, etc + */ + type: t.string.optional(), + /** + * Some data sources have a type id that can be used to identify the type of the column + */ + typeId: t.number.optional() +}); + +export const SourceTable = t.object({ + id: t.string, + group_id: pgwire_number, + connection_id: bigint, + relation_id: t.Null.or(jsonb_raw()), + schema_name: t.string, + table_name: t.string, + replica_id_columns: t.Null.or(jsonb(t.array(ColumnDescriptor))), + snapshot_done: t.boolean +}); + +export type SourceTable = t.Encoded; +export type SourceTableDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/SyncRules.ts b/modules/module-postgres-storage/src/types/models/SyncRules.ts new file mode 100644 index 000000000..8edc5eea4 --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/SyncRules.ts @@ -0,0 +1,50 @@ +import { framework, storage } from '@powersync/service-core'; +import * as t from 'ts-codec'; +import { bigint, pgwire_number } from '../codecs.js'; + +export const SyncRules = t.object({ + id: pgwire_number, + state: t.Enum(storage.SyncRuleState), + /** + * True if initial snapshot has been replicated. + * + * Can only be false if state == PROCESSING. + */ + snapshot_done: t.boolean, + /** + * The last consistent checkpoint. + * + * There may be higher OpIds used in the database if we're in the middle of replicating a large transaction. + */ + last_checkpoint: t.Null.or(bigint), + /** + * The LSN associated with the last consistent checkpoint. + */ + last_checkpoint_lsn: t.Null.or(t.string), + /** + * If set, no new checkpoints may be created < this value. + */ + no_checkpoint_before: t.Null.or(t.string), + slot_name: t.string, + /** + * Last time we persisted a checkpoint. + * + * This may be old if no data is incoming. + */ + last_checkpoint_ts: t.Null.or(framework.codecs.date), + /** + * Last time we persisted a checkpoint or keepalive. + * + * This should stay fairly current while replicating. + */ + last_keepalive_ts: t.Null.or(framework.codecs.date), + /** + * If an error is stopping replication, it will be stored here. + */ + last_fatal_error: t.Null.or(t.string), + keepalive_op: t.Null.or(bigint), + content: t.string +}); + +export type SyncRules = t.Encoded; +export type SyncRulesDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/WriteCheckpoint.ts b/modules/module-postgres-storage/src/types/models/WriteCheckpoint.ts new file mode 100644 index 000000000..1fd74ddc2 --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/WriteCheckpoint.ts @@ -0,0 +1,20 @@ +import * as t from 'ts-codec'; +import { bigint, jsonb } from '../codecs.js'; + +export const WriteCheckpoint = t.object({ + user_id: t.string, + lsns: jsonb(t.record(t.string)), + write_checkpoint: bigint +}); + +export type WriteCheckpoint = t.Encoded; +export type WriteCheckpointDecoded = t.Decoded; + +export const CustomWriteCheckpoint = t.object({ + user_id: t.string, + write_checkpoint: bigint, + sync_rules_id: bigint +}); + +export type CustomWriteCheckpoint = t.Encoded; +export type CustomWriteCheckpointDecoded = t.Decoded; diff --git a/modules/module-postgres-storage/src/types/models/models-index.ts b/modules/module-postgres-storage/src/types/models/models-index.ts new file mode 100644 index 000000000..fb5574608 --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/models-index.ts @@ -0,0 +1,10 @@ +export * from './ActiveCheckpoint.js'; +export * from './ActiveCheckpointNotification.js'; +export * from './BucketData.js'; +export * from './BucketParameters.js'; +export * from './CurrentData.js'; +export * from './Instance.js'; +export * from './Migration.js'; +export * from './SourceTable.js'; +export * from './SyncRules.js'; +export * from './WriteCheckpoint.js'; diff --git a/modules/module-postgres-storage/src/types/types.ts b/modules/module-postgres-storage/src/types/types.ts new file mode 100644 index 000000000..80ff067b4 --- /dev/null +++ b/modules/module-postgres-storage/src/types/types.ts @@ -0,0 +1,73 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; +import * as pg_wire from '@powersync/service-jpgwire'; +import { configFile } from '@powersync/service-types'; +import * as t from 'ts-codec'; +export * as models from './models/models-index.js'; + +export const MAX_BATCH_RECORD_COUNT = 2000; + +export const MAX_BATCH_ESTIMATED_SIZE = 5_000_000; + +export const MAX_BATCH_CURRENT_DATA_SIZE = 50_000_000; + +export const BatchLimits = t.object({ + /** + * Maximum size of operations we write in a single transaction. + */ + max_estimated_size: t.number.optional(), + /** + * Limit number of documents to write in a single transaction. + */ + max_record_count: t.number.optional() +}); + +export type BatchLimits = t.Encoded; + +export const OperationBatchLimits = BatchLimits.and( + t.object({ + /** + * Maximum size of size of current_data documents we lookup at a time. + */ + max_current_data_batch_size: t.number.optional() + }) +); + +export type OperationBatchLimits = t.Encoded; + +export const PostgresStorageConfig = configFile.BaseStorageConfig.and(lib_postgres.BasePostgresConnectionConfig).and( + t.object({ + /** + * Allow batch operation limits to be configurable. + * Postgres has less batch size restrictions compared to MongoDB. + * Increasing limits can drastically improve replication performance, but + * can come at the cost of higher memory usage or potential issues. + */ + batch_limits: OperationBatchLimits.optional() + }) +); + +export type PostgresStorageConfig = t.Encoded; +export type PostgresStorageConfigDecoded = t.Decoded; + +export type RequiredOperationBatchLimits = Required; + +export type NormalizedPostgresStorageConfig = pg_wire.NormalizedConnectionConfig & { + batch_limits: RequiredOperationBatchLimits; +}; + +export const normalizePostgresStorageConfig = ( + baseConfig: PostgresStorageConfigDecoded +): NormalizedPostgresStorageConfig => { + return { + ...lib_postgres.normalizeConnectionConfig(baseConfig), + batch_limits: { + max_current_data_batch_size: baseConfig.batch_limits?.max_current_data_batch_size ?? MAX_BATCH_CURRENT_DATA_SIZE, + max_estimated_size: baseConfig.batch_limits?.max_estimated_size ?? MAX_BATCH_ESTIMATED_SIZE, + max_record_count: baseConfig.batch_limits?.max_record_count ?? MAX_BATCH_RECORD_COUNT + } + }; +}; + +export const isPostgresStorageConfig = (config: configFile.BaseStorageConfig): config is PostgresStorageConfig => { + return config.type == lib_postgres.POSTGRES_CONNECTION_TYPE; +}; diff --git a/modules/module-postgres-storage/src/utils/bson.ts b/modules/module-postgres-storage/src/utils/bson.ts new file mode 100644 index 000000000..c60be1775 --- /dev/null +++ b/modules/module-postgres-storage/src/utils/bson.ts @@ -0,0 +1,17 @@ +import { storage, utils } from '@powersync/service-core'; +import * as uuid from 'uuid'; + +/** + * BSON is used to serialize certain documents for storage in BYTEA columns. + * JSONB columns do not directly support storing binary data which could be required in future. + */ + +export function replicaIdToSubkey(tableId: string, id: storage.ReplicaId): string { + // Hashed UUID from the table and id + if (storage.isUUID(id)) { + // Special case for UUID for backwards-compatiblity + return `${tableId}/${id.toHexString()}`; + } + const repr = storage.serializeBson({ table: tableId, id }); + return uuid.v5(repr, utils.ID_NAMESPACE); +} diff --git a/modules/module-postgres-storage/src/utils/bucket-data.ts b/modules/module-postgres-storage/src/utils/bucket-data.ts new file mode 100644 index 000000000..e4b7f504d --- /dev/null +++ b/modules/module-postgres-storage/src/utils/bucket-data.ts @@ -0,0 +1,25 @@ +import { utils } from '@powersync/service-core'; +import { models } from '../types/types.js'; +import { replicaIdToSubkey } from './bson.js'; + +export const mapOpEntry = (entry: models.BucketDataDecoded) => { + if (entry.op == models.OpType.PUT || entry.op == models.OpType.REMOVE) { + return { + op_id: utils.timestampToOpId(entry.op_id), + op: entry.op, + object_type: entry.table_name ?? undefined, + object_id: entry.row_id ?? undefined, + checksum: Number(entry.checksum), + subkey: replicaIdToSubkey(entry.source_table!, entry.source_key!), + data: entry.data + }; + } else { + // MOVE, CLEAR + + return { + op_id: utils.timestampToOpId(entry.op_id), + op: entry.op, + checksum: Number(entry.checksum) + }; + } +}; diff --git a/modules/module-postgres-storage/src/utils/db.ts b/modules/module-postgres-storage/src/utils/db.ts new file mode 100644 index 000000000..500cb3aa8 --- /dev/null +++ b/modules/module-postgres-storage/src/utils/db.ts @@ -0,0 +1,27 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; + +export const STORAGE_SCHEMA_NAME = 'powersync'; + +export const NOTIFICATION_CHANNEL = 'powersynccheckpoints'; + +/** + * Re export for prettier to detect the tag better + */ +export const sql = lib_postgres.sql; + +export const dropTables = async (client: lib_postgres.DatabaseClient) => { + // Lock a connection for automatic schema search paths + await client.lockConnection(async (db) => { + await db.sql`DROP TABLE IF EXISTS bucket_data`.execute(); + await db.sql`DROP TABLE IF EXISTS bucket_parameters`.execute(); + await db.sql`DROP TABLE IF EXISTS sync_rules`.execute(); + await db.sql`DROP TABLE IF EXISTS instance`.execute(); + await db.sql`DROP TABLE IF EXISTS bucket_data`.execute(); + await db.sql`DROP TABLE IF EXISTS current_data`.execute(); + await db.sql`DROP TABLE IF EXISTS source_tables`.execute(); + await db.sql`DROP TABLE IF EXISTS write_checkpoints`.execute(); + await db.sql`DROP TABLE IF EXISTS custom_write_checkpoints`.execute(); + await db.sql`DROP SEQUENCE IF EXISTS op_id_sequence`.execute(); + await db.sql`DROP SEQUENCE IF EXISTS sync_rules_id_sequence`.execute(); + }); +}; diff --git a/modules/module-postgres-storage/src/utils/ts-codec.ts b/modules/module-postgres-storage/src/utils/ts-codec.ts new file mode 100644 index 000000000..1740b10c6 --- /dev/null +++ b/modules/module-postgres-storage/src/utils/ts-codec.ts @@ -0,0 +1,14 @@ +import * as t from 'ts-codec'; + +/** + * Returns a new codec with a subset of keys. Equivalent to the TypeScript Pick utility. + */ +export const pick = (codec: t.ObjectCodec, keys: Keys[]) => { + // Filter the shape by the specified keys + const newShape = Object.fromEntries( + Object.entries(codec.props.shape).filter(([key]) => keys.includes(key as Keys)) + ) as Pick; + + // Return a new codec with the narrowed shape + return t.object(newShape) as t.ObjectCodec>; +}; diff --git a/modules/module-postgres-storage/src/utils/utils-index.ts b/modules/module-postgres-storage/src/utils/utils-index.ts new file mode 100644 index 000000000..65f808ff7 --- /dev/null +++ b/modules/module-postgres-storage/src/utils/utils-index.ts @@ -0,0 +1,4 @@ +export * from './bson.js'; +export * from './bucket-data.js'; +export * from './db.js'; +export * from './ts-codec.js'; diff --git a/modules/module-postgres-storage/test/src/__snapshots__/storage.test.ts.snap b/modules/module-postgres-storage/test/src/__snapshots__/storage.test.ts.snap new file mode 100644 index 000000000..10eb83682 --- /dev/null +++ b/modules/module-postgres-storage/test/src/__snapshots__/storage.test.ts.snap @@ -0,0 +1,9 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`Postgres Sync Bucket Storage > empty storage metrics 1`] = ` +{ + "operations_size_bytes": 16384, + "parameters_size_bytes": 32768, + "replication_size_bytes": 16384, +} +`; diff --git a/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap b/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap new file mode 100644 index 000000000..64e792029 --- /dev/null +++ b/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap @@ -0,0 +1,332 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`sync - postgres > compacting data - invalidate checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -93886621, + "count": 2, + }, + ], + "last_op_id": "2", + "write_checkpoint": undefined, + }, + }, +] +`; + +exports[`sync - postgres > compacting data - invalidate checkpoint 2`] = ` +[ + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": -93886621n, + "op": "CLEAR", + "op_id": "2", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "mybucket[]", + "checksum": 499012468, + "count": 4, + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "2", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 1859363232n, + "data": "{"id":"t1","description":"Test 1b"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "3", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3028503153n, + "data": "{"id":"t2","description":"Test 2b"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "4", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "4", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4", + }, + }, +] +`; + +exports[`sync - postgres > expired token 1`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - postgres > expiring token 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": 0, + "count": 0, + }, + ], + "last_op_id": "0", + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - postgres > expiring token 2`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - postgres > sync global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -93886621, + "count": 2, + }, + ], + "last_op_id": "2", + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 920318466n, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3280762209n, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - postgres > sync legacy non-raw data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -852817836, + "count": 1, + }, + ], + "last_op_id": "1", + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 3442149460n, + "data": { + "description": "Test +"string"", + "id": "t1", + "large_num": 12345678901234567890n, + }, + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - postgres > sync updates to global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": 0, + "count": 0, + }, + ], + "last_op_id": "0", + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - postgres > sync updates to global data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "mybucket[]", + "checksum": 920318466, + "count": 1, + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 920318466n, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - postgres > sync updates to global data 3`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "mybucket[]", + "checksum": -93886621, + "count": 2, + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 3280762209n, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; diff --git a/modules/module-postgres-storage/test/src/env.ts b/modules/module-postgres-storage/test/src/env.ts new file mode 100644 index 000000000..6047ebcfa --- /dev/null +++ b/modules/module-postgres-storage/test/src/env.ts @@ -0,0 +1,6 @@ +import { utils } from '@powersync/lib-services-framework'; + +export const env = utils.collectEnvironmentVariables({ + PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5431/powersync_storage_test'), + CI: utils.type.boolean.default('false') +}); diff --git a/modules/module-postgres-storage/test/src/migrations.test.ts b/modules/module-postgres-storage/test/src/migrations.test.ts new file mode 100644 index 000000000..ee3acd955 --- /dev/null +++ b/modules/module-postgres-storage/test/src/migrations.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from 'vitest'; + +import { register } from '@powersync/service-core-tests'; +import { PostgresMigrationAgent } from '../../src/migrations/PostgresMigrationAgent.js'; +import { env } from './env.js'; +import { POSTGRES_STORAGE_FACTORY } from './util.js'; + +const MIGRATION_AGENT_FACTORY = () => { + return new PostgresMigrationAgent({ type: 'postgresql', uri: env.PG_STORAGE_TEST_URL, sslmode: 'disable' }); +}; + +describe('Migrations', () => { + register.registerMigrationTests(MIGRATION_AGENT_FACTORY); + + it('Should have tables declared', async () => { + const { db } = await POSTGRES_STORAGE_FACTORY(); + + const tables = await db.sql` + SELECT + table_schema, + table_name + FROM + information_schema.tables + WHERE + table_type = 'BASE TABLE' + AND table_schema NOT IN ('pg_catalog', 'information_schema') + ORDER BY + table_schema, + table_name; + `.rows<{ table_schema: string; table_name: string }>(); + + expect(tables.find((t) => t.table_name == 'sync_rules')).exist; + }); +}); diff --git a/modules/module-postgres-storage/test/src/setup.ts b/modules/module-postgres-storage/test/src/setup.ts new file mode 100644 index 000000000..802007e89 --- /dev/null +++ b/modules/module-postgres-storage/test/src/setup.ts @@ -0,0 +1,16 @@ +import { container } from '@powersync/lib-services-framework'; +import { Metrics } from '@powersync/service-core'; +import { beforeAll } from 'vitest'; + +beforeAll(async () => { + // Executes for every test file + container.registerDefaults(); + + // The metrics need to be initialized before they can be used + await Metrics.initialise({ + disable_telemetry_sharing: true, + powersync_instance_id: 'test', + internal_metrics_endpoint: 'unused.for.tests.com' + }); + Metrics.getInstance().resetCounters(); +}); diff --git a/modules/module-postgres-storage/test/src/storage.test.ts b/modules/module-postgres-storage/test/src/storage.test.ts new file mode 100644 index 000000000..5977ec220 --- /dev/null +++ b/modules/module-postgres-storage/test/src/storage.test.ts @@ -0,0 +1,131 @@ +import { storage } from '@powersync/service-core'; +import { register, TEST_TABLE, test_utils } from '@powersync/service-core-tests'; +import { describe, expect, test } from 'vitest'; +import { POSTGRES_STORAGE_FACTORY } from './util.js'; + +describe('Sync Bucket Validation', register.registerBucketValidationTests); + +describe('Postgres Sync Bucket Storage', () => { + register.registerDataStorageTests(POSTGRES_STORAGE_FACTORY); + + /** + * The split of returned results can vary depending on storage drivers. + * The large rows here are 2MB large while the default chunk limit is 1mb. + * The Postgres storage driver will detect if the next row will increase the batch + * over the limit and separate that row into a new batch (or single row batch) if applicable. + */ + test('large batch (2)', async () => { + // Test syncing a batch of data that is small in count, + // but large enough in size to be split over multiple returned chunks. + // Similar to the above test, but splits over 1MB chunks. + const sync_rules = test_utils.testRules( + ` + bucket_definitions: + global: + data: + - SELECT id, description FROM "%" + ` + ); + using factory = await POSTGRES_STORAGE_FACTORY(); + const bucketStorage = factory.getInstance(sync_rules); + + const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { + const sourceTable = TEST_TABLE; + + const largeDescription = '0123456789'.repeat(2_000_00); + + await batch.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test1', + description: 'test1' + }, + afterReplicaId: test_utils.rid('test1') + }); + + await batch.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'large1', + description: largeDescription + }, + afterReplicaId: test_utils.rid('large1') + }); + + // Large enough to split the returned batch + await batch.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'large2', + description: largeDescription + }, + afterReplicaId: test_utils.rid('large2') + }); + + await batch.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'test3', + description: 'test3' + }, + afterReplicaId: test_utils.rid('test3') + }); + }); + + const checkpoint = result!.flushed_op; + + const options: storage.BucketDataBatchOptions = {}; + + const batch1 = await test_utils.fromAsync( + bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']]), options) + ); + expect(test_utils.getBatchData(batch1)).toEqual([ + { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 } + ]); + expect(test_utils.getBatchMeta(batch1)).toEqual({ + after: '0', + has_more: true, + next_after: '1' + }); + + const batch2 = await test_utils.fromAsync( + bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', batch1[0].batch.next_after]]), options) + ); + expect(test_utils.getBatchData(batch2)).toEqual([ + { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 1178768505 } + ]); + expect(test_utils.getBatchMeta(batch2)).toEqual({ + after: '1', + has_more: true, + next_after: '2' + }); + + const batch3 = await test_utils.fromAsync( + bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', batch2[0].batch.next_after]]), options) + ); + expect(test_utils.getBatchData(batch3)).toEqual([ + { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1607205872 } + ]); + expect(test_utils.getBatchMeta(batch3)).toEqual({ + after: '2', + has_more: true, + next_after: '3' + }); + + const batch4 = await test_utils.fromAsync( + bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', batch3[0].batch.next_after]]), options) + ); + expect(test_utils.getBatchData(batch4)).toEqual([ + { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 } + ]); + expect(test_utils.getBatchMeta(batch4)).toEqual({ + after: '3', + has_more: false, + next_after: '4' + }); + }); +}); diff --git a/modules/module-postgres-storage/test/src/storage_compacting.test.ts b/modules/module-postgres-storage/test/src/storage_compacting.test.ts new file mode 100644 index 000000000..f0b02b696 --- /dev/null +++ b/modules/module-postgres-storage/test/src/storage_compacting.test.ts @@ -0,0 +1,5 @@ +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { POSTGRES_STORAGE_FACTORY } from './util.js'; + +describe('Postgres Sync Bucket Storage Compact', () => register.registerCompactTests(POSTGRES_STORAGE_FACTORY, {})); diff --git a/modules/module-postgres-storage/test/src/storage_sync.test.ts b/modules/module-postgres-storage/test/src/storage_sync.test.ts new file mode 100644 index 000000000..d7aae902a --- /dev/null +++ b/modules/module-postgres-storage/test/src/storage_sync.test.ts @@ -0,0 +1,12 @@ +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { POSTGRES_STORAGE_FACTORY } from './util.js'; + +/** + * Bucket compacting is not yet implemented. + * This causes the internal compacting test to fail. + * Other tests have been verified manually. + */ +describe('sync - postgres', () => { + register.registerSyncTests(POSTGRES_STORAGE_FACTORY); +}); diff --git a/modules/module-postgres-storage/test/src/util.ts b/modules/module-postgres-storage/test/src/util.ts new file mode 100644 index 000000000..3f0cd4428 --- /dev/null +++ b/modules/module-postgres-storage/test/src/util.ts @@ -0,0 +1,34 @@ +import path from 'path'; +import { fileURLToPath } from 'url'; +import { normalizePostgresStorageConfig } from '../../src//types/types.js'; +import { PostgresMigrationAgent } from '../../src/migrations/PostgresMigrationAgent.js'; +import { PostgresTestStorageFactoryGenerator } from '../../src/storage/PostgresTestStorageFactoryGenerator.js'; +import { env } from './env.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +export const TEST_URI = env.PG_STORAGE_TEST_URL; + +const BASE_CONFIG = { + type: 'postgresql' as const, + uri: TEST_URI, + sslmode: 'disable' as const +}; + +export const TEST_CONNECTION_OPTIONS = normalizePostgresStorageConfig(BASE_CONFIG); + +/** + * Vitest tries to load the migrations via .ts files which fails. + * For tests this links to the relevant .js files correctly + */ +class TestPostgresMigrationAgent extends PostgresMigrationAgent { + getInternalScriptsDir(): string { + return path.resolve(__dirname, '../../dist/migrations/scripts'); + } +} + +export const POSTGRES_STORAGE_FACTORY = PostgresTestStorageFactoryGenerator({ + url: env.PG_STORAGE_TEST_URL, + migrationAgent: (config) => new TestPostgresMigrationAgent(config) +}); diff --git a/modules/module-postgres-storage/test/tsconfig.json b/modules/module-postgres-storage/test/tsconfig.json new file mode 100644 index 000000000..486ab8eb7 --- /dev/null +++ b/modules/module-postgres-storage/test/tsconfig.json @@ -0,0 +1,20 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "baseUrl": "./", + "noEmit": true, + "esModuleInterop": true, + "declarationDir": "dist/@types", + "tsBuildInfoFile": "dist/.tsbuildinfo", + "lib": ["ES2022", "esnext.disposable"], + "skipLibCheck": true, + "sourceMap": true + }, + "include": ["src"], + "references": [ + { + "path": "../" + } + ] +} diff --git a/modules/module-postgres-storage/tsconfig.json b/modules/module-postgres-storage/tsconfig.json new file mode 100644 index 000000000..edfd2bf74 --- /dev/null +++ b/modules/module-postgres-storage/tsconfig.json @@ -0,0 +1,36 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "declarationDir": "dist/@types", + "tsBuildInfoFile": "dist/.tsbuildinfo", + "rootDir": "src", + "target": "ES2022", + "lib": ["ES2022", "esnext.disposable"], + "skipLibCheck": true + }, + "include": ["src"], + "references": [ + { + "path": "../../packages/types" + }, + { + "path": "../../packages/jsonbig" + }, + { + "path": "../../packages/jpgwire" + }, + { + "path": "../../packages/sync-rules" + }, + { + "path": "../../packages/service-core" + }, + { + "path": "../../libs/lib-services" + }, + { + "path": "../../libs/lib-postgres" + } + ] +} diff --git a/modules/module-postgres-storage/vitest.config.ts b/modules/module-postgres-storage/vitest.config.ts new file mode 100644 index 000000000..885dab34e --- /dev/null +++ b/modules/module-postgres-storage/vitest.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + setupFiles: './test/src/setup.ts', + poolOptions: { + threads: { + singleThread: true + } + }, + pool: 'threads' + } +}); diff --git a/modules/module-postgres/package.json b/modules/module-postgres/package.json index 15eb45cb0..bad8ea2df 100644 --- a/modules/module-postgres/package.json +++ b/modules/module-postgres/package.json @@ -29,6 +29,7 @@ }, "dependencies": { "@powersync/lib-services-framework": "workspace:*", + "@powersync/lib-service-postgres": "workspace:*", "@powersync/service-core": "workspace:*", "@powersync/service-jpgwire": "workspace:*", "@powersync/service-jsonbig": "workspace:*", @@ -43,6 +44,7 @@ "devDependencies": { "@types/uuid": "^9.0.4", "@powersync/service-core-tests": "workspace:*", - "@powersync/service-module-mongodb-storage": "workspace:*" + "@powersync/service-module-mongodb-storage": "workspace:*", + "@powersync/service-module-postgres-storage": "workspace:*" } } diff --git a/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts b/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts index 76edcea68..2600657e1 100644 --- a/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts +++ b/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts @@ -1,13 +1,12 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; import { api, ParseSyncRulesOptions } from '@powersync/service-core'; import * as pgwire from '@powersync/service-jpgwire'; - import * as sync_rules from '@powersync/service-sync-rules'; import * as service_types from '@powersync/service-types'; import * as replication_utils from '../replication/replication-utils.js'; -import * as types from '../types/types.js'; -import * as pg_utils from '../utils/pgwire_utils.js'; import { getDebugTableInfo } from '../replication/replication-utils.js'; import { PUBLICATION_NAME } from '../replication/WalStream.js'; +import * as types from '../types/types.js'; export class PostgresRouteAPIAdapter implements api.RouteAPI { connectionTag: string; @@ -53,7 +52,7 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI { }; try { - await pg_utils.retriedQuery(this.pool, `SELECT 'PowerSync connection test'`); + await lib_postgres.retriedQuery(this.pool, `SELECT 'PowerSync connection test'`); } catch (e) { return { ...base, @@ -94,7 +93,7 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI { try { const result = await this.pool.query({ statement: query, - params: params.map(pg_utils.autoParameter) + params: params.map(lib_postgres.autoParameter) }); return service_types.internal_routes.ExecuteSqlResponse.encode({ @@ -146,7 +145,7 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI { if (tablePattern.isWildcard) { patternResult.tables = []; const prefix = tablePattern.tablePrefix; - const results = await pg_utils.retriedQuery(this.pool, { + const results = await lib_postgres.retriedQuery(this.pool, { statement: `SELECT c.oid AS relid, c.relname AS table_name FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace @@ -169,7 +168,7 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI { patternResult.tables.push(details); } } else { - const results = await pg_utils.retriedQuery(this.pool, { + const results = await lib_postgres.retriedQuery(this.pool, { statement: `SELECT c.oid AS relid, c.relname AS table_name FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace @@ -215,7 +214,7 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI { async getReplicationLag(options: api.ReplicationLagOptions): Promise { const { bucketStorage } = options; const slotName = bucketStorage.slot_name; - const results = await pg_utils.retriedQuery(this.pool, { + const results = await lib_postgres.retriedQuery(this.pool, { statement: `SELECT slot_name, confirmed_flush_lsn, @@ -237,7 +236,7 @@ FROM pg_replication_slots WHERE slot_name = $1 LIMIT 1;`, // However, on Aurora (Postgres compatible), it can return an entirely different LSN, // causing the write checkpoints to never be replicated back to the client. // For those, we need to use pg_current_wal_lsn() instead. - const { results } = await pg_utils.retriedQuery( + const { results } = await lib_postgres.retriedQuery( this.pool, { statement: `SELECT pg_current_wal_lsn() as lsn` }, { statement: `SELECT pg_logical_emit_message(false, 'powersync', 'ping')` } @@ -250,7 +249,7 @@ FROM pg_replication_slots WHERE slot_name = $1 LIMIT 1;`, async getConnectionSchema(): Promise { // https://github.com/Borvik/vscode-postgres/blob/88ec5ed061a0c9bced6c5d4ec122d0759c3f3247/src/language/server.ts - const results = await pg_utils.retriedQuery( + const results = await lib_postgres.retriedQuery( this.pool, `SELECT tbl.schemaname, diff --git a/modules/module-postgres/src/auth/SupabaseKeyCollector.ts b/modules/module-postgres/src/auth/SupabaseKeyCollector.ts index af300f251..187d68a80 100644 --- a/modules/module-postgres/src/auth/SupabaseKeyCollector.ts +++ b/modules/module-postgres/src/auth/SupabaseKeyCollector.ts @@ -1,9 +1,9 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; import { auth } from '@powersync/service-core'; import * as pgwire from '@powersync/service-jpgwire'; import * as jose from 'jose'; import * as types from '../types/types.js'; -import * as pgwire_utils from '../utils/pgwire_utils.js'; /** * Fetches key from the Supabase database. @@ -39,7 +39,7 @@ export class SupabaseKeyCollector implements auth.KeyCollector { let row: { jwt_secret: string }; try { const rows = pgwire.pgwireRows( - await pgwire_utils.retriedQuery(this.pool, `SELECT current_setting('app.settings.jwt_secret') as jwt_secret`) + await lib_postgres.retriedQuery(this.pool, `SELECT current_setting('app.settings.jwt_secret') as jwt_secret`) ); row = rows[0] as any; } catch (e) { diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index 34c168f90..92b8a2031 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -1,8 +1,10 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; import { container, errors, logger } from '@powersync/lib-services-framework'; import { getUuidReplicaIdentityBson, Metrics, SourceEntityDescriptor, storage } from '@powersync/service-core'; import * as pgwire from '@powersync/service-jpgwire'; import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern, toSyncRulesRow } from '@powersync/service-sync-rules'; import * as pg_utils from '../utils/pgwire_utils.js'; + import { PgManager } from './PgManager.js'; import { getPgOutputRelation, getRelId } from './PgRelation.js'; import { checkSourceConfiguration, getReplicationIdentityColumns } from './replication-utils.js'; @@ -63,7 +65,7 @@ export class WalStream { // Ping to speed up cancellation of streaming replication // We're not using pg_snapshot here, since it could be in the middle of // an initial replication transaction. - const promise = pg_utils.retriedQuery( + const promise = lib_postgres.retriedQuery( this.connections.pool, `SELECT * FROM pg_logical_emit_message(false, 'powersync', 'ping')` ); diff --git a/modules/module-postgres/src/replication/replication-utils.ts b/modules/module-postgres/src/replication/replication-utils.ts index c6b1e3fe1..6dde04035 100644 --- a/modules/module-postgres/src/replication/replication-utils.ts +++ b/modules/module-postgres/src/replication/replication-utils.ts @@ -1,13 +1,11 @@ import * as pgwire from '@powersync/service-jpgwire'; +import * as lib_postgres from '@powersync/lib-service-postgres'; +import { logger } from '@powersync/lib-services-framework'; import { PatternResult, storage } from '@powersync/service-core'; -import * as pgwire_utils from '../utils/pgwire_utils.js'; -import { ReplicationIdentity } from './PgRelation.js'; import * as sync_rules from '@powersync/service-sync-rules'; import * as service_types from '@powersync/service-types'; -import * as pg_utils from '../utils/pgwire_utils.js'; -import * as util from '../utils/pgwire_utils.js'; -import { logger } from '@powersync/lib-services-framework'; +import { ReplicationIdentity } from './PgRelation.js'; export interface ReplicaIdentityResult { replicationColumns: storage.ColumnDescriptor[]; @@ -20,7 +18,7 @@ export async function getPrimaryKeyColumns( mode: 'primary' | 'replident' ): Promise { const indexFlag = mode == 'primary' ? `i.indisprimary` : `i.indisreplident`; - const attrRows = await pgwire_utils.retriedQuery(db, { + const attrRows = await lib_postgres.retriedQuery(db, { statement: `SELECT a.attname as name, a.atttypid as typeid, t.typname as type, a.attnum as attnum FROM pg_index i JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY (i.indkey) @@ -41,7 +39,7 @@ export async function getPrimaryKeyColumns( } export async function getAllColumns(db: pgwire.PgClient, relationId: number): Promise { - const attrRows = await pgwire_utils.retriedQuery(db, { + const attrRows = await lib_postgres.retriedQuery(db, { statement: `SELECT a.attname as name, a.atttypid as typeid, t.typname as type, a.attnum as attnum FROM pg_attribute a JOIN pg_type t ON a.atttypid = t.oid @@ -62,7 +60,7 @@ export async function getReplicationIdentityColumns( db: pgwire.PgClient, relationId: number ): Promise { - const rows = await pgwire_utils.retriedQuery(db, { + const rows = await lib_postgres.retriedQuery(db, { statement: `SELECT CASE relreplident WHEN 'd' THEN 'default' WHEN 'n' THEN 'nothing' @@ -95,7 +93,7 @@ WHERE oid = $1::oid LIMIT 1`, export async function checkSourceConfiguration(db: pgwire.PgClient, publicationName: string): Promise { // Check basic config - await pgwire_utils.retriedQuery( + await lib_postgres.retriedQuery( db, `DO $$ BEGIN @@ -113,7 +111,7 @@ $$ LANGUAGE plpgsql;` ); // Check that publication exists - const rs = await pgwire_utils.retriedQuery(db, { + const rs = await lib_postgres.retriedQuery(db, { statement: `SELECT * FROM pg_publication WHERE pubname = $1`, params: [{ type: 'varchar', value: publicationName }] }); @@ -158,7 +156,7 @@ export async function getDebugTablesInfo(options: GetDebugTablesInfoOptions): Pr if (tablePattern.isWildcard) { patternResult.tables = []; const prefix = tablePattern.tablePrefix; - const results = await util.retriedQuery(db, { + const results = await lib_postgres.retriedQuery(db, { statement: `SELECT c.oid AS relid, c.relname AS table_name FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace @@ -189,7 +187,7 @@ export async function getDebugTablesInfo(options: GetDebugTablesInfoOptions): Pr patternResult.tables.push(details); } } else { - const results = await util.retriedQuery(db, { + const results = await lib_postgres.retriedQuery(db, { statement: `SELECT c.oid AS relid, c.relname AS table_name FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace @@ -284,14 +282,14 @@ export async function getDebugTableInfo(options: GetDebugTableInfoOptions): Prom let selectError = null; try { - await pg_utils.retriedQuery(db, `SELECT * FROM ${sourceTable.escapedIdentifier} LIMIT 1`); + await lib_postgres.retriedQuery(db, `SELECT * FROM ${sourceTable.escapedIdentifier} LIMIT 1`); } catch (e) { selectError = { level: 'fatal', message: e.message }; } let replicateError = null; - const publications = await pg_utils.retriedQuery(db, { + const publications = await lib_postgres.retriedQuery(db, { statement: `SELECT tablename FROM pg_publication_tables WHERE pubname = $1 AND schemaname = $2 AND tablename = $3`, params: [ { type: 'varchar', value: publicationName }, diff --git a/modules/module-postgres/src/types/types.ts b/modules/module-postgres/src/types/types.ts index 3629a8cb7..4de2ac0ed 100644 --- a/modules/module-postgres/src/types/types.ts +++ b/modules/module-postgres/src/types/types.ts @@ -1,56 +1,18 @@ +import * as lib_postgres from '@powersync/lib-service-postgres'; import * as service_types from '@powersync/service-types'; import * as t from 'ts-codec'; -import * as urijs from 'uri-js'; -export const POSTGRES_CONNECTION_TYPE = 'postgresql' as const; - -export interface NormalizedPostgresConnectionConfig { - id: string; - tag: string; - - hostname: string; - port: number; - database: string; - - username: string; - password: string; - - sslmode: 'verify-full' | 'verify-ca' | 'disable'; - cacert: string | undefined; - - client_certificate: string | undefined; - client_private_key: string | undefined; -} +// Maintain backwards compatibility by exporting these +export const validatePort = lib_postgres.validatePort; +export const baseUri = lib_postgres.baseUri; +export type NormalizedPostgresConnectionConfig = lib_postgres.NormalizedBasePostgresConnectionConfig; +export const POSTGRES_CONNECTION_TYPE = lib_postgres.POSTGRES_CONNECTION_TYPE; export const PostgresConnectionConfig = service_types.configFile.DataSourceConfig.and( + lib_postgres.BasePostgresConnectionConfig +).and( t.object({ - type: t.literal(POSTGRES_CONNECTION_TYPE), - /** Unique identifier for the connection - optional when a single connection is present. */ - id: t.string.optional(), - /** Tag used as reference in sync rules. Defaults to "default". Does not have to be unique. */ - tag: t.string.optional(), - uri: t.string.optional(), - hostname: t.string.optional(), - port: service_types.configFile.portCodec.optional(), - username: t.string.optional(), - password: t.string.optional(), - database: t.string.optional(), - - /** Defaults to verify-full */ - sslmode: t.literal('verify-full').or(t.literal('verify-ca')).or(t.literal('disable')).optional(), - /** Required for verify-ca, optional for verify-full */ - cacert: t.string.optional(), - - client_certificate: t.string.optional(), - client_private_key: t.string.optional(), - - /** Expose database credentials */ - demo_database: t.boolean.optional(), - - /** - * Prefix for the slot name. Defaults to "powersync_" - */ - slot_name_prefix: t.string.optional() + // Add any replication connection specific config here in future }) ); @@ -64,101 +26,19 @@ export type PostgresConnectionConfig = t.Decoded SQLite row. @@ -28,46 +26,3 @@ export function constructBeforeRecord(message: pgwire.PgoutputDelete | pgwire.Pg const record = pgwire.decodeTuple(message.relation, rawData); return toSyncRulesRow(record); } - -export function escapeIdentifier(identifier: string) { - return `"${identifier.replace(/"/g, '""').replace(/\./g, '"."')}"`; -} - -export function autoParameter(arg: SqliteJsonValue | boolean): pgwire.StatementParam { - if (arg == null) { - return { type: 'varchar', value: null }; - } else if (typeof arg == 'string') { - return { type: 'varchar', value: arg }; - } else if (typeof arg == 'number') { - if (Number.isInteger(arg)) { - return { type: 'int8', value: arg }; - } else { - return { type: 'float8', value: arg }; - } - } else if (typeof arg == 'boolean') { - return { type: 'bool', value: arg }; - } else if (typeof arg == 'bigint') { - return { type: 'int8', value: arg }; - } else { - throw new Error(`Unsupported query parameter: ${typeof arg}`); - } -} - -export async function retriedQuery(db: pgwire.PgClient, ...statements: pgwire.Statement[]): Promise; -export async function retriedQuery(db: pgwire.PgClient, query: string): Promise; - -/** - * Retry a simple query - up to 2 attempts total. - */ -export async function retriedQuery(db: pgwire.PgClient, ...args: any[]) { - for (let tries = 2; ; tries--) { - try { - return await db.query(...args); - } catch (e) { - if (tries == 1) { - throw e; - } - logger.warn('Query error, retrying', e); - } - } -} diff --git a/modules/module-postgres/test/src/__snapshots__/schema_changes.test.ts.snap b/modules/module-postgres/test/src/__snapshots__/schema_changes.test.ts.snap new file mode 100644 index 000000000..2ba06c6fe --- /dev/null +++ b/modules/module-postgres/test/src/__snapshots__/schema_changes.test.ts.snap @@ -0,0 +1,5 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`schema changes - mongodb > add to publication (not in sync rules) 1`] = `0`; + +exports[`schema changes - postgres > add to publication (not in sync rules) 1`] = `16384`; diff --git a/modules/module-postgres/test/src/env.ts b/modules/module-postgres/test/src/env.ts index 214b75ca6..58b69e235 100644 --- a/modules/module-postgres/test/src/env.ts +++ b/modules/module-postgres/test/src/env.ts @@ -2,7 +2,10 @@ import { utils } from '@powersync/lib-services-framework'; export const env = utils.collectEnvironmentVariables({ PG_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_test'), + PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5431/powersync_storage_test'), MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'), CI: utils.type.boolean.default('false'), - SLOW_TESTS: utils.type.boolean.default('false') + SLOW_TESTS: utils.type.boolean.default('false'), + TEST_MONGO_STORAGE: utils.type.boolean.default('true'), + TEST_POSTGRES_STORAGE: utils.type.boolean.default('true') }); diff --git a/modules/module-postgres/test/src/large_batch.test.ts b/modules/module-postgres/test/src/large_batch.test.ts index 4d49a259f..2806bad2f 100644 --- a/modules/module-postgres/test/src/large_batch.test.ts +++ b/modules/module-postgres/test/src/large_batch.test.ts @@ -3,10 +3,14 @@ import * as timers from 'timers/promises'; import { describe, expect, test } from 'vitest'; import { populateData } from '../../dist/utils/populate_test_data.js'; import { env } from './env.js'; -import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_CONNECTION_OPTIONS } from './util.js'; +import { + INITIALIZED_MONGO_STORAGE_FACTORY, + INITIALIZED_POSTGRES_STORAGE_FACTORY, + TEST_CONNECTION_OPTIONS +} from './util.js'; import { WalStreamTestContext } from './wal_stream_utils.js'; -describe('batch replication tests - mongodb', { timeout: 120_000 }, function () { +describe.skipIf(!env.TEST_MONGO_STORAGE)('batch replication tests - mongodb', { timeout: 120_000 }, function () { // These are slow but consistent tests. // Not run on every test run, but we do run on CI, or when manually debugging issues. if (env.CI || env.SLOW_TESTS) { @@ -17,6 +21,17 @@ describe('batch replication tests - mongodb', { timeout: 120_000 }, function () } }); +describe.skipIf(!env.TEST_POSTGRES_STORAGE)('batch replication tests - postgres', { timeout: 240_000 }, function () { + // These are slow but consistent tests. + // Not run on every test run, but we do run on CI, or when manually debugging issues. + if (env.CI || env.SLOW_TESTS) { + defineBatchTests(INITIALIZED_POSTGRES_STORAGE_FACTORY); + } else { + // Need something in this file. + test('no-op', () => {}); + } +}); + const BASIC_SYNC_RULES = `bucket_definitions: global: data: diff --git a/modules/module-postgres/test/src/schema_changes.test.ts b/modules/module-postgres/test/src/schema_changes.test.ts index c6bb3de23..3fffb3f6d 100644 --- a/modules/module-postgres/test/src/schema_changes.test.ts +++ b/modules/module-postgres/test/src/schema_changes.test.ts @@ -3,13 +3,18 @@ import * as timers from 'timers/promises'; import { describe, expect, test } from 'vitest'; import { storage } from '@powersync/service-core'; -import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; +import { env } from './env.js'; +import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js'; import { WalStreamTestContext } from './wal_stream_utils.js'; -describe('schema changes', { timeout: 20_000 }, function () { +describe.skipIf(!env.TEST_MONGO_STORAGE)('schema changes - mongodb', { timeout: 20_000 }, function () { defineTests(INITIALIZED_MONGO_STORAGE_FACTORY); }); +describe.skipIf(!env.TEST_POSTGRES_STORAGE)('schema changes - postgres', { timeout: 20_000 }, function () { + defineTests(INITIALIZED_POSTGRES_STORAGE_FACTORY); +}); + const BASIC_SYNC_RULES = ` bucket_definitions: global: @@ -432,7 +437,7 @@ function defineTests(factory: storage.TestStorageFactory) { expect(data).toMatchObject([]); const metrics = await storage.factory.getStorageMetrics(); - expect(metrics.replication_size_bytes).toEqual(0); + expect(metrics.replication_size_bytes).toMatchSnapshot(); }); test('replica identity nothing', async () => { diff --git a/modules/module-postgres/test/src/setup.ts b/modules/module-postgres/test/src/setup.ts index debe66011..43946a1aa 100644 --- a/modules/module-postgres/test/src/setup.ts +++ b/modules/module-postgres/test/src/setup.ts @@ -1,9 +1,13 @@ import { container } from '@powersync/lib-services-framework'; import { test_utils } from '@powersync/service-core-tests'; -import { beforeAll } from 'vitest'; +import { beforeAll, beforeEach } from 'vitest'; beforeAll(async () => { // Executes for every test file container.registerDefaults(); await test_utils.initMetrics(); }); + +beforeEach(async () => { + await test_utils.resetMetrics(); +}); diff --git a/modules/module-postgres/test/src/slow_tests.test.ts b/modules/module-postgres/test/src/slow_tests.test.ts index fd02ed90b..438d3bb98 100644 --- a/modules/module-postgres/test/src/slow_tests.test.ts +++ b/modules/module-postgres/test/src/slow_tests.test.ts @@ -7,6 +7,7 @@ import { connectPgPool, getClientCheckpoint, INITIALIZED_MONGO_STORAGE_FACTORY, + INITIALIZED_POSTGRES_STORAGE_FACTORY, TEST_CONNECTION_OPTIONS } from './util.js'; @@ -17,9 +18,10 @@ import { PgManager } from '@module/replication/PgManager.js'; import { storage } from '@powersync/service-core'; import { test_utils } from '@powersync/service-core-tests'; import * as mongo_storage from '@powersync/service-module-mongodb-storage'; +import * as postgres_storage from '@powersync/service-module-postgres-storage'; import * as timers from 'node:timers/promises'; -describe('slow tests - mongodb', function () { +describe.skipIf(!env.TEST_MONGO_STORAGE)('slow tests - mongodb', function () { // These are slow, inconsistent tests. // Not run on every test run, but we do run on CI, or when manually debugging issues. if (env.CI || env.SLOW_TESTS) { @@ -30,6 +32,17 @@ describe('slow tests - mongodb', function () { } }); +describe.skipIf(!env.TEST_POSTGRES_STORAGE)('slow tests - postgres', function () { + // These are slow, inconsistent tests. + // Not run on every test run, but we do run on CI, or when manually debugging issues. + if (env.CI || env.SLOW_TESTS) { + defineSlowTests(INITIALIZED_POSTGRES_STORAGE_FACTORY); + } else { + // Need something in this file. + test('no-op', () => {}); + } +}); + function defineSlowTests(factory: storage.TestStorageFactory) { let walStream: WalStream | undefined; let connections: PgManager | undefined; @@ -79,7 +92,7 @@ function defineSlowTests(factory: storage.TestStorageFactory) { const replicationConnection = await connections.replicationConnection(); const pool = connections.pool; await clearTestDb(pool); - const f = (await factory()) as mongo_storage.storage.MongoBucketStorage; + await using f = await factory(); const syncRuleContent = ` bucket_definitions: @@ -174,15 +187,50 @@ bucket_definitions: } const checkpoint = BigInt((await storage.getCheckpoint()).checkpoint); - const opsBefore = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray()) - .filter((row) => row._id.o <= checkpoint) - .map(mongo_storage.storage.mapOpEntry); - await storage.compact({ maxOpId: checkpoint }); - const opsAfter = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray()) - .filter((row) => row._id.o <= checkpoint) - .map(mongo_storage.storage.mapOpEntry); - - test_utils.validateCompactedBucket(opsBefore, opsAfter); + if (f instanceof mongo_storage.storage.MongoBucketStorage) { + const opsBefore = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray()) + .filter((row) => row._id.o <= checkpoint) + .map(mongo_storage.storage.mapOpEntry); + await storage.compact({ maxOpId: checkpoint }); + const opsAfter = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray()) + .filter((row) => row._id.o <= checkpoint) + .map(mongo_storage.storage.mapOpEntry); + + test_utils.validateCompactedBucket(opsBefore, opsAfter); + } else if (f instanceof postgres_storage.PostgresBucketStorageFactory) { + const { db } = f; + const opsBefore = ( + await db.sql` + SELECT + * + FROM + bucket_data + WHERE + op_id <= ${{ type: 'int8', value: checkpoint }} + ORDER BY + op_id ASC + ` + .decoded(postgres_storage.models.BucketData) + .rows() + ).map(postgres_storage.utils.mapOpEntry); + await storage.compact({ maxOpId: checkpoint }); + const opsAfter = ( + await db.sql` + SELECT + * + FROM + bucket_data + WHERE + op_id <= ${{ type: 'int8', value: checkpoint }} + ORDER BY + op_id ASC + ` + .decoded(postgres_storage.models.BucketData) + .rows() + ).map(postgres_storage.utils.mapOpEntry); + + test_utils.validateCompactedBucket(opsBefore, opsAfter); + } } }; @@ -196,26 +244,66 @@ bucket_definitions: // Wait for replication to finish let checkpoint = await getClientCheckpoint(pool, storage.factory, { timeout: TIMEOUT_MARGIN_MS }); - // Check that all inserts have been deleted again - const docs = await f.db.current_data.find().toArray(); - const transformed = docs.map((doc) => { - return bson.deserialize(doc.data.buffer) as SqliteRow; - }); - expect(transformed).toEqual([]); - - // Check that each PUT has a REMOVE - const ops = await f.db.bucket_data.find().sort({ _id: 1 }).toArray(); - - // All a single bucket in this test - const bucket = ops.map((op) => mongo_storage.storage.mapOpEntry(op)); - const reduced = test_utils.reduceBucket(bucket); - expect(reduced).toMatchObject([ - { - op_id: '0', - op: 'CLEAR' - } - // Should contain no additional data - ]); + if (f instanceof mongo_storage.storage.MongoBucketStorage) { + // Check that all inserts have been deleted again + const docs = await f.db.current_data.find().toArray(); + const transformed = docs.map((doc) => { + return bson.deserialize(doc.data.buffer) as SqliteRow; + }); + expect(transformed).toEqual([]); + + // Check that each PUT has a REMOVE + const ops = await f.db.bucket_data.find().sort({ _id: 1 }).toArray(); + + // All a single bucket in this test + const bucket = ops.map((op) => mongo_storage.storage.mapOpEntry(op)); + const reduced = test_utils.reduceBucket(bucket); + expect(reduced).toMatchObject([ + { + op_id: '0', + op: 'CLEAR' + } + // Should contain no additional data + ]); + } else if (f instanceof postgres_storage.storage.PostgresBucketStorageFactory) { + const { db } = f; + // Check that all inserts have been deleted again + const docs = await db.sql` + SELECT + * + FROM + current_data + ` + .decoded(postgres_storage.models.CurrentData) + .rows(); + const transformed = docs.map((doc) => { + return bson.deserialize(doc.data) as SqliteRow; + }); + expect(transformed).toEqual([]); + + // Check that each PUT has a REMOVE + const ops = await db.sql` + SELECT + * + FROM + bucket_data + ORDER BY + op_id ASC + ` + .decoded(postgres_storage.models.BucketData) + .rows(); + + // All a single bucket in this test + const bucket = ops.map((op) => postgres_storage.utils.mapOpEntry(op)); + const reduced = test_utils.reduceBucket(bucket); + expect(reduced).toMatchObject([ + { + op_id: '0', + op: 'CLEAR' + } + // Should contain no additional data + ]); + } } abortController.abort(); @@ -231,7 +319,7 @@ bucket_definitions: async () => { const pool = await connectPgPool(); await clearTestDb(pool); - const f = await factory(); + await using f = await factory(); const syncRuleContent = ` bucket_definitions: diff --git a/modules/module-postgres/test/src/util.ts b/modules/module-postgres/test/src/util.ts index 7499dfd1b..dca5521fa 100644 --- a/modules/module-postgres/test/src/util.ts +++ b/modules/module-postgres/test/src/util.ts @@ -1,10 +1,11 @@ import { PostgresRouteAPIAdapter } from '@module/api/PostgresRouteAPIAdapter.js'; import * as types from '@module/types/types.js'; -import * as pg_utils from '@module/utils/pgwire_utils.js'; +import * as lib_postgres from '@powersync/lib-service-postgres'; import { logger } from '@powersync/lib-services-framework'; import { BucketStorageFactory, OpId } from '@powersync/service-core'; import * as pgwire from '@powersync/service-jpgwire'; import * as mongo_storage from '@powersync/service-module-mongodb-storage'; +import * as postgres_storage from '@powersync/service-module-postgres-storage'; import { env } from './env.js'; export const TEST_URI = env.PG_TEST_URL; @@ -14,6 +15,10 @@ export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.MongoTestStorageF isCI: env.CI }); +export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.PostgresTestStorageFactoryGenerator({ + url: env.PG_STORAGE_TEST_URL +}); + export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({ type: 'postgresql', uri: TEST_URI, @@ -40,7 +45,7 @@ export async function clearTestDb(db: pgwire.PgClient) { for (let row of tableRows) { const name = row.table_name; if (name.startsWith('test_')) { - await db.query(`DROP TABLE public.${pg_utils.escapeIdentifier(name)}`); + await db.query(`DROP TABLE public.${lib_postgres.escapeIdentifier(name)}`); } } } diff --git a/modules/module-postgres/test/src/wal_stream.test.ts b/modules/module-postgres/test/src/wal_stream.test.ts index f444696b4..fea606643 100644 --- a/modules/module-postgres/test/src/wal_stream.test.ts +++ b/modules/module-postgres/test/src/wal_stream.test.ts @@ -4,7 +4,8 @@ import { putOp, removeOp } from '@powersync/service-core-tests'; import { pgwireRows } from '@powersync/service-jpgwire'; import * as crypto from 'crypto'; import { describe, expect, test } from 'vitest'; -import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; +import { env } from './env.js'; +import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js'; import { WalStreamTestContext } from './wal_stream_utils.js'; const BASIC_SYNC_RULES = ` @@ -14,10 +15,14 @@ bucket_definitions: - SELECT id, description FROM "test_data" `; -describe('wal stream - mongodb', { timeout: 20_000 }, function () { +describe.skipIf(!env.TEST_MONGO_STORAGE)('wal stream - mongodb', { timeout: 20_000 }, function () { defineWalStreamTests(INITIALIZED_MONGO_STORAGE_FACTORY); }); +describe.skipIf(!env.TEST_POSTGRES_STORAGE)('wal stream - postgres', { timeout: 20_000 }, function () { + defineWalStreamTests(INITIALIZED_POSTGRES_STORAGE_FACTORY); +}); + function defineWalStreamTests(factory: storage.TestStorageFactory) { test('replicating basic values', async () => { await using context = await WalStreamTestContext.open(factory); diff --git a/modules/module-postgres/test/src/wal_stream_utils.ts b/modules/module-postgres/test/src/wal_stream_utils.ts index ae549b70e..f25d6d083 100644 --- a/modules/module-postgres/test/src/wal_stream_utils.ts +++ b/modules/module-postgres/test/src/wal_stream_utils.ts @@ -46,6 +46,7 @@ export class WalStreamTestContext implements AsyncDisposable { await this.streamPromise; await this.connectionManager.destroy(); this.storage?.[Symbol.dispose](); + await this.factory?.[Symbol.asyncDispose](); } get pool() { diff --git a/modules/module-postgres/tsconfig.json b/modules/module-postgres/tsconfig.json index 9ceadec40..77a56fc5a 100644 --- a/modules/module-postgres/tsconfig.json +++ b/modules/module-postgres/tsconfig.json @@ -26,6 +26,9 @@ }, { "path": "../../libs/lib-services" + }, + { + "path": "../../libs/lib-postgres" } ] } diff --git a/package.json b/package.json index a0ff8792f..b737003f2 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,9 @@ "concurrently": "^8.2.2", "inquirer": "^9.2.7", "npm-check-updates": "^17.1.2", - "prettier": "^3.3.3", + "prettier": "^3.4.1", + "prettier-plugin-embed": "^0.4.15", + "prettier-plugin-sql": "^0.18.1", "rsocket-core": "1.0.0-alpha.3", "rsocket-websocket-client": "1.0.0-alpha.3", "semver": "^7.5.4", diff --git a/packages/service-core-tests/src/test-utils/metrics-utils.ts b/packages/service-core-tests/src/test-utils/metrics-utils.ts index 9fe704d78..8f61d250b 100644 --- a/packages/service-core-tests/src/test-utils/metrics-utils.ts +++ b/packages/service-core-tests/src/test-utils/metrics-utils.ts @@ -6,5 +6,9 @@ export const initMetrics = async () => { powersync_instance_id: 'test', internal_metrics_endpoint: 'unused.for.tests.com' }); + await resetMetrics(); +}; + +export const resetMetrics = async () => { Metrics.getInstance().resetCounters(); }; diff --git a/packages/service-core-tests/src/tests/register-compacting-tests.ts b/packages/service-core-tests/src/tests/register-compacting-tests.ts index e7ee55ab7..97c5ed1b1 100644 --- a/packages/service-core-tests/src/tests/register-compacting-tests.ts +++ b/packages/service-core-tests/src/tests/register-compacting-tests.ts @@ -26,7 +26,7 @@ bucket_definitions: data: [select * from test] `); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -128,7 +128,7 @@ bucket_definitions: data: [select * from test] `); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -238,7 +238,7 @@ bucket_definitions: data: [select * from test] `); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { diff --git a/packages/service-core-tests/src/tests/register-data-storage-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-tests.ts index f0d4d781f..01e337be7 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-tests.ts @@ -36,7 +36,7 @@ bucket_definitions: data: [] `); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -84,7 +84,7 @@ bucket_definitions: ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const result1 = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -208,7 +208,7 @@ bucket_definitions: ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -253,7 +253,7 @@ bucket_definitions: ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -299,7 +299,7 @@ bucket_definitions: - SELECT id, description FROM "%" ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -367,7 +367,7 @@ bucket_definitions: ); const sync_rules = sync_rules_content.parsed(test_utils.PARSE_OPTIONS).sync_rules; - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules_content); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -418,7 +418,7 @@ bucket_definitions: ); const sync_rules = sync_rules_content.parsed(test_utils.PARSE_OPTIONS).sync_rules; - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules_content); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -493,7 +493,7 @@ bucket_definitions: ); const sync_rules = sync_rules_content.parsed(test_utils.PARSE_OPTIONS).sync_rules; - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules_content); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -581,7 +581,7 @@ bucket_definitions: - SELECT client_id as id, description FROM "%" ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); @@ -647,7 +647,7 @@ bucket_definitions: - SELECT id, description FROM "%" ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -724,7 +724,7 @@ bucket_definitions: - SELECT id, description FROM "%" ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -846,7 +846,7 @@ bucket_definitions: ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -888,7 +888,7 @@ bucket_definitions: - SELECT id, description FROM "test" ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); // Pre-setup @@ -1046,7 +1046,7 @@ bucket_definitions: { name: 'description', type: 'VARCHAR', typeId: 25 } ]); } - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const sourceTable = test_utils.makeTestTable('test', ['id', 'description']); @@ -1154,7 +1154,7 @@ bucket_definitions: ]); } - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const sourceTable = test_utils.makeTestTable('test', ['id', 'description']); @@ -1252,7 +1252,7 @@ bucket_definitions: - SELECT id, description FROM "%" ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -1351,7 +1351,7 @@ bucket_definitions: - SELECT id, description FROM "%" ` ); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { @@ -1422,7 +1422,7 @@ bucket_definitions: data: [] `); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); let isDisposed = false; @@ -1461,7 +1461,7 @@ bucket_definitions: data: [] `); - using factory = await generateStorageFactory(); + await using factory = await generateStorageFactory(); const bucketStorage = factory.getInstance(sync_rules); let isDisposed = false; @@ -1494,7 +1494,7 @@ bucket_definitions: }); test('empty storage metrics', async () => { - using f = await generateStorageFactory({ dropAll: true }); + await using f = await generateStorageFactory({ dropAll: true }); const metrics = await f.getStorageMetrics(); expect(metrics).toEqual({ operations_size_bytes: 0, @@ -1507,11 +1507,7 @@ bucket_definitions: await storage.autoActivate(); const metrics2 = await f.getStorageMetrics(); - expect(metrics2).toEqual({ - operations_size_bytes: 0, - parameters_size_bytes: 0, - replication_size_bytes: 0 - }); + expect(metrics2).toMatchSnapshot(); }); test('invalidate cached parsed sync rules', async () => { @@ -1526,7 +1522,7 @@ bucket_definitions: ` ); - using bucketStorageFactory = await generateStorageFactory(); + await using bucketStorageFactory = await generateStorageFactory(); const syncBucketStorage = bucketStorageFactory.getInstance(sync_rules_content); const parsedSchema1 = syncBucketStorage.getParsedSyncRules({ diff --git a/packages/service-core-tests/src/tests/register-migration-tests.ts b/packages/service-core-tests/src/tests/register-migration-tests.ts new file mode 100644 index 000000000..e6ac8faa2 --- /dev/null +++ b/packages/service-core-tests/src/tests/register-migration-tests.ts @@ -0,0 +1,130 @@ +import { AbstractPowerSyncMigrationAgent, framework, PowerSyncMigrationManager } from '@powersync/service-core'; +import { expect, test, vi } from 'vitest'; + +const generateTestMigrations = (length: number, start: number = 0) => { + const results: string[] = []; + return { + results, + tests: Array.from({ length }).map((v, index) => { + const i = index + start; + return { + down: vi.fn(async () => { + results.push(`down - ${i}`); + }), + up: vi.fn(async () => { + results.push(`up - ${i}`); + }), + name: i.toString() + }; + }) + }; +}; + +/** + * Reset the factory as part of disposal. This helps cleanup after tests. + */ +const managedResetAgent = (factory: () => AbstractPowerSyncMigrationAgent) => { + const agent = factory(); + return { + agent, + // Reset the store for the next tests + [Symbol.asyncDispose]: () => agent.resetStore() + }; +}; + +export const registerMigrationTests = (migrationAgentFactory: () => AbstractPowerSyncMigrationAgent) => { + test('Should run migrations correctly', async () => { + await using manager = new framework.migrations.MigrationManager() as PowerSyncMigrationManager; + // Disposal is executed in reverse order. The store will be reset before the manager disposes it. + await using managedAgent = managedResetAgent(migrationAgentFactory); + + await managedAgent.agent.resetStore(); + + manager.registerMigrationAgent(managedAgent.agent); + + const length = 10; + const { tests, results } = generateTestMigrations(length); + manager.registerMigrations(tests); + + await manager.migrate({ + direction: framework.migrations.Direction.Up + }); + + const upLogs = Array.from({ length }).map((v, index) => `up - ${index}`); + + expect(results).deep.equals(upLogs); + + // Running up again should not run any migrations + await manager.migrate({ + direction: framework.migrations.Direction.Up + }); + + expect(results.length).equals(length); + + // Clear the results + results.splice(0, length); + + await manager.migrate({ + direction: framework.migrations.Direction.Down + }); + + const downLogs = Array.from({ length }) + .map((v, index) => `down - ${index}`) + .reverse(); + expect(results).deep.equals(downLogs); + + // Running down again should not run any additional migrations + await manager.migrate({ + direction: framework.migrations.Direction.Down + }); + + expect(results.length).equals(length); + + // Clear the results + results.splice(0, length); + + // Running up should run the up migrations again + await manager.migrate({ + direction: framework.migrations.Direction.Up + }); + + expect(results).deep.equals(upLogs); + }); + + test('Should run migrations with additions', async () => { + await using manager = new framework.migrations.MigrationManager() as PowerSyncMigrationManager; + // Disposal is executed in reverse order. The store will be reset before the manager disposes it. + await using managedAgent = managedResetAgent(migrationAgentFactory); + + await managedAgent.agent.resetStore(); + + manager.registerMigrationAgent(managedAgent.agent); + + const length = 10; + const { tests, results } = generateTestMigrations(length); + manager.registerMigrations(tests); + + await manager.migrate({ + direction: framework.migrations.Direction.Up + }); + + const upLogs = Array.from({ length }).map((v, index) => `up - ${index}`); + + expect(results).deep.equals(upLogs); + + // Add a new migration + const { results: newResults, tests: newTests } = generateTestMigrations(1, 10); + manager.registerMigrations(newTests); + + // Running up again should not run any migrations + await manager.migrate({ + direction: framework.migrations.Direction.Up + }); + + // The original tests should not have been executed again + expect(results.length).equals(length); + + // The new migration should have been executed + expect(newResults).deep.equals([`up - 10`]); + }); +}; diff --git a/packages/service-core-tests/src/tests/register-sync-tests.ts b/packages/service-core-tests/src/tests/register-sync-tests.ts index 9736f8cdb..2cf14cb54 100644 --- a/packages/service-core-tests/src/tests/register-sync-tests.ts +++ b/packages/service-core-tests/src/tests/register-sync-tests.ts @@ -33,7 +33,7 @@ export function registerSyncTests(factory: storage.TestStorageFactory) { const tracker = new sync.RequestTracker(); test('sync global data', async () => { - using f = await factory(); + await using f = await factory(); const syncRules = await f.updateSyncRules({ content: BASIC_SYNC_RULES @@ -128,7 +128,7 @@ export function registerSyncTests(factory: storage.TestStorageFactory) { }); test('expired token', async () => { - const f = await factory(); + await using f = await factory(); const syncRules = await f.updateSyncRules({ content: BASIC_SYNC_RULES @@ -155,7 +155,7 @@ export function registerSyncTests(factory: storage.TestStorageFactory) { }); test('sync updates to global data', async () => { - using f = await factory(); + await using f = await factory(); const syncRules = await f.updateSyncRules({ content: BASIC_SYNC_RULES @@ -216,7 +216,7 @@ export function registerSyncTests(factory: storage.TestStorageFactory) { }); test('expiring token', async () => { - using f = await factory(); + await using f = await factory(); const syncRules = await f.updateSyncRules({ content: BASIC_SYNC_RULES @@ -254,7 +254,7 @@ export function registerSyncTests(factory: storage.TestStorageFactory) { // This is expected to be rare in practice, but it is important to handle // this case correctly to maintain consistency on the client. - using f = await factory(); + await using f = await factory(); const syncRules = await f.updateSyncRules({ content: BASIC_SYNC_RULES @@ -391,7 +391,7 @@ export function registerSyncTests(factory: storage.TestStorageFactory) { }); test('write checkpoint', async () => { - using f = await factory(); + await using f = await factory(); const syncRules = await f.updateSyncRules({ content: BASIC_SYNC_RULES diff --git a/packages/service-core-tests/src/tests/tests-index.ts b/packages/service-core-tests/src/tests/tests-index.ts index 4f0e017fc..558f8c5a1 100644 --- a/packages/service-core-tests/src/tests/tests-index.ts +++ b/packages/service-core-tests/src/tests/tests-index.ts @@ -1,4 +1,5 @@ export * from './register-bucket-validation-tests.js'; export * from './register-compacting-tests.js'; export * from './register-data-storage-tests.js'; +export * from './register-migration-tests.js'; export * from './register-sync-tests.js'; diff --git a/packages/service-core/src/storage/BucketStorage.ts b/packages/service-core/src/storage/BucketStorage.ts index eaa212355..ff7b653f5 100644 --- a/packages/service-core/src/storage/BucketStorage.ts +++ b/packages/service-core/src/storage/BucketStorage.ts @@ -1,4 +1,8 @@ -import { DisposableListener, DisposableObserverClient } from '@powersync/lib-services-framework'; +import { + AsyncDisposableObserverClient, + DisposableListener, + DisposableObserverClient +} from '@powersync/lib-services-framework'; import { EvaluatedParameters, EvaluatedRow, @@ -56,7 +60,7 @@ export interface BucketStorageFactoryListener extends DisposableListener { replicationEvent: (event: ReplicationEventPayload) => void; } -export interface BucketStorageFactory extends DisposableObserverClient { +export interface BucketStorageFactory extends AsyncDisposableObserverClient { /** * Update sync rules from configuration, if changed. */ diff --git a/packages/service-core/src/util/utils.ts b/packages/service-core/src/util/utils.ts index b34cf7491..673ad2091 100644 --- a/packages/service-core/src/util/utils.ts +++ b/packages/service-core/src/util/utils.ts @@ -222,3 +222,29 @@ export function flatstr(s: string) { function rowKey(entry: OplogEntry) { return `${entry.object_type}/${entry.object_id}/${entry.subkey}`; } + +/** + * Estimate in-memory size of row. + */ +export function estimateRowSize(record: sync_rules.ToastableSqliteRow | undefined) { + if (record == null) { + return 12; + } + let size = 0; + for (let [key, value] of Object.entries(record)) { + size += 12 + key.length; + // number | string | null | bigint | Uint8Array + if (value == null) { + size += 4; + } else if (typeof value == 'number') { + size += 8; + } else if (typeof value == 'bigint') { + size += 8; + } else if (typeof value == 'string') { + size += value.length; + } else if (value instanceof Uint8Array) { + size += value.byteLength; + } + } + return size; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 37d5d994b..63a93834a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -30,8 +30,14 @@ importers: specifier: ^17.1.2 version: 17.1.3 prettier: - specifier: ^3.3.3 - version: 3.3.3 + specifier: ^3.4.1 + version: 3.4.2 + prettier-plugin-embed: + specifier: ^0.4.15 + version: 0.4.15 + prettier-plugin-sql: + specifier: ^0.18.1 + version: 0.18.1(prettier@3.4.2) rsocket-core: specifier: 1.0.0-alpha.3 version: 1.0.0-alpha.3 @@ -78,6 +84,37 @@ importers: specifier: ^4.4.1 version: 4.4.1 + libs/lib-postgres: + dependencies: + '@powersync/lib-services-framework': + specifier: workspace:* + version: link:../lib-services + '@powersync/service-jpgwire': + specifier: workspace:* + version: link:../../packages/jpgwire + '@powersync/service-sync-rules': + specifier: workspace:* + version: link:../../packages/sync-rules + '@powersync/service-types': + specifier: workspace:* + version: link:../../packages/types + p-defer: + specifier: ^4.0.1 + version: 4.0.1 + ts-codec: + specifier: ^1.3.0 + version: 1.3.0 + uri-js: + specifier: ^4.4.1 + version: 4.4.1 + uuid: + specifier: ^9.0.1 + version: 9.0.1 + devDependencies: + '@types/uuid': + specifier: ^9.0.4 + version: 9.0.8 + libs/lib-services: dependencies: ajv: @@ -154,6 +191,9 @@ importers: '@powersync/service-module-mongodb-storage': specifier: workspace:* version: link:../module-mongodb-storage + '@powersync/service-module-postgres-storage': + specifier: workspace:* + version: link:../module-postgres-storage '@types/uuid': specifier: ^9.0.4 version: 9.0.8 @@ -246,6 +286,9 @@ importers: '@powersync/service-module-mongodb-storage': specifier: workspace:* version: link:../module-mongodb-storage + '@powersync/service-module-postgres-storage': + specifier: workspace:* + version: link:../module-postgres-storage '@types/async': specifier: ^3.2.24 version: 3.2.24 @@ -258,6 +301,9 @@ importers: modules/module-postgres: dependencies: + '@powersync/lib-service-postgres': + specifier: workspace:* + version: link:../../libs/lib-postgres '@powersync/lib-services-framework': specifier: workspace:* version: link:../../libs/lib-services @@ -298,9 +344,61 @@ importers: '@powersync/service-module-mongodb-storage': specifier: workspace:* version: link:../module-mongodb-storage + '@powersync/service-module-postgres-storage': + specifier: workspace:* + version: link:../module-postgres-storage + '@types/uuid': + specifier: ^9.0.4 + version: 9.0.8 + + modules/module-postgres-storage: + dependencies: + '@powersync/lib-service-postgres': + specifier: workspace:* + version: link:../../libs/lib-postgres + '@powersync/lib-services-framework': + specifier: workspace:* + version: link:../../libs/lib-services + '@powersync/service-core': + specifier: workspace:* + version: link:../../packages/service-core + '@powersync/service-core-tests': + specifier: workspace:* + version: link:../../packages/service-core-tests + '@powersync/service-jpgwire': + specifier: workspace:* + version: link:../../packages/jpgwire + '@powersync/service-jsonbig': + specifier: ^0.17.10 + version: 0.17.10 + '@powersync/service-sync-rules': + specifier: workspace:* + version: link:../../packages/sync-rules + '@powersync/service-types': + specifier: workspace:* + version: link:../../packages/types + ix: + specifier: ^5.0.0 + version: 5.0.0 + lru-cache: + specifier: ^10.2.2 + version: 10.4.3 + p-defer: + specifier: ^4.0.1 + version: 4.0.1 + ts-codec: + specifier: ^1.3.0 + version: 1.3.0 + uuid: + specifier: ^9.0.1 + version: 9.0.1 + devDependencies: '@types/uuid': specifier: ^9.0.4 version: 9.0.8 + typescript: + specifier: ^5.2.2 + version: 5.6.2 packages/jpgwire: dependencies: @@ -557,6 +655,9 @@ importers: '@powersync/service-module-postgres': specifier: workspace:* version: link:../modules/module-postgres + '@powersync/service-module-postgres-storage': + specifier: workspace:* + version: link:../modules/module-postgres-storage '@powersync/service-rsocket-router': specifier: workspace:* version: link:../packages/rsocket-router @@ -1250,6 +1351,9 @@ packages: resolution: {integrity: sha512-2GjOxVws+wtbb+xFUJe4Ozzkp/f0Gsna0fje9art76bmz6yfLCW4K3Mf2/M310xMnAIp8eP9hsJ6DYwwZCo1RA==} engines: {node: '>=20.0.0'} + '@powersync/service-jsonbig@0.17.10': + resolution: {integrity: sha512-BgxgUewuw4HFCM9MzuzlIuRKHya6rimNPYqUItt7CO3ySUeUnX8Qn9eZpMxu9AT5Y8zqkSyxvduY36zZueNojg==} + '@prisma/instrumentation@5.16.1': resolution: {integrity: sha512-4m5gRFWnQb8s/yTyGbMZkL7A5uJgqOWcWJxapwcAD0T0kh5sGPEVSQl/zTQvE9aduXhFAxOtC3gO+R8Hb5xO1Q==} @@ -1918,6 +2022,14 @@ packages: resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} engines: {node: '>=10'} + dedent@1.5.3: + resolution: {integrity: sha512-NHQtfOOW68WD8lgypbLA5oT+Bt0xXJhiYvoR6SmmNXZfpzOGXwdKWmcwG8N7PwVVWV3eF/68nmD9BaJSsTBhyQ==} + peerDependencies: + babel-plugin-macros: ^3.1.0 + peerDependenciesMeta: + babel-plugin-macros: + optional: true + deep-eql@5.0.2: resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} engines: {node: '>=6'} @@ -2094,6 +2206,10 @@ packages: resolution: {integrity: sha512-+SrHpvQ52Q6W9f3wJoJBbAQULJuNEEQwBvlvYwACDhBTLOTMiQ0HYWh4+vC3OivGP2ENcTI1oKlFA2OepJNjhQ==} engines: {node: '>=14'} + find-up-simple@1.0.0: + resolution: {integrity: sha512-q7Us7kcjj2VMePAa02hDAF6d+MzsdsAWEwYyOpwUtlerRBkOEPBCRZrAV4XfcSN8fHAgaD0hP7miwoay6DCprw==} + engines: {node: '>=18'} + find-up@4.1.0: resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==} engines: {node: '>=8'} @@ -2491,6 +2607,10 @@ packages: resolution: {integrity: sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==} engines: {node: '>=0.10.0'} + jsox@1.2.121: + resolution: {integrity: sha512-9Ag50tKhpTwS6r5wh3MJSAvpSof0UBr39Pto8OnzFT32Z/pAbxAsKHzyvsyMEHVslELvHyO/4/jaQELHk8wDcw==} + hasBin: true + keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -2588,6 +2708,9 @@ packages: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} + micro-memoize@4.1.3: + resolution: {integrity: sha512-DzRMi8smUZXT7rCGikRwldEh6eO6qzKiPPopcr1+2EY3AYKpy5fu159PKWwIS9A6IWnrvPKDMcuFtyrroZa8Bw==} + micromatch@4.0.7: resolution: {integrity: sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==} engines: {node: '>=8.6'} @@ -2756,6 +2879,10 @@ packages: engines: {node: ^12.13 || ^14.13 || >=16} hasBin: true + node-sql-parser@4.18.0: + resolution: {integrity: sha512-2YEOR5qlI1zUFbGMLKNfsrR5JUvFg9LxIRVE+xJe962pfVLH0rnItqLzv96XVs1Y1UIR8FxsXAuvX/lYAWZ2BQ==} + engines: {node: '>=8'} + nodemon@3.1.4: resolution: {integrity: sha512-wjPBbFhtpJwmIeY2yP7QF+UKzPfltVGtfce1g/bB15/8vCGZj8uxD62b/b9M9/WVgme0NZudpownKN+c0plXlQ==} engines: {node: '>=10'} @@ -2866,6 +2993,10 @@ packages: resolution: {integrity: sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==} engines: {node: '>=12.20'} + p-defer@4.0.1: + resolution: {integrity: sha512-Mr5KC5efvAK5VUptYEIopP1bakB85k2IWXaRC0rsh1uwn1L6M0LVml8OIQ4Gudg4oyZakf7FmeRLkMMtZW1i5A==} + engines: {node: '>=12'} + p-filter@2.1.0: resolution: {integrity: sha512-ZBxxZ5sL2HghephhpGAQdoskxplTwr7ICaehZwLIlfL6acuVgZPm8yBNuRAFBGEqtD/hmUeq9eqLg2ys9Xr/yw==} engines: {node: '>=8'} @@ -2908,6 +3039,10 @@ packages: package-manager-detector@0.2.0: resolution: {integrity: sha512-E385OSk9qDcXhcM9LNSe4sdhx8a9mAPrZ4sMLW+tmxl5ZuGtPUcdFu+MPP2jbgiWAZ6Pfe5soGFMd+0Db5Vrog==} + package-up@5.0.0: + resolution: {integrity: sha512-MQEgDUvXCa3sGvqHg3pzHO8e9gqTCMPVrWUko3vPQGntwegmFo52mZb2abIVTjFnUcW0BcPz0D93jV5Cas1DWA==} + engines: {node: '>=18'} + pacote@15.2.0: resolution: {integrity: sha512-rJVZeIwHTUta23sIZgEIM62WYwbmGbThdbnkt81ravBplQv+HjyroqnLRNH2+sLJHcGZmLRmhPwACqhfTcOmnA==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -3014,13 +3149,22 @@ packages: resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} engines: {node: '>=0.10.0'} + prettier-plugin-embed@0.4.15: + resolution: {integrity: sha512-9pZVIp3bw2jw+Ge+iAMZ4j+sIVC9cPruZ93H2tj5Wa/3YDFDJ/uYyVWdUGfcFUnv28drhW2Bmome9xSGXsPKOw==} + + prettier-plugin-sql@0.18.1: + resolution: {integrity: sha512-2+Nob2sg7hzLAKJoE6sfgtkhBZCqOzrWHZPvE4Kee/e80oOyI4qwy9vypeltqNBJwTtq3uiKPrCxlT03bBpOaw==} + engines: {node: ^14.18.0 || >=16.0.0} + peerDependencies: + prettier: ^3.0.3 + prettier@2.8.8: resolution: {integrity: sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==} engines: {node: '>=10.13.0'} hasBin: true - prettier@3.3.3: - resolution: {integrity: sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==} + prettier@3.4.2: + resolution: {integrity: sha512-e9MewbtFo+Fevyuxn/4rrcDAaq0IYxPGLvObpQjiZBMAzB9IGmzlnG9RZy3FFas+eBMu2vA0CszMeduow5dIuQ==} engines: {node: '>=14'} hasBin: true @@ -3405,6 +3549,10 @@ packages: sprintf-js@1.1.3: resolution: {integrity: sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==} + sql-formatter@15.4.9: + resolution: {integrity: sha512-5vmt2HlCAVozxsBZuXWkAki/KGawaK+b5GG5x+BtXOFVpN/8cqppblFUxHl4jxdA0cvo14lABhM+KBnrUapOlw==} + hasBin: true + sqlstring@2.3.3: resolution: {integrity: sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==} engines: {node: '>= 0.6'} @@ -3506,6 +3654,9 @@ packages: through@2.3.8: resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==} + tiny-jsonc@1.0.1: + resolution: {integrity: sha512-ik6BCxzva9DoiEfDX/li0L2cWKPPENYvixUprFdl3YPi4bZZUhDnNI9YUkacrv+uIG90dnxR5mNqaoD6UhD6Bw==} + tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} @@ -3619,6 +3770,10 @@ packages: resolution: {integrity: sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==} engines: {node: '>=12.20'} + type-fest@4.31.0: + resolution: {integrity: sha512-yCxltHW07Nkhv/1F6wWBr8kz+5BGMfP+RbRSYFnegVb0qV/UMT0G0ElBloPVerqn4M2ZV80Ir1FtCcYv1cT6vQ==} + engines: {node: '>=16'} + typedarray-to-buffer@3.1.5: resolution: {integrity: sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==} @@ -4653,6 +4808,10 @@ snapshots: big-integer: 1.6.51 iconv-lite: 0.6.3 + '@powersync/service-jsonbig@0.17.10': + dependencies: + lossless-json: 2.0.11 + '@prisma/instrumentation@5.16.1': dependencies: '@opentelemetry/api': 1.8.0 @@ -4740,7 +4899,7 @@ snapshots: '@opentelemetry/semantic-conventions': 1.25.1 '@prisma/instrumentation': 5.16.1 '@sentry/core': 8.17.0 - '@sentry/opentelemetry': 8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.6.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/semantic-conventions@1.25.1) + '@sentry/opentelemetry': 8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.1) '@sentry/types': 8.17.0 '@sentry/utils': 8.17.0 optionalDependencies: @@ -4748,7 +4907,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@sentry/opentelemetry@8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.6.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/semantic-conventions@1.25.1)': + '@sentry/opentelemetry@8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.1)': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.9.0) @@ -5355,6 +5514,8 @@ snapshots: dependencies: mimic-response: 3.1.0 + dedent@1.5.3: {} + deep-eql@5.0.2: {} deep-extend@0.6.0: {} @@ -5551,6 +5712,8 @@ snapshots: fast-querystring: 1.1.2 safe-regex2: 2.0.0 + find-up-simple@1.0.0: {} + find-up@4.1.0: dependencies: locate-path: 5.0.0 @@ -5934,6 +6097,8 @@ snapshots: jsonpointer@5.0.1: {} + jsox@1.2.121: {} + keyv@4.5.4: dependencies: json-buffer: 3.0.1 @@ -6057,6 +6222,8 @@ snapshots: merge2@1.4.1: {} + micro-memoize@4.1.3: {} + micromatch@4.0.7: dependencies: braces: 3.0.3 @@ -6217,6 +6384,10 @@ snapshots: - bluebird - supports-color + node-sql-parser@4.18.0: + dependencies: + big-integer: 1.6.51 + nodemon@3.1.4: dependencies: chokidar: 3.6.0 @@ -6383,6 +6554,8 @@ snapshots: p-cancelable@3.0.0: {} + p-defer@4.0.1: {} + p-filter@2.1.0: dependencies: p-map: 2.1.0 @@ -6422,6 +6595,10 @@ snapshots: package-manager-detector@0.2.0: {} + package-up@5.0.0: + dependencies: + find-up-simple: 1.0.0 + pacote@15.2.0: dependencies: '@npmcli/git': 4.1.0 @@ -6535,9 +6712,28 @@ snapshots: dependencies: xtend: 4.0.2 + prettier-plugin-embed@0.4.15: + dependencies: + '@types/estree': 1.0.5 + dedent: 1.5.3 + micro-memoize: 4.1.3 + package-up: 5.0.0 + tiny-jsonc: 1.0.1 + type-fest: 4.31.0 + transitivePeerDependencies: + - babel-plugin-macros + + prettier-plugin-sql@0.18.1(prettier@3.4.2): + dependencies: + jsox: 1.2.121 + node-sql-parser: 4.18.0 + prettier: 3.4.2 + sql-formatter: 15.4.9 + tslib: 2.6.3 + prettier@2.8.8: {} - prettier@3.3.3: {} + prettier@3.4.2: {} proc-log@3.0.0: {} @@ -6921,6 +7117,12 @@ snapshots: sprintf-js@1.1.3: {} + sql-formatter@15.4.9: + dependencies: + argparse: 2.0.1 + get-stdin: 8.0.0 + nearley: 2.20.1 + sqlstring@2.3.3: {} ssri@10.0.6: @@ -7017,6 +7219,8 @@ snapshots: through@2.3.8: {} + tiny-jsonc@1.0.1: {} + tinybench@2.9.0: {} tinyexec@0.3.0: {} @@ -7120,6 +7324,8 @@ snapshots: type-fest@2.19.0: {} + type-fest@4.31.0: {} + typedarray-to-buffer@3.1.5: dependencies: is-typedarray: 1.0.0 diff --git a/service/Dockerfile b/service/Dockerfile index 4acd8d355..ea01a10fe 100644 --- a/service/Dockerfile +++ b/service/Dockerfile @@ -15,8 +15,10 @@ COPY packages/types/package.json packages/types/tsconfig.json packages/types/ COPY libs/lib-services/package.json libs/lib-services/tsconfig.json libs/lib-services/ COPY libs/lib-mongodb/package.json libs/lib-mongodb/tsconfig.json libs/lib-mongodb/ +COPY libs/lib-postgres/package.json libs/lib-postgres/tsconfig.json libs/lib-postgres/ COPY modules/module-postgres/package.json modules/module-postgres/tsconfig.json modules/module-postgres/ +COPY modules/module-postgres-storage/package.json modules/module-postgres-storage/tsconfig.json modules/module-postgres-storage/ COPY modules/module-mongodb/package.json modules/module-mongodb/tsconfig.json modules/module-mongodb/ COPY modules/module-mongodb-storage/package.json modules/module-mongodb-storage/tsconfig.json modules/module-mongodb-storage/ COPY modules/module-mysql/package.json modules/module-mysql/tsconfig.json modules/module-mysql/ @@ -36,8 +38,10 @@ COPY packages/types/src packages/types/src/ COPY libs/lib-services/src libs/lib-services/src/ COPY libs/lib-mongodb/src libs/lib-mongodb/src/ +COPY libs/lib-postgres/src libs/lib-postgres/src/ COPY modules/module-postgres/src modules/module-postgres/src/ +COPY modules/module-postgres-storage/src modules/module-postgres-storage/src/ COPY modules/module-mongodb/src modules/module-mongodb/src/ COPY modules/module-mongodb-storage/src modules/module-mongodb-storage/src/ COPY modules/module-mysql/src modules/module-mysql/src/ diff --git a/service/package.json b/service/package.json index 487742c11..23a27a1b1 100644 --- a/service/package.json +++ b/service/package.json @@ -17,6 +17,7 @@ "@powersync/service-core": "workspace:*", "@powersync/lib-services-framework": "workspace:*", "@powersync/service-module-postgres": "workspace:*", + "@powersync/service-module-postgres-storage": "workspace:*", "@powersync/service-module-mongodb": "workspace:*", "@powersync/service-module-mongodb-storage": "workspace:*", "@powersync/service-module-mysql": "workspace:*", diff --git a/service/src/entry.ts b/service/src/entry.ts index e133c04e4..06a062a5b 100644 --- a/service/src/entry.ts +++ b/service/src/entry.ts @@ -5,6 +5,7 @@ import { MongoModule } from '@powersync/service-module-mongodb'; import { MongoStorageModule } from '@powersync/service-module-mongodb-storage'; import { MySQLModule } from '@powersync/service-module-mysql'; import { PostgresModule } from '@powersync/service-module-postgres'; +import { PostgresStorageModule } from '@powersync/service-module-postgres-storage'; import { startServer } from './runners/server.js'; import { startStreamRunner } from './runners/stream-worker.js'; import { startUnifiedRunner } from './runners/unified-runner.js'; @@ -15,7 +16,13 @@ container.registerDefaults(); container.register(ContainerImplementation.REPORTER, createSentryReporter()); const moduleManager = new core.modules.ModuleManager(); -moduleManager.register([new PostgresModule(), new MySQLModule(), new MongoModule(), new MongoStorageModule()]); +moduleManager.register([ + new PostgresModule(), + new MySQLModule(), + new MongoModule(), + new MongoStorageModule(), + new PostgresStorageModule() +]); // This is a bit of a hack. Commands such as the teardown command or even migrations might // want access to the ModuleManager in order to use modules container.register(core.ModuleManager, moduleManager); diff --git a/tsconfig.json b/tsconfig.json index 4eaac0158..fc9861cc0 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -19,6 +19,9 @@ { "path": "./modules/module-postgres" }, + { + "path": "./modules/module-postgres-storage" + }, { "path": "./modules/module-mysql" }, @@ -34,6 +37,9 @@ { "path": "./libs/lib-mongodb" }, + { + "path": "./libs/lib-postgres" + }, { "path": "./packages/types" },