From 04cc1bd399a5d22c568586e03e7176d6c35d4f05 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 10 Sep 2024 18:05:25 +0200 Subject: [PATCH 01/35] Basic MongoDB replication structure. --- modules/module-mongodb/CHANGELOG.md | 1 + modules/module-mongodb/LICENSE | 67 ++++ modules/module-mongodb/README.md | 3 + modules/module-mongodb/package.json | 47 +++ .../src/api/MongoRouteAPIAdapter.ts | 75 +++++ modules/module-mongodb/src/index.ts | 5 + .../module-mongodb/src/module/MongoModule.ts | 52 +++ .../src/replication/ChangeStream.ts | 304 ++++++++++++++++++ .../replication/ChangeStreamReplicationJob.ts | 109 +++++++ .../src/replication/ChangeStreamReplicator.ts | 34 ++ .../replication/ConnectionManagerFactory.ts | 27 ++ .../src/replication/MongoErrorRateLimiter.ts | 45 +++ .../src/replication/MongoManager.ts | 47 +++ .../src/replication/MongoRelation.ts | 104 ++++++ .../src/replication/replication-index.ts | 4 + modules/module-mongodb/src/types/types.ts | 143 ++++++++ modules/module-mongodb/test/tsconfig.json | 28 ++ modules/module-mongodb/tsconfig.json | 28 ++ modules/module-mongodb/vitest.config.ts | 9 + pnpm-lock.yaml | 46 +++ service/package.json | 1 + service/src/entry.ts | 3 +- service/tsconfig.json | 3 + 23 files changed, 1184 insertions(+), 1 deletion(-) create mode 100644 modules/module-mongodb/CHANGELOG.md create mode 100644 modules/module-mongodb/LICENSE create mode 100644 modules/module-mongodb/README.md create mode 100644 modules/module-mongodb/package.json create mode 100644 modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts create mode 100644 modules/module-mongodb/src/index.ts create mode 100644 modules/module-mongodb/src/module/MongoModule.ts create mode 100644 modules/module-mongodb/src/replication/ChangeStream.ts create mode 100644 modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts create mode 100644 modules/module-mongodb/src/replication/ChangeStreamReplicator.ts create mode 100644 modules/module-mongodb/src/replication/ConnectionManagerFactory.ts create mode 100644 modules/module-mongodb/src/replication/MongoErrorRateLimiter.ts create mode 100644 modules/module-mongodb/src/replication/MongoManager.ts create mode 100644 modules/module-mongodb/src/replication/MongoRelation.ts create mode 100644 modules/module-mongodb/src/replication/replication-index.ts create mode 100644 modules/module-mongodb/src/types/types.ts create mode 100644 modules/module-mongodb/test/tsconfig.json create mode 100644 modules/module-mongodb/tsconfig.json create mode 100644 modules/module-mongodb/vitest.config.ts diff --git a/modules/module-mongodb/CHANGELOG.md b/modules/module-mongodb/CHANGELOG.md new file mode 100644 index 000000000..05f7d8b81 --- /dev/null +++ b/modules/module-mongodb/CHANGELOG.md @@ -0,0 +1 @@ +# @powersync/service-module-mongodb diff --git a/modules/module-mongodb/LICENSE b/modules/module-mongodb/LICENSE new file mode 100644 index 000000000..c8efd46cc --- /dev/null +++ b/modules/module-mongodb/LICENSE @@ -0,0 +1,67 @@ +# Functional Source License, Version 1.1, Apache 2.0 Future License + +## Abbreviation + +FSL-1.1-Apache-2.0 + +## Notice + +Copyright 2023-2024 Journey Mobile, Inc. + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under these Terms and Conditions, as indicated by our inclusion of these Terms and Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, Redistribution and Trademark clauses below, we hereby grant you the right to use, copy, modify, create derivative works, publicly perform, publicly display and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use means making the Software available to others in a commercial product or service that: + +1. substitutes for the Software; +2. substitutes for any other product or service we offer using the Software that exists as of the date we make the Software available; or +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; +2. for non-commercial education; +3. for non-commercial research; and +4. in connection with professional services that you provide to a licensee using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our patents, the license grant above includes a license under our patents. If you make a claim against any party that the Software infringes or contributes to the infringement of any patent, then your patent license to the Software ends immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of the Software. +If you redistribute any copies, modifications or derivatives of the Software, you must include a copy of or a link to these Terms and Conditions and not remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of the Software, you have no right under these Terms and Conditions to use our trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under the Apache License, Version 2.0 that is effective on the second anniversary of the date we make the Software available. On or after that date, you may use the Software under the Apache License, Version 2.0, in which case the following will apply: + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/modules/module-mongodb/README.md b/modules/module-mongodb/README.md new file mode 100644 index 000000000..f9e9e4c64 --- /dev/null +++ b/modules/module-mongodb/README.md @@ -0,0 +1,3 @@ +# PowerSync Service Module MongoDB + +MongoDB replication module for PowerSync diff --git a/modules/module-mongodb/package.json b/modules/module-mongodb/package.json new file mode 100644 index 000000000..dfcd52e0e --- /dev/null +++ b/modules/module-mongodb/package.json @@ -0,0 +1,47 @@ +{ + "name": "@powersync/service-module-mongodb", + "repository": "https://github.com/powersync-ja/powersync-service", + "types": "dist/index.d.ts", + "publishConfig": { + "access": "restricted" + }, + "version": "0.0.1", + "main": "dist/index.js", + "license": "FSL-1.1-Apache-2.0", + "type": "module", + "scripts": { + "build": "tsc -b", + "build:tests": "tsc -b test/tsconfig.json", + "clean": "rm -rf ./lib && tsc -b --clean", + "test": "vitest --no-threads" + }, + "exports": { + ".": { + "import": "./dist/index.js", + "require": "./dist/index.js", + "default": "./dist/index.js" + }, + "./types": { + "import": "./dist/types/types.js", + "require": "./dist/types/types.js", + "default": "./dist/types/types.js" + } + }, + "dependencies": { + "@powersync/lib-services-framework": "workspace:*", + "@powersync/service-core": "workspace:*", + "@powersync/service-jsonbig": "workspace:*", + "@powersync/service-sync-rules": "workspace:*", + "@powersync/service-types": "workspace:*", + "mongodb": "^6.7.0", + "ts-codec": "^1.2.2", + "uuid": "^9.0.1", + "uri-js": "^4.4.1" + }, + "devDependencies": { + "@types/uuid": "^9.0.4", + "typescript": "^5.2.2", + "vitest": "^0.34.6", + "vite-tsconfig-paths": "^4.3.2" + } +} diff --git a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts new file mode 100644 index 000000000..61aed14f1 --- /dev/null +++ b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts @@ -0,0 +1,75 @@ +import { api } from '@powersync/service-core'; +import * as mongo from 'mongodb'; + +import * as sync_rules from '@powersync/service-sync-rules'; +import * as service_types from '@powersync/service-types'; +import * as types from '../types/types.js'; +import { MongoManager } from '../replication/MongoManager.js'; + +export class MongoRouteAPIAdapter implements api.RouteAPI { + protected client: mongo.MongoClient; + + connectionTag: string; + + constructor(protected config: types.ResolvedConnectionConfig) { + this.client = new MongoManager(config).client; + this.connectionTag = config.tag ?? sync_rules.DEFAULT_TAG; + } + + async shutdown(): Promise { + await this.client.close(); + } + + async getSourceConfig(): Promise { + return this.config; + } + + async getConnectionStatus(): Promise { + // TODO: Implement + const base = { + id: this.config.id, + uri: types.baseUri(this.config) + }; + return { + ...base, + connected: true, + errors: [] + }; + } + + async executeQuery(query: string, params: any[]): Promise { + return service_types.internal_routes.ExecuteSqlResponse.encode({ + results: { + columns: [], + rows: [] + }, + success: false, + error: 'SQL querying is not supported for MongoDB' + }); + } + + async getDebugTablesInfo( + tablePatterns: sync_rules.TablePattern[], + sqlSyncRules: sync_rules.SqlSyncRules + ): Promise { + // TODO: Implement + return []; + } + + async getReplicationLag(syncRulesId: string): Promise { + // TODO: Implement + + return 0; + } + + async getReplicationHead(): Promise { + // TODO: implement + return ''; + } + + async getConnectionSchema(): Promise { + // TODO: Implement + + return []; + } +} diff --git a/modules/module-mongodb/src/index.ts b/modules/module-mongodb/src/index.ts new file mode 100644 index 000000000..4cfc25695 --- /dev/null +++ b/modules/module-mongodb/src/index.ts @@ -0,0 +1,5 @@ +import { MongoModule } from './module/MongoModule.js'; + +export const module = new MongoModule(); + +export default module; diff --git a/modules/module-mongodb/src/module/MongoModule.ts b/modules/module-mongodb/src/module/MongoModule.ts new file mode 100644 index 000000000..3f6e27636 --- /dev/null +++ b/modules/module-mongodb/src/module/MongoModule.ts @@ -0,0 +1,52 @@ +import { api, ConfigurationFileSyncRulesProvider, replication, system, TearDownOptions } from '@powersync/service-core'; +import { MongoRouteAPIAdapter } from '../api/MongoRouteAPIAdapter.js'; +import { ConnectionManagerFactory } from '../replication/ConnectionManagerFactory.js'; +import { MongoErrorRateLimiter } from '../replication/MongoErrorRateLimiter.js'; +import { ChangeStreamReplicator } from '../replication/ChangeStreamReplicator.js'; +import * as types from '../types/types.js'; + +export class MongoModule extends replication.ReplicationModule { + constructor() { + super({ + name: 'MongoDB', + type: types.MONGO_CONNECTION_TYPE, + configSchema: types.MongoConnectionConfig + }); + } + + async initialize(context: system.ServiceContextContainer): Promise { + await super.initialize(context); + } + + protected createRouteAPIAdapter(): api.RouteAPI { + return new MongoRouteAPIAdapter(this.resolveConfig(this.decodedConfig!)); + } + + protected createReplicator(context: system.ServiceContext): replication.AbstractReplicator { + const normalisedConfig = this.resolveConfig(this.decodedConfig!); + const syncRuleProvider = new ConfigurationFileSyncRulesProvider(context.configuration.sync_rules); + const connectionFactory = new ConnectionManagerFactory(normalisedConfig); + + return new ChangeStreamReplicator({ + id: this.getDefaultId(normalisedConfig.database ?? ''), + syncRuleProvider: syncRuleProvider, + storageEngine: context.storageEngine, + connectionFactory: connectionFactory, + rateLimiter: new MongoErrorRateLimiter() + }); + } + + /** + * Combines base config with normalized connection settings + */ + private resolveConfig(config: types.MongoConnectionConfig): types.ResolvedConnectionConfig { + return { + ...config, + ...types.normalizeConnectionConfig(config) + }; + } + + async teardown(options: TearDownOptions): Promise { + // TODO: Implement? + } +} diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts new file mode 100644 index 000000000..8ad5759df --- /dev/null +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -0,0 +1,304 @@ +import { container, logger } from '@powersync/lib-services-framework'; +import { Metrics, SourceEntityDescriptor, storage } from '@powersync/service-core'; +import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern, toSyncRulesRow } from '@powersync/service-sync-rules'; +import * as mongo from 'mongodb'; +import { MongoManager } from './MongoManager.js'; +import { constructAfterRecord, getMongoLsn, getMongoRelation } from './MongoRelation.js'; + +export const ZERO_LSN = '00000000'; + +export interface WalStreamOptions { + connections: MongoManager; + storage: storage.SyncRulesBucketStorage; + abort_signal: AbortSignal; +} + +interface InitResult { + needsInitialSync: boolean; +} + +export class MissingReplicationSlotError extends Error { + constructor(message: string) { + super(message); + } +} + +export class ChangeStream { + sync_rules: SqlSyncRules; + group_id: number; + + connection_id = 1; + + private readonly storage: storage.SyncRulesBucketStorage; + + private connections: MongoManager; + private readonly client: mongo.MongoClient; + + private abort_signal: AbortSignal; + + private relation_cache = new Map(); + + constructor(options: WalStreamOptions) { + this.storage = options.storage; + this.sync_rules = options.storage.sync_rules; + this.group_id = options.storage.group_id; + this.connections = options.connections; + this.client = this.connections.client; + + this.abort_signal = options.abort_signal; + this.abort_signal.addEventListener( + 'abort', + () => { + // TODO: Fast abort? + }, + { once: true } + ); + } + + get stopped() { + return this.abort_signal.aborted; + } + + async getQualifiedTableNames( + batch: storage.BucketStorageBatch, + tablePattern: TablePattern + ): Promise { + const schema = tablePattern.schema; + if (tablePattern.connectionTag != this.connections.connectionTag) { + return []; + } + + const prefix = tablePattern.isWildcard ? tablePattern.tablePrefix : undefined; + if (tablePattern.isWildcard) { + // TODO: Implement + throw new Error('not supported yet'); + } + let result: storage.SourceTable[] = []; + + const name = tablePattern.name; + + const table = await this.handleRelation( + batch, + { + name, + schema, + objectId: name, + replicationColumns: [{ name: '_id' }] + } as SourceEntityDescriptor, + false + ); + + result.push(table); + return result; + } + + async initSlot(): Promise { + const status = await this.storage.getStatus(); + if (status.snapshot_done && status.checkpoint_lsn) { + logger.info(`Initial replication already done`); + return { needsInitialSync: false }; + } + + return { needsInitialSync: true }; + } + + async estimatedCount(table: storage.SourceTable): Promise { + const db = this.client.db(table.schema); + const count = db.collection(table.table).estimatedDocumentCount(); + return `~${count}`; + } + + /** + * Start initial replication. + * + * If (partial) replication was done before on this slot, this clears the state + * and starts again from scratch. + */ + async startInitialReplication() { + await this.storage.clear(); + await this.initialReplication(); + } + + async initialReplication() { + const sourceTables = this.sync_rules.getSourceTables(); + await this.storage.startBatch({ zeroLSN: ZERO_LSN }, async (batch) => { + for (let tablePattern of sourceTables) { + const tables = await this.getQualifiedTableNames(batch, tablePattern); + for (let table of tables) { + await this.snapshotTable(batch, table); + await batch.markSnapshotDone([table], ZERO_LSN); + + await touch(); + } + } + await batch.commit(ZERO_LSN); + }); + } + + static *getQueryData(results: Iterable): Generator { + for (let row of results) { + yield toSyncRulesRow(row); + } + } + + private async snapshotTable(batch: storage.BucketStorageBatch, table: storage.SourceTable) { + logger.info(`Replicating ${table.qualifiedName}`); + const estimatedCount = await this.estimatedCount(table); + let at = 0; + + const db = this.client.db(table.schema); + const collection = db.collection(table.table); + const query = collection.find({}, {}); + + const cursor = query.stream(); + + for await (let document of cursor) { + if (this.abort_signal.aborted) { + throw new Error(`Aborted initial replication`); + } + + const record = constructAfterRecord(document); + + // This auto-flushes when the batch reaches its size limit + await batch.save({ tag: 'insert', sourceTable: table, before: undefined, after: record }); + + at += 1; + Metrics.getInstance().rows_replicated_total.add(1); + + await touch(); + } + + await batch.flush(); + } + + async handleRelation(batch: storage.BucketStorageBatch, descriptor: SourceEntityDescriptor, snapshot: boolean) { + if (!descriptor.objectId && typeof descriptor.objectId != 'string') { + throw new Error('objectId expected'); + } + const result = await this.storage.resolveTable({ + group_id: this.group_id, + connection_id: this.connection_id, + connection_tag: this.connections.connectionTag, + entity_descriptor: descriptor, + sync_rules: this.sync_rules + }); + this.relation_cache.set(descriptor.objectId, result.table); + + // Drop conflicting tables. This includes for example renamed tables. + await batch.drop(result.dropTables); + + // Snapshot if: + // 1. Snapshot is requested (false for initial snapshot, since that process handles it elsewhere) + // 2. Snapshot is not already done, AND: + // 3. The table is used in sync rules. + const shouldSnapshot = snapshot && !result.table.snapshotComplete && result.table.syncAny; + + if (shouldSnapshot) { + // Truncate this table, in case a previous snapshot was interrupted. + await batch.truncate([result.table]); + + let lsn: string = ZERO_LSN; + // TODO: Transaction / consistency + await this.snapshotTable(batch, result.table); + const [table] = await batch.markSnapshotDone([result.table], lsn); + return table; + } + + return result.table; + } + + async writeChange( + batch: storage.BucketStorageBatch, + table: storage.SourceTable, + change: mongo.ChangeStreamDocument + ): Promise { + if (!table.syncAny) { + logger.debug(`Collection ${table.qualifiedName} not used in sync rules - skipping`); + return null; + } + + Metrics.getInstance().rows_replicated_total.add(1); + if (change.operationType == 'insert') { + const baseRecord = constructAfterRecord(change.fullDocument); + return await batch.save({ tag: 'insert', sourceTable: table, before: undefined, after: baseRecord }); + } else if (change.operationType == 'update') { + const before = undefined; // TODO: handle changing _id? + const after = constructAfterRecord(change.fullDocument!); + return await batch.save({ tag: 'update', sourceTable: table, before: before, after: after }); + } else if (change.operationType == 'delete') { + const key = constructAfterRecord(change.documentKey); + console.log('delete', key); + return await batch.save({ tag: 'delete', sourceTable: table, before: key }); + } else { + throw new Error(`Unsupported operation: ${change.operationType}`); + } + } + + async replicate() { + try { + // If anything errors here, the entire replication process is halted, and + // all connections automatically closed, including this one. + + await this.initReplication(); + await this.streamChanges(); + } catch (e) { + await this.storage.reportError(e); + throw e; + } + } + + async initReplication() { + const result = await this.initSlot(); + if (result.needsInitialSync) { + await this.startInitialReplication(); + } + } + + async streamChanges() { + // Auto-activate as soon as initial replication is done + await this.storage.autoActivate(); + + await this.storage.startBatch({ zeroLSN: ZERO_LSN }, async (batch) => { + // TODO: Resume replication + const stream = this.client.watch(undefined, { + fullDocument: 'updateLookup' // FIXME: figure this one out + }); + this.abort_signal.addEventListener('abort', () => { + stream.close(); + }); + + for await (const changeDocument of stream) { + await touch(); + + if (this.abort_signal.aborted) { + break; + } + + if ( + changeDocument.operationType == 'insert' || + changeDocument.operationType == 'update' || + changeDocument.operationType == 'delete' + ) { + const rel = getMongoRelation(changeDocument.ns); + const table = await this.handleRelation(batch, rel, true); + // TODO: Support transactions + if (table.syncAny) { + await this.writeChange(batch, table, changeDocument); + + if (changeDocument.clusterTime != null) { + const lsn = getMongoLsn(changeDocument.clusterTime); + await batch.commit(lsn); + } + } + } + } + }); + } +} + +async function touch() { + // FIXME: The hosted Kubernetes probe does not actually check the timestamp on this. + // FIXME: We need a timeout of around 5+ minutes in Kubernetes if we do start checking the timestamp, + // or reduce PING_INTERVAL here. + return container.probes.touch(); +} diff --git a/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts b/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts new file mode 100644 index 000000000..b3b993d7a --- /dev/null +++ b/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts @@ -0,0 +1,109 @@ +import { container } from '@powersync/lib-services-framework'; +import { MongoManager } from './MongoManager.js'; +import { MissingReplicationSlotError, ChangeStream } from './ChangeStream.js'; + +import { replication } from '@powersync/service-core'; +import { ConnectionManagerFactory } from './ConnectionManagerFactory.js'; + +export interface ChangeStreamReplicationJobOptions extends replication.AbstractReplicationJobOptions { + connectionFactory: ConnectionManagerFactory; +} + +export class ChangeStreamReplicationJob extends replication.AbstractReplicationJob { + private connectionFactory: ConnectionManagerFactory; + private readonly connectionManager: MongoManager; + + constructor(options: ChangeStreamReplicationJobOptions) { + super(options); + this.connectionFactory = options.connectionFactory; + this.connectionManager = this.connectionFactory.create(); + } + + async cleanUp(): Promise { + // TODO: Implement? + } + + async keepAlive() { + // TODO: Implement? + } + + async replicate() { + try { + await this.replicateLoop(); + } catch (e) { + // Fatal exception + container.reporter.captureException(e, { + metadata: {} + }); + this.logger.error(`Replication failed`, e); + } finally { + this.abortController.abort(); + } + } + + async replicateLoop() { + while (!this.isStopped) { + await this.replicateOnce(); + + if (!this.isStopped) { + await new Promise((resolve) => setTimeout(resolve, 5000)); + } + } + } + + async replicateOnce() { + // New connections on every iteration (every error with retry), + // otherwise we risk repeating errors related to the connection, + // such as caused by cached PG schemas. + const connectionManager = this.connectionFactory.create(); + try { + await this.rateLimiter?.waitUntilAllowed({ signal: this.abortController.signal }); + if (this.isStopped) { + return; + } + const stream = new ChangeStream({ + abort_signal: this.abortController.signal, + storage: this.options.storage, + connections: connectionManager + }); + await stream.replicate(); + } catch (e) { + this.logger.error(`Replication error`, e); + if (e.cause != null) { + // Example: + // PgError.conn_ended: Unable to do postgres query on ended connection + // at PgConnection.stream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:315:13) + // at stream.next () + // at PgResult.fromStream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:1174:22) + // at PgConnection.query (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:311:21) + // at WalStream.startInitialReplication (file:///.../powersync/powersync-service/lib/replication/WalStream.js:266:22) + // ... + // cause: TypeError: match is not iterable + // at timestamptzToSqlite (file:///.../powersync/packages/jpgwire/dist/util.js:140:50) + // at PgType.decode (file:///.../powersync/packages/jpgwire/dist/pgwire_types.js:25:24) + // at PgConnection._recvDataRow (file:///.../powersync/packages/jpgwire/dist/util.js:88:22) + // at PgConnection._recvMessages (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:656:30) + // at PgConnection._ioloopAttempt (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:563:20) + // at process.processTicksAndRejections (node:internal/process/task_queues:95:5) + // at async PgConnection._ioloop (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:517:14), + // [Symbol(pg.ErrorCode)]: 'conn_ended', + // [Symbol(pg.ErrorResponse)]: undefined + // } + // Without this additional log, the cause would not be visible in the logs. + this.logger.error(`cause`, e.cause); + } + if (e instanceof MissingReplicationSlotError) { + throw e; + } else { + // Report the error if relevant, before retrying + container.reporter.captureException(e, { + metadata: {} + }); + // This sets the retry delay + this.rateLimiter?.reportError(e); + } + } finally { + await connectionManager.end(); + } + } +} diff --git a/modules/module-mongodb/src/replication/ChangeStreamReplicator.ts b/modules/module-mongodb/src/replication/ChangeStreamReplicator.ts new file mode 100644 index 000000000..d4c1314dd --- /dev/null +++ b/modules/module-mongodb/src/replication/ChangeStreamReplicator.ts @@ -0,0 +1,34 @@ +import { storage, replication } from '@powersync/service-core'; +import { ChangeStreamReplicationJob } from './ChangeStreamReplicationJob.js'; +import { ConnectionManagerFactory } from './ConnectionManagerFactory.js'; + +export interface WalStreamReplicatorOptions extends replication.AbstractReplicatorOptions { + connectionFactory: ConnectionManagerFactory; +} + +export class ChangeStreamReplicator extends replication.AbstractReplicator { + private readonly connectionFactory: ConnectionManagerFactory; + + constructor(options: WalStreamReplicatorOptions) { + super(options); + this.connectionFactory = options.connectionFactory; + } + + createJob(options: replication.CreateJobOptions): ChangeStreamReplicationJob { + return new ChangeStreamReplicationJob({ + id: this.createJobId(options.storage.group_id), + storage: options.storage, + connectionFactory: this.connectionFactory, + lock: options.lock + }); + } + + async cleanUp(syncRulesStorage: storage.SyncRulesBucketStorage): Promise { + // TODO: Implement anything? + } + + async stop(): Promise { + await super.stop(); + await this.connectionFactory.shutdown(); + } +} diff --git a/modules/module-mongodb/src/replication/ConnectionManagerFactory.ts b/modules/module-mongodb/src/replication/ConnectionManagerFactory.ts new file mode 100644 index 000000000..c84c28e05 --- /dev/null +++ b/modules/module-mongodb/src/replication/ConnectionManagerFactory.ts @@ -0,0 +1,27 @@ +import { logger } from '@powersync/lib-services-framework'; +import { NormalizedMongoConnectionConfig } from '../types/types.js'; +import { MongoManager } from './MongoManager.js'; + +export class ConnectionManagerFactory { + private readonly connectionManagers: MongoManager[]; + private readonly dbConnectionConfig: NormalizedMongoConnectionConfig; + + constructor(dbConnectionConfig: NormalizedMongoConnectionConfig) { + this.dbConnectionConfig = dbConnectionConfig; + this.connectionManagers = []; + } + + create() { + const manager = new MongoManager(this.dbConnectionConfig); + this.connectionManagers.push(manager); + return manager; + } + + async shutdown() { + logger.info('Shutting down MongoDB connection Managers...'); + for (const manager of this.connectionManagers) { + await manager.end(); + } + logger.info('MongoDB connection Managers shutdown completed.'); + } +} diff --git a/modules/module-mongodb/src/replication/MongoErrorRateLimiter.ts b/modules/module-mongodb/src/replication/MongoErrorRateLimiter.ts new file mode 100644 index 000000000..bcf58db63 --- /dev/null +++ b/modules/module-mongodb/src/replication/MongoErrorRateLimiter.ts @@ -0,0 +1,45 @@ +import { setTimeout } from 'timers/promises'; +import { ErrorRateLimiter } from '@powersync/service-core'; + +export class MongoErrorRateLimiter implements ErrorRateLimiter { + nextAllowed: number = Date.now(); + + async waitUntilAllowed(options?: { signal?: AbortSignal | undefined } | undefined): Promise { + const delay = Math.max(0, this.nextAllowed - Date.now()); + // Minimum delay between connections, even without errors + this.setDelay(500); + await setTimeout(delay, undefined, { signal: options?.signal }); + } + + mayPing(): boolean { + return Date.now() >= this.nextAllowed; + } + + reportError(e: any): void { + // FIXME: Check mongodb-specific requirements + const message = (e.message as string) ?? ''; + if (message.includes('password authentication failed')) { + // Wait 15 minutes, to avoid triggering Supabase's fail2ban + this.setDelay(900_000); + } else if (message.includes('ENOTFOUND')) { + // DNS lookup issue - incorrect URI or deleted instance + this.setDelay(120_000); + } else if (message.includes('ECONNREFUSED')) { + // Could be fail2ban or similar + this.setDelay(120_000); + } else if ( + message.includes('Unable to do postgres query on ended pool') || + message.includes('Postgres unexpectedly closed connection') + ) { + // Connection timed out - ignore / immediately retry + // We don't explicitly set the delay to 0, since there could have been another error that + // we need to respect. + } else { + this.setDelay(30_000); + } + } + + private setDelay(delay: number) { + this.nextAllowed = Math.max(this.nextAllowed, Date.now() + delay); + } +} diff --git a/modules/module-mongodb/src/replication/MongoManager.ts b/modules/module-mongodb/src/replication/MongoManager.ts new file mode 100644 index 000000000..cb2f9d54f --- /dev/null +++ b/modules/module-mongodb/src/replication/MongoManager.ts @@ -0,0 +1,47 @@ +import * as mongo from 'mongodb'; +import { NormalizedMongoConnectionConfig } from '../types/types.js'; + +export class MongoManager { + /** + * Do not use this for any transactions. + */ + public readonly client: mongo.MongoClient; + public readonly db: mongo.Db; + + constructor(public options: NormalizedMongoConnectionConfig) { + // The pool is lazy - no connections are opened until a query is performed. + this.client = new mongo.MongoClient(options.uri, { + auth: { + username: options.username, + password: options.password + }, + // Time for connection to timeout + connectTimeoutMS: 5_000, + // Time for individual requests to timeout + socketTimeoutMS: 60_000, + // How long to wait for new primary selection + serverSelectionTimeoutMS: 30_000, + + // Avoid too many connections: + // 1. It can overwhelm the source database. + // 2. Processing too many queries in parallel can cause the process to run out of memory. + maxPoolSize: 8, + + maxConnecting: 3, + maxIdleTimeMS: 60_000 + }); + this.db = this.client.db(options.database, {}); + } + + public get connectionTag() { + return this.options.tag; + } + + async end(): Promise { + await this.client.close(); + } + + async destroy() { + // TODO: Implement? + } +} diff --git a/modules/module-mongodb/src/replication/MongoRelation.ts b/modules/module-mongodb/src/replication/MongoRelation.ts new file mode 100644 index 000000000..0573eda4a --- /dev/null +++ b/modules/module-mongodb/src/replication/MongoRelation.ts @@ -0,0 +1,104 @@ +import { storage } from '@powersync/service-core'; +import { SqliteRow, SqliteValue, toSyncRulesRow } from '@powersync/service-sync-rules'; +import * as mongo from 'mongodb'; +import { JSONBig, JsonContainer } from '@powersync/service-jsonbig'; + +export function getMongoRelation(source: mongo.ChangeStreamNameSpace): storage.SourceEntityDescriptor { + return { + name: source.coll, + schema: source.db, + objectId: source.coll, + replicationColumns: [{ name: '_id' }] + } satisfies storage.SourceEntityDescriptor; +} + +export function getMongoLsn(timestamp: mongo.Timestamp) { + const a = timestamp.high.toString(16).padStart(4, '0'); + const b = timestamp.low.toString(16).padStart(4, '0'); + return a + b; +} + +export function constructAfterRecord(document: mongo.Document): SqliteRow { + let record: SqliteRow = {}; + for (let key of Object.keys(document)) { + record[key] = toMongoSyncRulesValue(document[key]); + } + console.log('convert', document, record); + return record; +} + +export function toMongoSyncRulesValue(data: any): SqliteValue { + const autoBigNum = true; + if (data == null) { + // null or undefined + return data; + } else if (typeof data == 'string') { + return data; + } else if (typeof data == 'number') { + if (Number.isInteger(data) && autoBigNum) { + return BigInt(data); + } else { + return data; + } + } else if (typeof data == 'bigint') { + return data; + } else if (typeof data == 'boolean') { + return data ? 1n : 0n; + } else if (data instanceof mongo.ObjectId) { + return data.toHexString(); + } else if (data instanceof mongo.UUID) { + return data.toHexString(); + } else if (Array.isArray(data)) { + // We may be able to avoid some parse + stringify cycles here for JsonSqliteContainer. + return JSONBig.stringify(data.map((element) => filterJsonData(element))); + } else if (data instanceof Uint8Array) { + return data; + } else if (data instanceof JsonContainer) { + return data.toString(); + } else if (typeof data == 'object') { + let record: Record = {}; + for (let key of Object.keys(data)) { + record[key] = filterJsonData(data[key]); + } + return JSONBig.stringify(record); + } else { + return null; + } +} + +const DEPTH_LIMIT = 20; + +function filterJsonData(data: any, depth = 0): any { + if (depth > DEPTH_LIMIT) { + // This is primarily to prevent infinite recursion + throw new Error(`json nested object depth exceeds the limit of ${DEPTH_LIMIT}`); + } + if (data == null) { + return data; // null or undefined + } else if (typeof data == 'string' || typeof data == 'number') { + return data; + } else if (typeof data == 'boolean') { + return data ? 1n : 0n; + } else if (typeof data == 'bigint') { + return data; + } else if (data instanceof mongo.ObjectId) { + return data.toHexString(); + } else if (data instanceof mongo.UUID) { + return data.toHexString(); + } else if (Array.isArray(data)) { + return data.map((element) => filterJsonData(element, depth + 1)); + } else if (ArrayBuffer.isView(data)) { + return undefined; + } else if (data instanceof JsonContainer) { + // Can be stringified directly when using our JSONBig implementation + return data; + } else if (typeof data == 'object') { + let record: Record = {}; + for (let key of Object.keys(data)) { + record[key] = filterJsonData(data[key], depth + 1); + } + return record; + } else { + return undefined; + } +} diff --git a/modules/module-mongodb/src/replication/replication-index.ts b/modules/module-mongodb/src/replication/replication-index.ts new file mode 100644 index 000000000..4ff43b56a --- /dev/null +++ b/modules/module-mongodb/src/replication/replication-index.ts @@ -0,0 +1,4 @@ +export * from './MongoRelation.js'; +export * from './ChangeStream.js'; +export * from './ChangeStreamReplicator.js'; +export * from './ChangeStreamReplicationJob.js'; diff --git a/modules/module-mongodb/src/types/types.ts b/modules/module-mongodb/src/types/types.ts new file mode 100644 index 000000000..93e9fc846 --- /dev/null +++ b/modules/module-mongodb/src/types/types.ts @@ -0,0 +1,143 @@ +import * as service_types from '@powersync/service-types'; +import * as t from 'ts-codec'; +import * as urijs from 'uri-js'; + +export const MONGO_CONNECTION_TYPE = 'mongodb' as const; + +export interface NormalizedMongoConnectionConfig { + id: string; + tag: string; + + uri: string; + hostname?: string; + port?: number; + database?: string; + + username?: string; + password?: string; + + sslmode: 'verify-full' | 'verify-ca' | 'disable'; + cacert: string | undefined; + + client_certificate: string | undefined; + client_private_key: string | undefined; +} + +export const MongoConnectionConfig = service_types.configFile.dataSourceConfig.and( + t.object({ + type: t.literal(MONGO_CONNECTION_TYPE), + /** Unique identifier for the connection - optional when a single connection is present. */ + id: t.string.optional(), + /** Tag used as reference in sync rules. Defaults to "default". Does not have to be unique. */ + tag: t.string.optional(), + uri: t.string.optional(), + hostname: t.string.optional(), + port: service_types.configFile.portCodec.optional(), + username: t.string.optional(), + password: t.string.optional(), + database: t.string.optional(), + + /** Defaults to verify-full */ + sslmode: t.literal('verify-full').or(t.literal('verify-ca')).or(t.literal('disable')).optional(), + /** Required for verify-ca, optional for verify-full */ + cacert: t.string.optional(), + + client_certificate: t.string.optional(), + client_private_key: t.string.optional() + }) +); + +/** + * Config input specified when starting services + */ +export type MongoConnectionConfig = t.Decoded; + +/** + * Resolved version of {@link MongoConnectionConfig} + */ +export type ResolvedConnectionConfig = MongoConnectionConfig & NormalizedMongoConnectionConfig; + +/** + * Validate and normalize connection options. + * + * Returns destructured options. + */ +export function normalizeConnectionConfig(options: MongoConnectionConfig): NormalizedMongoConnectionConfig { + let uri: urijs.URIComponents; + if (options.uri) { + uri = urijs.parse(options.uri); + if (uri.scheme != 'mongodb') { + `Invalid URI - protocol must be postgresql, got ${uri.scheme}`; + } + } else { + uri = urijs.parse('mongodb:///'); + } + + const hostname = options.hostname ?? uri.host ?? ''; + const port = validatePort(options.port ?? uri.port ?? 5432); + + const database = options.database ?? uri.path?.substring(1) ?? ''; + + const [uri_username, uri_password] = (uri.userinfo ?? '').split(':'); + + const username = options.username ?? uri_username; + const password = options.password ?? uri_password; + + const sslmode = options.sslmode ?? 'verify-full'; // Configuration not supported via URI + const cacert = options.cacert; + + if (sslmode == 'verify-ca' && cacert == null) { + throw new Error('Explicit cacert is required for sslmode=verify-ca'); + } + + if (hostname == '') { + throw new Error(`hostname required`); + } + + if (database == '') { + throw new Error(`database required`); + } + + return { + id: options.id ?? 'default', + tag: options.tag ?? 'default', + + uri: options.uri ?? '', + hostname, + port, + database, + + username, + password, + sslmode, + cacert, + + client_certificate: options.client_certificate ?? undefined, + client_private_key: options.client_private_key ?? undefined + }; +} + +/** + * Check whether the port is in a "safe" range. + * + * We do not support connecting to "privileged" ports. + */ +export function validatePort(port: string | number): number { + if (typeof port == 'string') { + port = parseInt(port); + } + if (port >= 1024 && port <= 49151) { + return port; + } else { + throw new Error(`Port ${port} not supported`); + } +} + +/** + * Construct a mongodb URI, without username, password or ssl options. + * + * Only contains hostname, port, database. + */ +export function baseUri(options: NormalizedMongoConnectionConfig) { + return `mongodb://${options.hostname}:${options.port}/${options.database}`; +} diff --git a/modules/module-mongodb/test/tsconfig.json b/modules/module-mongodb/test/tsconfig.json new file mode 100644 index 000000000..18898c4ee --- /dev/null +++ b/modules/module-mongodb/test/tsconfig.json @@ -0,0 +1,28 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "baseUrl": "./", + "noEmit": true, + "esModuleInterop": true, + "skipLibCheck": true, + "sourceMap": true, + "paths": { + "@/*": ["../../../packages/service-core/src/*"], + "@module/*": ["../src/*"], + "@core-tests/*": ["../../../packages/service-core/test/src/*"] + } + }, + "include": ["src"], + "references": [ + { + "path": "../" + }, + { + "path": "../../../packages/service-core/test" + }, + { + "path": "../../../packages/service-core/" + } + ] +} diff --git a/modules/module-mongodb/tsconfig.json b/modules/module-mongodb/tsconfig.json new file mode 100644 index 000000000..6afdde02f --- /dev/null +++ b/modules/module-mongodb/tsconfig.json @@ -0,0 +1,28 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "esModuleInterop": true, + "skipLibCheck": true, + "sourceMap": true + }, + "include": ["src"], + "references": [ + { + "path": "../../packages/types" + }, + { + "path": "../../packages/jsonbig" + }, + { + "path": "../../packages/sync-rules" + }, + { + "path": "../../packages/service-core" + }, + { + "path": "../../libs/lib-services" + } + ] +} diff --git a/modules/module-mongodb/vitest.config.ts b/modules/module-mongodb/vitest.config.ts new file mode 100644 index 000000000..b392696b7 --- /dev/null +++ b/modules/module-mongodb/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'vitest/config'; +import tsconfigPaths from 'vite-tsconfig-paths'; + +export default defineConfig({ + plugins: [tsconfigPaths()], + test: { + setupFiles: './test/src/setup.ts' + } +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cd8bcf24b..320c292db 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -94,6 +94,49 @@ importers: specifier: ^0.34.6 version: 0.34.6 + modules/module-mongodb: + dependencies: + '@powersync/lib-services-framework': + specifier: workspace:* + version: link:../../libs/lib-services + '@powersync/service-core': + specifier: workspace:* + version: link:../../packages/service-core + '@powersync/service-jsonbig': + specifier: workspace:* + version: link:../../packages/jsonbig + '@powersync/service-sync-rules': + specifier: workspace:* + version: link:../../packages/sync-rules + '@powersync/service-types': + specifier: workspace:* + version: link:../../packages/types + mongodb: + specifier: ^6.7.0 + version: 6.8.0(socks@2.8.3) + ts-codec: + specifier: ^1.2.2 + version: 1.2.2 + uri-js: + specifier: ^4.4.1 + version: 4.4.1 + uuid: + specifier: ^9.0.1 + version: 9.0.1 + devDependencies: + '@types/uuid': + specifier: ^9.0.4 + version: 9.0.8 + typescript: + specifier: ^5.2.2 + version: 5.2.2 + vite-tsconfig-paths: + specifier: ^4.3.2 + version: 4.3.2(typescript@5.2.2)(vite@5.3.3(@types/node@18.11.11)) + vitest: + specifier: ^0.34.6 + version: 0.34.6 + modules/module-postgres: dependencies: '@powersync/lib-services-framework': @@ -376,6 +419,9 @@ importers: '@powersync/service-jsonbig': specifier: workspace:* version: link:../packages/jsonbig + '@powersync/service-module-mongodb': + specifier: workspace:* + version: link:../modules/module-mongodb '@powersync/service-module-postgres': specifier: workspace:* version: link:../modules/module-postgres diff --git a/service/package.json b/service/package.json index e5eb849cf..2f8aee5d0 100644 --- a/service/package.json +++ b/service/package.json @@ -17,6 +17,7 @@ "@powersync/service-core": "workspace:*", "@powersync/lib-services-framework": "workspace:*", "@powersync/service-module-postgres": "workspace:*", + "@powersync/service-module-mongodb": "workspace:*", "@powersync/service-jpgwire": "workspace:*", "@powersync/service-jsonbig": "workspace:*", "@powersync/service-rsocket-router": "workspace:*", diff --git a/service/src/entry.ts b/service/src/entry.ts index d817f46fe..c92d19397 100644 --- a/service/src/entry.ts +++ b/service/src/entry.ts @@ -1,6 +1,7 @@ import { container, ContainerImplementation } from '@powersync/lib-services-framework'; import * as core from '@powersync/service-core'; import PostgresModule from '@powersync/service-module-postgres'; +import MongoModule from '@powersync/service-module-mongodb'; import { startServer } from './runners/server.js'; import { startStreamRunner } from './runners/stream-worker.js'; @@ -12,7 +13,7 @@ container.registerDefaults(); container.register(ContainerImplementation.REPORTER, createSentryReporter()); const moduleManager = new core.modules.ModuleManager(); -moduleManager.register([PostgresModule]); +moduleManager.register([PostgresModule, MongoModule]); // This is a bit of a hack. Commands such as the teardown command or even migrations might // want access to the ModuleManager in order to use modules container.register(core.ModuleManager, moduleManager); diff --git a/service/tsconfig.json b/service/tsconfig.json index 363c72bdb..40c9a5329 100644 --- a/service/tsconfig.json +++ b/service/tsconfig.json @@ -32,6 +32,9 @@ }, { "path": "../modules/module-postgres" + }, + { + "path": "../modules/module-mongodb" } ] } From 4ae6b23fef757c2c50b25174f07a2ed4b5b3ccf8 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 12 Sep 2024 18:33:33 +0200 Subject: [PATCH 02/35] Support resuming; multi-document operations. --- .../src/replication/ChangeStream.ts | 46 ++++++++++++++++--- .../src/replication/MongoRelation.ts | 13 +++++- modules/module-mongodb/src/types/types.ts | 36 ++------------- .../service-core/src/storage/BucketStorage.ts | 5 ++ .../src/storage/mongo/MongoBucketBatch.ts | 4 ++ 5 files changed, 64 insertions(+), 40 deletions(-) diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 8ad5759df..4020190e8 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -3,7 +3,7 @@ import { Metrics, SourceEntityDescriptor, storage } from '@powersync/service-cor import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern, toSyncRulesRow } from '@powersync/service-sync-rules'; import * as mongo from 'mongodb'; import { MongoManager } from './MongoManager.js'; -import { constructAfterRecord, getMongoLsn, getMongoRelation } from './MongoRelation.js'; +import { constructAfterRecord, getMongoLsn, getMongoRelation, mongoLsnToTimestamp } from './MongoRelation.js'; export const ZERO_LSN = '00000000'; @@ -33,6 +33,7 @@ export class ChangeStream { private connections: MongoManager; private readonly client: mongo.MongoClient; + private readonly defaultDb: mongo.Db; private abort_signal: AbortSignal; @@ -44,6 +45,7 @@ export class ChangeStream { this.group_id = options.storage.group_id; this.connections = options.connections; this.client = this.connections.client; + this.defaultDb = this.connections.db; this.abort_signal = options.abort_signal; this.abort_signal.addEventListener( @@ -260,13 +262,24 @@ export class ChangeStream { await this.storage.startBatch({ zeroLSN: ZERO_LSN }, async (batch) => { // TODO: Resume replication + const lastLsn = batch.lastCheckpointLsn; + const startAfter = mongoLsnToTimestamp(lastLsn) ?? undefined; + logger.info(`Resume streaming at ${startAfter?.inspect()} / ${lastLsn}`); + + // TODO: Use changeStreamSplitLargeEvent + const stream = this.client.watch(undefined, { + startAtOperationTime: startAfter, + showExpandedEvents: true, + useBigInt64: true, fullDocument: 'updateLookup' // FIXME: figure this one out }); this.abort_signal.addEventListener('abort', () => { stream.close(); }); + let lastEventTimestamp: mongo.Timestamp | null = null; + for await (const changeDocument of stream) { await touch(); @@ -274,6 +287,30 @@ export class ChangeStream { break; } + if (startAfter != null && changeDocument.clusterTime?.lte(startAfter)) { + continue; + } + + if ( + lastEventTimestamp != null && + changeDocument.clusterTime != null && + changeDocument.clusterTime.neq(lastEventTimestamp) + ) { + const lsn = getMongoLsn(lastEventTimestamp); + await batch.commit(lsn); + lastEventTimestamp = null; + } + + if ((changeDocument as any).ns?.db != this.defaultDb.databaseName) { + // HACK: Ignore events to other databases, but only _after_ + // the above commit. + + // TODO: filter out storage db on in the pipeline + // TODO: support non-default dbs + continue; + } + console.log('event', changeDocument); + if ( changeDocument.operationType == 'insert' || changeDocument.operationType == 'update' || @@ -281,13 +318,10 @@ export class ChangeStream { ) { const rel = getMongoRelation(changeDocument.ns); const table = await this.handleRelation(batch, rel, true); - // TODO: Support transactions if (table.syncAny) { await this.writeChange(batch, table, changeDocument); - - if (changeDocument.clusterTime != null) { - const lsn = getMongoLsn(changeDocument.clusterTime); - await batch.commit(lsn); + if (changeDocument.clusterTime) { + lastEventTimestamp = changeDocument.clusterTime; } } } diff --git a/modules/module-mongodb/src/replication/MongoRelation.ts b/modules/module-mongodb/src/replication/MongoRelation.ts index 0573eda4a..924cda2af 100644 --- a/modules/module-mongodb/src/replication/MongoRelation.ts +++ b/modules/module-mongodb/src/replication/MongoRelation.ts @@ -13,11 +13,20 @@ export function getMongoRelation(source: mongo.ChangeStreamNameSpace): storage.S } export function getMongoLsn(timestamp: mongo.Timestamp) { - const a = timestamp.high.toString(16).padStart(4, '0'); - const b = timestamp.low.toString(16).padStart(4, '0'); + const a = timestamp.high.toString(16).padStart(8, '0'); + const b = timestamp.low.toString(16).padStart(8, '0'); return a + b; } +export function mongoLsnToTimestamp(lsn: string | null) { + if (lsn == null) { + return null; + } + const a = parseInt(lsn.substring(0, 8), 16); + const b = parseInt(lsn.substring(8, 16), 16); + return mongo.Timestamp.fromBits(b, a); +} + export function constructAfterRecord(document: mongo.Document): SqliteRow { let record: SqliteRow = {}; for (let key of Object.keys(document)) { diff --git a/modules/module-mongodb/src/types/types.ts b/modules/module-mongodb/src/types/types.ts index 93e9fc846..4aeef79d3 100644 --- a/modules/module-mongodb/src/types/types.ts +++ b/modules/module-mongodb/src/types/types.ts @@ -9,18 +9,10 @@ export interface NormalizedMongoConnectionConfig { tag: string; uri: string; - hostname?: string; - port?: number; - database?: string; + database: string; username?: string; password?: string; - - sslmode: 'verify-full' | 'verify-ca' | 'disable'; - cacert: string | undefined; - - client_certificate: string | undefined; - client_private_key: string | undefined; } export const MongoConnectionConfig = service_types.configFile.dataSourceConfig.and( @@ -73,9 +65,6 @@ export function normalizeConnectionConfig(options: MongoConnectionConfig): Norma uri = urijs.parse('mongodb:///'); } - const hostname = options.hostname ?? uri.host ?? ''; - const port = validatePort(options.port ?? uri.port ?? 5432); - const database = options.database ?? uri.path?.substring(1) ?? ''; const [uri_username, uri_password] = (uri.userinfo ?? '').split(':'); @@ -83,17 +72,6 @@ export function normalizeConnectionConfig(options: MongoConnectionConfig): Norma const username = options.username ?? uri_username; const password = options.password ?? uri_password; - const sslmode = options.sslmode ?? 'verify-full'; // Configuration not supported via URI - const cacert = options.cacert; - - if (sslmode == 'verify-ca' && cacert == null) { - throw new Error('Explicit cacert is required for sslmode=verify-ca'); - } - - if (hostname == '') { - throw new Error(`hostname required`); - } - if (database == '') { throw new Error(`database required`); } @@ -102,18 +80,12 @@ export function normalizeConnectionConfig(options: MongoConnectionConfig): Norma id: options.id ?? 'default', tag: options.tag ?? 'default', + // TODO: remove username & password from uri uri: options.uri ?? '', - hostname, - port, database, username, - password, - sslmode, - cacert, - - client_certificate: options.client_certificate ?? undefined, - client_private_key: options.client_private_key ?? undefined + password }; } @@ -139,5 +111,5 @@ export function validatePort(port: string | number): number { * Only contains hostname, port, database. */ export function baseUri(options: NormalizedMongoConnectionConfig) { - return `mongodb://${options.hostname}:${options.port}/${options.database}`; + return options.uri; } diff --git a/packages/service-core/src/storage/BucketStorage.ts b/packages/service-core/src/storage/BucketStorage.ts index 6f8ea0220..8615f1206 100644 --- a/packages/service-core/src/storage/BucketStorage.ts +++ b/packages/service-core/src/storage/BucketStorage.ts @@ -333,6 +333,11 @@ export interface BucketStorageBatch { */ keepalive(lsn: string): Promise; + /** + * Get the last checkpoint LSN, from either commit or keepalive. + */ + lastCheckpointLsn: string | null; + markSnapshotDone(tables: SourceTable[], no_checkpoint_before_lsn: string): Promise; } diff --git a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts index 973f6a28a..c49cdf689 100644 --- a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts +++ b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts @@ -73,6 +73,10 @@ export class MongoBucketBatch implements BucketStorageBatch { this.no_checkpoint_before_lsn = no_checkpoint_before_lsn; } + get lastCheckpointLsn() { + return this.last_checkpoint_lsn; + } + async flush(): Promise { let result: FlushedResult | null = null; // One flush may be split over multiple transactions. From bed97ebe23b97cd59696a9aec0e44cc878f9c9a9 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 13 Sep 2024 13:20:42 +0200 Subject: [PATCH 03/35] Workaround for transaction boundaries; filter ns in pipeline. --- .../src/replication/ChangeStream.ts | 64 +++++++++++++++---- .../src/storage/mongo/MongoBucketBatch.ts | 2 +- 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 4020190e8..e5ab7d771 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -70,7 +70,6 @@ export class ChangeStream { return []; } - const prefix = tablePattern.isWildcard ? tablePattern.tablePrefix : undefined; if (tablePattern.isWildcard) { // TODO: Implement throw new Error('not supported yet'); @@ -137,6 +136,28 @@ export class ChangeStream { }); } + private getSourceNamespaceFilters() { + const sourceTables = this.sync_rules.getSourceTables(); + + let filters: any[] = []; + for (let tablePattern of sourceTables) { + if (tablePattern.connectionTag != this.connections.connectionTag) { + continue; + } + + if (tablePattern.isWildcard) { + // TODO: Implement + throw new Error('wildcard collections not supported yet'); + } + + filters.push({ + db: tablePattern.schema, + coll: tablePattern.name + }); + } + return { $in: filters }; + } + static *getQueryData(results: Iterable): Generator { for (let row of results) { yield toSyncRulesRow(row); @@ -268,10 +289,22 @@ export class ChangeStream { // TODO: Use changeStreamSplitLargeEvent - const stream = this.client.watch(undefined, { + const nsFilter = this.getSourceNamespaceFilters(); + nsFilter.$in.push({ ns: nsFilter }); + + const pipeline: mongo.Document[] = [ + { + $match: { + ns: this.getSourceNamespaceFilters() + } + } + ]; + + const stream = this.client.watch(pipeline, { startAtOperationTime: startAfter, showExpandedEvents: true, useBigInt64: true, + maxAwaitTimeMS: 200, fullDocument: 'updateLookup' // FIXME: figure this one out }); this.abort_signal.addEventListener('abort', () => { @@ -280,7 +313,22 @@ export class ChangeStream { let lastEventTimestamp: mongo.Timestamp | null = null; - for await (const changeDocument of stream) { + while (true) { + const changeDocument = await stream.tryNext(); + if (changeDocument == null) { + // We don't get events for transaction commit. + // So if no events are available in the stream within maxAwaitTimeMS, + // we assume the transaction is complete. + // This is not foolproof - we may end up with a commit in the middle + // of a transaction. + if (lastEventTimestamp != null) { + const lsn = getMongoLsn(lastEventTimestamp); + await batch.commit(lsn); + lastEventTimestamp = null; + } + + continue; + } await touch(); if (this.abort_signal.aborted) { @@ -301,15 +349,7 @@ export class ChangeStream { lastEventTimestamp = null; } - if ((changeDocument as any).ns?.db != this.defaultDb.databaseName) { - // HACK: Ignore events to other databases, but only _after_ - // the above commit. - - // TODO: filter out storage db on in the pipeline - // TODO: support non-default dbs - continue; - } - console.log('event', changeDocument); + // console.log('event', changeDocument); if ( changeDocument.operationType == 'insert' || diff --git a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts index c49cdf689..a184ac408 100644 --- a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts +++ b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts @@ -539,7 +539,7 @@ export class MongoBucketBatch implements BucketStorageBatch { async commit(lsn: string): Promise { await this.flush(); - if (this.last_checkpoint_lsn != null && lsn <= this.last_checkpoint_lsn) { + if (this.last_checkpoint_lsn != null && lsn < this.last_checkpoint_lsn) { // When re-applying transactions, don't create a new checkpoint until // we are past the last transaction. logger.info(`Re-applied transaction ${lsn} - skipping checkpoint`); From f68a6ec155e463a05833db4acdf4666170c625cb Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 13 Sep 2024 13:57:41 +0200 Subject: [PATCH 04/35] Proper initial snapshot + streaming afterwards. --- .../src/replication/ChangeStream.ts | 50 +++++++++++++------ .../replication/ChangeStreamReplicationJob.ts | 36 ++++++------- .../src/replication/MongoRelation.ts | 1 - 3 files changed, 49 insertions(+), 38 deletions(-) diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index e5ab7d771..647909f14 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -5,7 +5,7 @@ import * as mongo from 'mongodb'; import { MongoManager } from './MongoManager.js'; import { constructAfterRecord, getMongoLsn, getMongoRelation, mongoLsnToTimestamp } from './MongoRelation.js'; -export const ZERO_LSN = '00000000'; +export const ZERO_LSN = '0000000000000000'; export interface WalStreamOptions { connections: MongoManager; @@ -72,7 +72,7 @@ export class ChangeStream { if (tablePattern.isWildcard) { // TODO: Implement - throw new Error('not supported yet'); + throw new Error('Wildcard collections not supported yet'); } let result: storage.SourceTable[] = []; @@ -122,18 +122,35 @@ export class ChangeStream { async initialReplication() { const sourceTables = this.sync_rules.getSourceTables(); - await this.storage.startBatch({ zeroLSN: ZERO_LSN }, async (batch) => { - for (let tablePattern of sourceTables) { - const tables = await this.getQualifiedTableNames(batch, tablePattern); - for (let table of tables) { - await this.snapshotTable(batch, table); - await batch.markSnapshotDone([table], ZERO_LSN); + await this.client.connect(); - await touch(); - } - } - await batch.commit(ZERO_LSN); + const session = await this.client.startSession({ + snapshot: true }); + try { + await this.storage.startBatch({ zeroLSN: ZERO_LSN }, async (batch) => { + for (let tablePattern of sourceTables) { + const tables = await this.getQualifiedTableNames(batch, tablePattern); + for (let table of tables) { + await this.snapshotTable(batch, table, session); + await batch.markSnapshotDone([table], ZERO_LSN); + + await touch(); + } + } + const time = session.clusterTime; + + if (time != null) { + const lsn = getMongoLsn(time.clusterTime); + logger.info(`Snapshot commit at ${time.clusterTime.inspect()} / ${lsn}`); + await batch.commit(lsn); + } else { + logger.info(`No snapshot clusterTime (no snapshot data?) - skipping commit.`); + } + }); + } finally { + session.endSession(); + } } private getSourceNamespaceFilters() { @@ -164,14 +181,18 @@ export class ChangeStream { } } - private async snapshotTable(batch: storage.BucketStorageBatch, table: storage.SourceTable) { + private async snapshotTable( + batch: storage.BucketStorageBatch, + table: storage.SourceTable, + session?: mongo.ClientSession + ) { logger.info(`Replicating ${table.qualifiedName}`); const estimatedCount = await this.estimatedCount(table); let at = 0; const db = this.client.db(table.schema); const collection = db.collection(table.table); - const query = collection.find({}, {}); + const query = collection.find({}, { session }); const cursor = query.stream(); @@ -282,7 +303,6 @@ export class ChangeStream { await this.storage.autoActivate(); await this.storage.startBatch({ zeroLSN: ZERO_LSN }, async (batch) => { - // TODO: Resume replication const lastLsn = batch.lastCheckpointLsn; const startAfter = mongoLsnToTimestamp(lastLsn) ?? undefined; logger.info(`Resume streaming at ${startAfter?.inspect()} / ${lastLsn}`); diff --git a/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts b/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts index b3b993d7a..05b6c1e26 100644 --- a/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts +++ b/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts @@ -5,6 +5,8 @@ import { MissingReplicationSlotError, ChangeStream } from './ChangeStream.js'; import { replication } from '@powersync/service-core'; import { ConnectionManagerFactory } from './ConnectionManagerFactory.js'; +import * as mongo from 'mongodb'; + export interface ChangeStreamReplicationJobOptions extends replication.AbstractReplicationJobOptions { connectionFactory: ConnectionManagerFactory; } @@ -27,6 +29,10 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ // TODO: Implement? } + private get slotName() { + return this.options.storage.slot_name; + } + async replicate() { try { await this.replicateLoop(); @@ -36,6 +42,11 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ metadata: {} }); this.logger.error(`Replication failed`, e); + + if (e instanceof MissingReplicationSlotError) { + // This stops replication on this slot, and creates a new slot + await this.options.storage.factory.slotRemoved(this.slotName); + } } finally { this.abortController.abort(); } @@ -70,30 +81,11 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ } catch (e) { this.logger.error(`Replication error`, e); if (e.cause != null) { - // Example: - // PgError.conn_ended: Unable to do postgres query on ended connection - // at PgConnection.stream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:315:13) - // at stream.next () - // at PgResult.fromStream (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:1174:22) - // at PgConnection.query (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:311:21) - // at WalStream.startInitialReplication (file:///.../powersync/powersync-service/lib/replication/WalStream.js:266:22) - // ... - // cause: TypeError: match is not iterable - // at timestamptzToSqlite (file:///.../powersync/packages/jpgwire/dist/util.js:140:50) - // at PgType.decode (file:///.../powersync/packages/jpgwire/dist/pgwire_types.js:25:24) - // at PgConnection._recvDataRow (file:///.../powersync/packages/jpgwire/dist/util.js:88:22) - // at PgConnection._recvMessages (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:656:30) - // at PgConnection._ioloopAttempt (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:563:20) - // at process.processTicksAndRejections (node:internal/process/task_queues:95:5) - // at async PgConnection._ioloop (file:///.../powersync/node_modules/.pnpm/github.com+kagis+pgwire@f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87/node_modules/pgwire/mod.js:517:14), - // [Symbol(pg.ErrorCode)]: 'conn_ended', - // [Symbol(pg.ErrorResponse)]: undefined - // } - // Without this additional log, the cause would not be visible in the logs. + // Without this additional log, the cause may not be visible in the logs. this.logger.error(`cause`, e.cause); } - if (e instanceof MissingReplicationSlotError) { - throw e; + if (e instanceof mongo.MongoError && e.hasErrorLabel('NonResumableChangeStreamError')) { + throw new MissingReplicationSlotError(e.message); } else { // Report the error if relevant, before retrying container.reporter.captureException(e, { diff --git a/modules/module-mongodb/src/replication/MongoRelation.ts b/modules/module-mongodb/src/replication/MongoRelation.ts index 924cda2af..d64e531f2 100644 --- a/modules/module-mongodb/src/replication/MongoRelation.ts +++ b/modules/module-mongodb/src/replication/MongoRelation.ts @@ -32,7 +32,6 @@ export function constructAfterRecord(document: mongo.Document): SqliteRow { for (let key of Object.keys(document)) { record[key] = toMongoSyncRulesValue(document[key]); } - console.log('convert', document, record); return record; } From 9d68d5c9d38e3733d8e44a09fec133498dbcecdc Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Sep 2024 12:20:14 +0200 Subject: [PATCH 05/35] Use _id directly as replica identity. --- .../src/replication/ChangeStream.ts | 39 +++++++++++++++---- .../src/replication/WalStream.ts | 38 +++++++++++++++--- .../service-core/src/storage/BucketStorage.ts | 11 +++++- .../src/storage/mongo/MongoBucketBatch.ts | 6 +-- .../src/storage/mongo/MongoCompactor.ts | 3 +- .../src/storage/mongo/OperationBatch.ts | 26 +++++++------ .../src/storage/mongo/PersistedBatch.ts | 5 ++- .../service-core/src/storage/mongo/models.ts | 5 ++- .../service-core/src/storage/mongo/util.ts | 32 ++++++++++++++- packages/service-core/src/util/utils.ts | 16 -------- 10 files changed, 129 insertions(+), 52 deletions(-) diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 647909f14..16685186a 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -204,7 +204,14 @@ export class ChangeStream { const record = constructAfterRecord(document); // This auto-flushes when the batch reaches its size limit - await batch.save({ tag: 'insert', sourceTable: table, before: undefined, after: record }); + await batch.save({ + tag: 'insert', + sourceTable: table, + before: undefined, + beforeReplicaId: undefined, + after: record, + afterReplicaId: document._id + }); at += 1; Metrics.getInstance().rows_replicated_total.add(1); @@ -264,15 +271,31 @@ export class ChangeStream { Metrics.getInstance().rows_replicated_total.add(1); if (change.operationType == 'insert') { const baseRecord = constructAfterRecord(change.fullDocument); - return await batch.save({ tag: 'insert', sourceTable: table, before: undefined, after: baseRecord }); + return await batch.save({ + tag: 'insert', + sourceTable: table, + before: undefined, + beforeReplicaId: undefined, + after: baseRecord, + afterReplicaId: change.documentKey._id + }); } else if (change.operationType == 'update') { - const before = undefined; // TODO: handle changing _id? - const after = constructAfterRecord(change.fullDocument!); - return await batch.save({ tag: 'update', sourceTable: table, before: before, after: after }); + const after = constructAfterRecord(change.fullDocument ?? {}); + return await batch.save({ + tag: 'update', + sourceTable: table, + before: undefined, + beforeReplicaId: undefined, + after: after, + afterReplicaId: change.documentKey._id + }); } else if (change.operationType == 'delete') { - const key = constructAfterRecord(change.documentKey); - console.log('delete', key); - return await batch.save({ tag: 'delete', sourceTable: table, before: key }); + return await batch.save({ + tag: 'delete', + sourceTable: table, + before: undefined, + beforeReplicaId: change.documentKey._id + }); } else { throw new Error(`Unsupported operation: ${change.operationType}`); } diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index 29ac6bfa5..c519a63d4 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -3,7 +3,7 @@ import * as util from '../utils/pgwire_utils.js'; import { container, errors, logger } from '@powersync/lib-services-framework'; import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern, toSyncRulesRow } from '@powersync/service-sync-rules'; import { getPgOutputRelation, getRelId } from './PgRelation.js'; -import { Metrics, SourceEntityDescriptor, storage } from '@powersync/service-core'; +import { getUuidReplicaIdentityBson, Metrics, SourceEntityDescriptor, storage } from '@powersync/service-core'; import { checkSourceConfiguration, getReplicationIdentityColumns } from './replication-utils.js'; import { PgManager } from './PgManager.js'; @@ -389,7 +389,14 @@ WHERE oid = $1::regclass`, for (let record of WalStream.getQueryData(rows)) { // This auto-flushes when the batch reaches its size limit - await batch.save({ tag: 'insert', sourceTable: table, before: undefined, after: record }); + await batch.save({ + tag: 'insert', + sourceTable: table, + before: undefined, + beforeReplicaId: undefined, + after: record, + afterReplicaId: getUuidReplicaIdentityBson(record, table.replicaIdColumns) + }); } at += rows.length; Metrics.getInstance().rows_replicated_total.add(rows.length); @@ -481,19 +488,40 @@ WHERE oid = $1::regclass`, if (msg.tag == 'insert') { Metrics.getInstance().rows_replicated_total.add(1); const baseRecord = util.constructAfterRecord(msg); - return await batch.save({ tag: 'insert', sourceTable: table, before: undefined, after: baseRecord }); + return await batch.save({ + tag: 'insert', + sourceTable: table, + before: undefined, + beforeReplicaId: undefined, + after: baseRecord, + afterReplicaId: getUuidReplicaIdentityBson(baseRecord, table.replicaIdColumns) + }); } else if (msg.tag == 'update') { Metrics.getInstance().rows_replicated_total.add(1); // "before" may be null if the replica id columns are unchanged // It's fine to treat that the same as an insert. const before = util.constructBeforeRecord(msg); const after = util.constructAfterRecord(msg); - return await batch.save({ tag: 'update', sourceTable: table, before: before, after: after }); + return await batch.save({ + tag: 'update', + sourceTable: table, + before: before, + beforeReplicaId: before ? getUuidReplicaIdentityBson(before, table.replicaIdColumns) : undefined, + after: after, + afterReplicaId: getUuidReplicaIdentityBson(after, table.replicaIdColumns) + }); } else if (msg.tag == 'delete') { Metrics.getInstance().rows_replicated_total.add(1); const before = util.constructBeforeRecord(msg)!; - return await batch.save({ tag: 'delete', sourceTable: table, before: before, after: undefined }); + return await batch.save({ + tag: 'delete', + sourceTable: table, + before: before, + beforeReplicaId: getUuidReplicaIdentityBson(before, table.replicaIdColumns), + after: undefined, + afterReplicaId: undefined + }); } } else if (msg.tag == 'truncate') { let tables: storage.SourceTable[] = []; diff --git a/packages/service-core/src/storage/BucketStorage.ts b/packages/service-core/src/storage/BucketStorage.ts index 8615f1206..2cf706528 100644 --- a/packages/service-core/src/storage/BucketStorage.ts +++ b/packages/service-core/src/storage/BucketStorage.ts @@ -10,6 +10,7 @@ import { import * as util from '../util/util-index.js'; import { SourceTable } from './SourceTable.js'; import { SourceEntityDescriptor } from './SourceEntity.js'; +import * as bson from 'bson'; export interface BucketStorageFactory { /** @@ -358,11 +359,15 @@ export interface SaveBucketData { export type SaveOptions = SaveInsert | SaveUpdate | SaveDelete; +export type ReplicaId = string | bson.UUID | bson.Document; + export interface SaveInsert { tag: 'insert'; sourceTable: SourceTable; before?: undefined; + beforeReplicaId?: undefined; after: SqliteRow; + afterReplicaId: ReplicaId; } export interface SaveUpdate { @@ -373,6 +378,7 @@ export interface SaveUpdate { * This is only present when the id has changed, and will only contain replica identity columns. */ before?: SqliteRow; + beforeReplicaId?: ReplicaId; /** * A null value means null column. @@ -380,13 +386,16 @@ export interface SaveUpdate { * An undefined value means it's a TOAST value - must be copied from another record. */ after: ToastableSqliteRow; + afterReplicaId: ReplicaId; } export interface SaveDelete { tag: 'delete'; sourceTable: SourceTable; - before: SqliteRow; + before?: SqliteRow; + beforeReplicaId: ReplicaId; after?: undefined; + afterReplicaId?: undefined; } export interface SyncBucketDataBatch { diff --git a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts index a184ac408..d82759029 100644 --- a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts +++ b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts @@ -11,7 +11,7 @@ import { CurrentBucket, CurrentDataDocument, SourceKey } from './models.js'; import { MongoIdSequence } from './MongoIdSequence.js'; import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js'; import { PersistedBatch } from './PersistedBatch.js'; -import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, serializeLookup } from './util.js'; +import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, replicaIdEquals, serializeLookup } from './util.js'; /** * 15MB @@ -305,7 +305,7 @@ export class MongoBucketBatch implements BucketStorageBatch { } // 2. Save bucket data - if (beforeId != null && (afterId == null || !beforeId.equals(afterId))) { + if (beforeId != null && (afterId == null || !replicaIdEquals(beforeId, afterId))) { // Source ID updated if (sourceTable.syncData) { // Delete old record @@ -435,7 +435,7 @@ export class MongoBucketBatch implements BucketStorageBatch { }; } - if (afterId == null || !beforeId.equals(afterId)) { + if (afterId == null || !replicaIdEquals(beforeId, afterId)) { // Either a delete (afterId == null), or replaced the old replication id batch.deleteCurrentData(before_key); } diff --git a/packages/service-core/src/storage/mongo/MongoCompactor.ts b/packages/service-core/src/storage/mongo/MongoCompactor.ts index 3c52936ba..ff754973d 100644 --- a/packages/service-core/src/storage/mongo/MongoCompactor.ts +++ b/packages/service-core/src/storage/mongo/MongoCompactor.ts @@ -4,6 +4,7 @@ import { addChecksums } from '../../util/utils.js'; import { PowerSyncMongo } from './db.js'; import { BucketDataDocument, BucketDataKey } from './models.js'; import { CompactOptions } from '../BucketStorage.js'; +import { cacheKey } from './OperationBatch.js'; interface CurrentBucketState { /** Bucket name */ @@ -168,7 +169,7 @@ export class MongoCompactor { let isPersistentPut = doc.op == 'PUT'; if (doc.op == 'REMOVE' || doc.op == 'PUT') { - const key = `${doc.table}/${doc.row_id}/${doc.source_table}/${doc.source_key?.toHexString()}`; + const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table!, doc.source_key!)}`; const targetOp = currentState.seen.get(key); if (targetOp) { // Will convert to MOVE, so don't count as PUT diff --git a/packages/service-core/src/storage/mongo/OperationBatch.ts b/packages/service-core/src/storage/mongo/OperationBatch.ts index 9e1edbb6d..75babb5a6 100644 --- a/packages/service-core/src/storage/mongo/OperationBatch.ts +++ b/packages/service-core/src/storage/mongo/OperationBatch.ts @@ -1,8 +1,7 @@ -import * as bson from 'bson'; import { ToastableSqliteRow } from '@powersync/service-sync-rules'; +import * as bson from 'bson'; -import * as util from '../../util/util-index.js'; -import { SaveOptions } from '../BucketStorage.js'; +import { ReplicaId, SaveOptions } from '../BucketStorage.js'; /** * Maximum number of operations in a batch. @@ -63,18 +62,15 @@ export class OperationBatch { } export class RecordOperation { - public readonly afterId: bson.UUID | null; - public readonly beforeId: bson.UUID; + public readonly afterId: ReplicaId | null; + public readonly beforeId: ReplicaId; public readonly internalBeforeKey: string; public readonly internalAfterKey: string | null; public readonly estimatedSize: number; constructor(public readonly record: SaveOptions) { - const after = record.after; - const afterId = after ? util.getUuidReplicaIdentityBson(after, record.sourceTable.replicaIdColumns!) : null; - const beforeId = record.before - ? util.getUuidReplicaIdentityBson(record.before, record.sourceTable.replicaIdColumns!) - : afterId!; + const afterId = record.afterReplicaId ?? null; + const beforeId = record.beforeReplicaId ?? record.afterReplicaId; this.afterId = afterId; this.beforeId = beforeId; this.internalBeforeKey = cacheKey(record.sourceTable.id, beforeId); @@ -84,8 +80,14 @@ export class RecordOperation { } } -export function cacheKey(table: bson.ObjectId, id: bson.UUID) { - return `${table.toHexString()}.${id.toHexString()}`; +export function cacheKey(table: bson.ObjectId, id: ReplicaId) { + if (id instanceof bson.UUID) { + return `${table.toHexString()}.${id.toHexString()}`; + } else if (typeof id == 'string') { + return `${table.toHexString()}.${id}`; + } else { + return `${table.toHexString()}.${(bson.serialize(id) as Buffer).toString('base64')}`; + } } /** diff --git a/packages/service-core/src/storage/mongo/PersistedBatch.ts b/packages/service-core/src/storage/mongo/PersistedBatch.ts index 486c9d800..0e5d2fabc 100644 --- a/packages/service-core/src/storage/mongo/PersistedBatch.ts +++ b/packages/service-core/src/storage/mongo/PersistedBatch.ts @@ -17,6 +17,7 @@ import { } from './models.js'; import { serializeLookup } from './util.js'; import { logger } from '@powersync/lib-services-framework'; +import { ReplicaId } from '../BucketStorage.js'; /** * Maximum size of operations we write in a single transaction. @@ -59,7 +60,7 @@ export class PersistedBatch { saveBucketData(options: { op_seq: MongoIdSequence; - sourceKey: bson.UUID; + sourceKey: ReplicaId; table: SourceTable; evaluated: EvaluatedRow[]; before_buckets: CurrentBucket[]; @@ -134,7 +135,7 @@ export class PersistedBatch { saveParameterData(data: { op_seq: MongoIdSequence; - sourceKey: bson.UUID; + sourceKey: ReplicaId; sourceTable: SourceTable; evaluated: EvaluatedParameters[]; existing_lookups: bson.Binary[]; diff --git a/packages/service-core/src/storage/mongo/models.ts b/packages/service-core/src/storage/mongo/models.ts index ef26564bc..19dbf0fbc 100644 --- a/packages/service-core/src/storage/mongo/models.ts +++ b/packages/service-core/src/storage/mongo/models.ts @@ -1,5 +1,6 @@ import * as bson from 'bson'; import { SqliteJsonValue } from '@powersync/service-sync-rules'; +import { ReplicaId } from '../BucketStorage.js'; export interface SourceKey { /** group_id */ @@ -7,7 +8,7 @@ export interface SourceKey { /** source table id */ t: bson.ObjectId; /** source key */ - k: bson.UUID; + k: ReplicaId; } export interface BucketDataKey { @@ -43,7 +44,7 @@ export interface BucketDataDocument { _id: BucketDataKey; op: OpType; source_table?: bson.ObjectId; - source_key?: bson.UUID; + source_key?: ReplicaId; table?: string; row_id?: string; checksum: number; diff --git a/packages/service-core/src/storage/mongo/util.ts b/packages/service-core/src/storage/mongo/util.ts index fef4bc396..8dc5ac40b 100644 --- a/packages/service-core/src/storage/mongo/util.ts +++ b/packages/service-core/src/storage/mongo/util.ts @@ -3,8 +3,10 @@ import * as bson from 'bson'; import * as mongo from 'mongodb'; import * as crypto from 'crypto'; import { BucketDataDocument } from './models.js'; -import { timestampToOpId } from '../../util/utils.js'; +import { ID_NAMESPACE, timestampToOpId } from '../../util/utils.js'; import { OplogEntry } from '../../util/protocol-types.js'; +import { ReplicaId } from '../BucketStorage.js'; +import * as uuid from 'uuid'; /** * Lookup serialization must be number-agnostic. I.e. normalize numbers, instead of preserving numbers. @@ -98,7 +100,7 @@ export function mapOpEntry(row: BucketDataDocument): OplogEntry { object_type: row.table, object_id: row.row_id, checksum: Number(row.checksum), - subkey: `${row.source_table}/${row.source_key!.toHexString()}`, + subkey: `${row.source_table}/${replicaIdToSubkey(row.source_key!)}`, data: row.data }; } else { @@ -111,3 +113,29 @@ export function mapOpEntry(row: BucketDataDocument): OplogEntry { }; } } + +export function replicaIdEquals(a: ReplicaId, b: ReplicaId) { + if (typeof a == 'string' && typeof b == 'string') { + return a == b; + } else if (a instanceof bson.UUID && b instanceof bson.UUID) { + return a.equals(b); + } else if (a == null && b == null) { + return true; + } else if (a != null || b != null) { + return false; + } else { + return (bson.serialize(a) as Buffer).equals(bson.serialize(b)); + } +} + +export function replicaIdToSubkey(id: ReplicaId) { + if (id instanceof bson.UUID) { + return id.toHexString(); + } else if (typeof id == 'string') { + return id; + } else { + const repr = bson.serialize(id); + return uuid.v5(repr, ID_NAMESPACE); + return; + } +} diff --git a/packages/service-core/src/util/utils.ts b/packages/service-core/src/util/utils.ts index 84cfa634e..9a40785cd 100644 --- a/packages/service-core/src/util/utils.ts +++ b/packages/service-core/src/util/utils.ts @@ -93,22 +93,6 @@ function getRawReplicaIdentity( return result; } -export function getUuidReplicaIdentityString( - tuple: sync_rules.ToastableSqliteRow, - columns: storage.ColumnDescriptor[] -): string { - const rawIdentity = getRawReplicaIdentity(tuple, columns); - - return uuidForRow(rawIdentity); -} - -export function uuidForRow(row: sync_rules.SqliteRow): string { - // Important: This must not change, since it will affect how ids are generated. - // Use BSON so that it's a well-defined format without encoding ambiguities. - const repr = bson.serialize(row); - return uuid.v5(repr, ID_NAMESPACE); -} - export function getUuidReplicaIdentityBson( tuple: sync_rules.ToastableSqliteRow, columns: storage.ColumnDescriptor[] From 5cf147453592f174f3aa40cc6e2886b53853fead Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Sep 2024 13:49:16 +0200 Subject: [PATCH 06/35] Configurable defaultSchema. --- .../src/api/MongoRouteAPIAdapter.ts | 13 +++++- .../src/replication/ChangeStream.ts | 43 +++++++++++-------- .../src/api/PostgresRouteAPIAdapter.ts | 8 +++- .../src/replication/WalStream.ts | 7 +-- .../test/src/wal_stream_utils.ts | 2 +- packages/service-core/src/api/RouteAPI.ts | 6 +++ packages/service-core/src/api/diagnostics.ts | 4 +- .../src/entry/commands/compact-action.ts | 2 +- .../src/replication/AbstractReplicator.ts | 5 +-- .../src/routes/endpoints/admin.ts | 12 +++--- .../src/routes/endpoints/socket-route.ts | 1 + .../src/routes/endpoints/sync-rules.ts | 16 +++++-- .../src/routes/endpoints/sync-stream.ts | 1 + packages/service-core/src/runner/teardown.ts | 2 +- .../service-core/src/storage/BucketStorage.ts | 19 +++++--- .../src/storage/MongoBucketStorage.ts | 32 +++++++------- .../service-core/src/storage/SourceTable.ts | 3 +- .../mongo/MongoPersistedSyncRulesContent.ts | 6 +-- .../storage/mongo/MongoSyncBucketStorage.ts | 14 +++++- .../src/storage/mongo/OperationBatch.ts | 8 +++- .../service-core/src/storage/mongo/util.ts | 19 ++++++-- packages/service-core/src/sync/sync.ts | 12 +++--- .../sync-rules/src/SqlBucketDescriptor.ts | 9 ++-- packages/sync-rules/src/SqlDataQuery.ts | 6 ++- packages/sync-rules/src/SqlParameterQuery.ts | 9 ++-- packages/sync-rules/src/SqlSyncRules.ts | 29 +++++++++---- packages/sync-rules/src/TablePattern.ts | 4 +- packages/sync-rules/src/types.ts | 3 +- 28 files changed, 189 insertions(+), 106 deletions(-) diff --git a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts index 61aed14f1..3d862db5e 100644 --- a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts +++ b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts @@ -1,4 +1,4 @@ -import { api } from '@powersync/service-core'; +import { api, ParseSyncRulesOptions } from '@powersync/service-core'; import * as mongo from 'mongodb'; import * as sync_rules from '@powersync/service-sync-rules'; @@ -10,12 +10,21 @@ export class MongoRouteAPIAdapter implements api.RouteAPI { protected client: mongo.MongoClient; connectionTag: string; + defaultSchema: string; constructor(protected config: types.ResolvedConnectionConfig) { - this.client = new MongoManager(config).client; + const manager = new MongoManager(config); + this.client = manager.client; + this.defaultSchema = manager.db.databaseName; this.connectionTag = config.tag ?? sync_rules.DEFAULT_TAG; } + getParseSyncRulesOptions(): ParseSyncRulesOptions { + return { + defaultSchema: this.defaultSchema + }; + } + async shutdown(): Promise { await this.client.close(); } diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 16685186a..150cb9970 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -41,11 +41,13 @@ export class ChangeStream { constructor(options: WalStreamOptions) { this.storage = options.storage; - this.sync_rules = options.storage.sync_rules; this.group_id = options.storage.group_id; this.connections = options.connections; this.client = this.connections.client; this.defaultDb = this.connections.db; + this.sync_rules = options.storage.getParsedSyncRules({ + defaultSchema: this.defaultDb.databaseName + }); this.abort_signal = options.abort_signal; this.abort_signal.addEventListener( @@ -128,26 +130,29 @@ export class ChangeStream { snapshot: true }); try { - await this.storage.startBatch({ zeroLSN: ZERO_LSN }, async (batch) => { - for (let tablePattern of sourceTables) { - const tables = await this.getQualifiedTableNames(batch, tablePattern); - for (let table of tables) { - await this.snapshotTable(batch, table, session); - await batch.markSnapshotDone([table], ZERO_LSN); - - await touch(); + await this.storage.startBatch( + { zeroLSN: ZERO_LSN, defaultSchema: this.defaultDb.databaseName }, + async (batch) => { + for (let tablePattern of sourceTables) { + const tables = await this.getQualifiedTableNames(batch, tablePattern); + for (let table of tables) { + await this.snapshotTable(batch, table, session); + await batch.markSnapshotDone([table], ZERO_LSN); + + await touch(); + } } - } - const time = session.clusterTime; + const time = session.clusterTime; - if (time != null) { - const lsn = getMongoLsn(time.clusterTime); - logger.info(`Snapshot commit at ${time.clusterTime.inspect()} / ${lsn}`); - await batch.commit(lsn); - } else { - logger.info(`No snapshot clusterTime (no snapshot data?) - skipping commit.`); + if (time != null) { + const lsn = getMongoLsn(time.clusterTime); + logger.info(`Snapshot commit at ${time.clusterTime.inspect()} / ${lsn}`); + await batch.commit(lsn); + } else { + logger.info(`No snapshot clusterTime (no snapshot data?) - skipping commit.`); + } } - }); + ); } finally { session.endSession(); } @@ -325,7 +330,7 @@ export class ChangeStream { // Auto-activate as soon as initial replication is done await this.storage.autoActivate(); - await this.storage.startBatch({ zeroLSN: ZERO_LSN }, async (batch) => { + await this.storage.startBatch({ zeroLSN: ZERO_LSN, defaultSchema: this.defaultDb.databaseName }, async (batch) => { const lastLsn = batch.lastCheckpointLsn; const startAfter = mongoLsnToTimestamp(lastLsn) ?? undefined; logger.info(`Resume streaming at ${startAfter?.inspect()} / ${lastLsn}`); diff --git a/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts b/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts index cfd11b132..205a2c744 100644 --- a/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts +++ b/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts @@ -1,4 +1,4 @@ -import { api } from '@powersync/service-core'; +import { api, ParseSyncRulesOptions } from '@powersync/service-core'; import * as pgwire from '@powersync/service-jpgwire'; import * as sync_rules from '@powersync/service-sync-rules'; @@ -23,6 +23,12 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI { this.connectionTag = config.tag ?? sync_rules.DEFAULT_TAG; } + getParseSyncRulesOptions(): ParseSyncRulesOptions { + return { + defaultSchema: 'public' + }; + } + async shutdown(): Promise { await this.pool.end(); } diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index c519a63d4..69cffc80e 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -9,6 +9,7 @@ import { PgManager } from './PgManager.js'; export const ZERO_LSN = '00000000/00000000'; export const PUBLICATION_NAME = 'powersync'; +export const POSTGRES_DEFAULT_SCHEMA = 'public'; export interface WalStreamOptions { connections: PgManager; @@ -46,7 +47,7 @@ export class WalStream { constructor(options: WalStreamOptions) { this.storage = options.storage; - this.sync_rules = options.storage.sync_rules; + this.sync_rules = options.storage.getParsedSyncRules({ defaultSchema: POSTGRES_DEFAULT_SCHEMA }); this.group_id = options.storage.group_id; this.slot_name = options.storage.slot_name; this.connections = options.connections; @@ -333,7 +334,7 @@ WHERE oid = $1::regclass`, async initialReplication(db: pgwire.PgConnection, lsn: string) { const sourceTables = this.sync_rules.getSourceTables(); - await this.storage.startBatch({ zeroLSN: ZERO_LSN }, async (batch) => { + await this.storage.startBatch({ zeroLSN: ZERO_LSN, defaultSchema: POSTGRES_DEFAULT_SCHEMA }, async (batch) => { for (let tablePattern of sourceTables) { const tables = await this.getQualifiedTableNames(batch, db, tablePattern); for (let table of tables) { @@ -569,7 +570,7 @@ WHERE oid = $1::regclass`, // Auto-activate as soon as initial replication is done await this.storage.autoActivate(); - await this.storage.startBatch({ zeroLSN: ZERO_LSN }, async (batch) => { + await this.storage.startBatch({ zeroLSN: ZERO_LSN, defaultSchema: POSTGRES_DEFAULT_SCHEMA }, async (batch) => { // Replication never starts in the middle of a transaction let inTx = false; let count = 0; diff --git a/modules/module-postgres/test/src/wal_stream_utils.ts b/modules/module-postgres/test/src/wal_stream_utils.ts index 0de968e8e..d9e4f2609 100644 --- a/modules/module-postgres/test/src/wal_stream_utils.ts +++ b/modules/module-postgres/test/src/wal_stream_utils.ts @@ -58,7 +58,7 @@ export class WalStreamTestContext { async updateSyncRules(content: string) { const syncRules = await this.factory.updateSyncRules({ content: content }); - this.storage = this.factory.getInstance(syncRules.parsed()); + this.storage = this.factory.getInstance(syncRules); return this.storage!; } diff --git a/packages/service-core/src/api/RouteAPI.ts b/packages/service-core/src/api/RouteAPI.ts index b5ad5ccab..393d56342 100644 --- a/packages/service-core/src/api/RouteAPI.ts +++ b/packages/service-core/src/api/RouteAPI.ts @@ -1,5 +1,6 @@ import { SqlSyncRules, TablePattern } from '@powersync/service-sync-rules'; import * as types from '@powersync/service-types'; +import { ParseSyncRulesOptions } from '../storage/BucketStorage.js'; export interface PatternResult { schema: string; @@ -69,4 +70,9 @@ export interface RouteAPI { * Close any resources that need graceful termination. */ shutdown(): Promise; + + /** + * Get the default schema (or database) when only a table name is specified in sync rules. + */ + getParseSyncRulesOptions(): ParseSyncRulesOptions; } diff --git a/packages/service-core/src/api/diagnostics.ts b/packages/service-core/src/api/diagnostics.ts index 7fd4ef440..7b6da99ce 100644 --- a/packages/service-core/src/api/diagnostics.ts +++ b/packages/service-core/src/api/diagnostics.ts @@ -43,7 +43,7 @@ export async function getSyncRulesStatus( let rules: SqlSyncRules; let persisted: storage.PersistedSyncRules; try { - persisted = sync_rules.parsed(); + persisted = sync_rules.parsed(apiHandler.getParseSyncRulesOptions()); rules = persisted.sync_rules; } catch (e) { return { @@ -53,7 +53,7 @@ export async function getSyncRulesStatus( }; } - const systemStorage = live_status ? bucketStorage.getInstance(persisted) : undefined; + const systemStorage = live_status ? bucketStorage.getInstance(sync_rules) : undefined; const status = await systemStorage?.getStatus(); let replication_lag_bytes: number | undefined = undefined; diff --git a/packages/service-core/src/entry/commands/compact-action.ts b/packages/service-core/src/entry/commands/compact-action.ts index 6016644e5..5a6636bf1 100644 --- a/packages/service-core/src/entry/commands/compact-action.ts +++ b/packages/service-core/src/entry/commands/compact-action.ts @@ -34,7 +34,7 @@ export function registerCompactAction(program: Command) { await client.connect(); try { const bucketStorage = new storage.MongoBucketStorage(psdb, { slot_name_prefix: configuration.slot_name_prefix }); - const active = await bucketStorage.getActiveSyncRules(); + const active = await bucketStorage.getActiveSyncRulesContent(); if (active == null) { logger.info('No active instance to compact'); return; diff --git a/packages/service-core/src/replication/AbstractReplicator.ts b/packages/service-core/src/replication/AbstractReplicator.ts index bcf480fc9..6c5a2a934 100644 --- a/packages/service-core/src/replication/AbstractReplicator.ts +++ b/packages/service-core/src/replication/AbstractReplicator.ts @@ -166,8 +166,7 @@ export abstract class AbstractReplicator try { for await (const data of sync.streamResponse({ storage: activeBucketStorage, + parseOptions: routerEngine!.getAPI().getParseSyncRulesOptions(), params: { ...params, binary_data: true // always true for web sockets diff --git a/packages/service-core/src/routes/endpoints/sync-rules.ts b/packages/service-core/src/routes/endpoints/sync-rules.ts index 605d3c82d..1129a5855 100644 --- a/packages/service-core/src/routes/endpoints/sync-rules.ts +++ b/packages/service-core/src/routes/endpoints/sync-rules.ts @@ -53,7 +53,12 @@ export const deploySyncRules = routeDefinition({ const content = payload.params.content; try { - SqlSyncRules.fromYaml(payload.params.content); + const apiHandler = service_context.routerEngine!.getAPI(); + SqlSyncRules.fromYaml(payload.params.content, { + ...apiHandler.getParseSyncRulesOptions(), + // We don't do any schema-level validation at this point + schema: undefined + }); } catch (e) { throw new errors.JourneyError({ status: 422, @@ -151,7 +156,8 @@ export const reprocessSyncRules = routeDefinition({ const { storageEngine: { activeBucketStorage } } = payload.context.service_context; - const sync_rules = await activeBucketStorage.getActiveSyncRules(); + const apiHandler = payload.context.service_context.routerEngine!.getAPI(); + const sync_rules = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions()); if (sync_rules == null) { throw new errors.JourneyError({ status: 422, @@ -181,7 +187,11 @@ function replyPrettyJson(payload: any) { async function debugSyncRules(apiHandler: RouteAPI, sync_rules: string) { try { - const rules = SqlSyncRules.fromYaml(sync_rules); + const rules = SqlSyncRules.fromYaml(sync_rules, { + ...apiHandler.getParseSyncRulesOptions(), + // No schema-based validation at this point + schema: undefined + }); const source_table_patterns = rules.getSourceTables(); const resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules); diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index e637e99ea..83d8f3994 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -54,6 +54,7 @@ export const syncStreamed = routeDefinition({ sync.ndjson( sync.streamResponse({ storage: storageEngine.activeBucketStorage, + parseOptions: routerEngine!.getAPI().getParseSyncRulesOptions(), params, syncParams, token: payload.context.token_payload!, diff --git a/packages/service-core/src/runner/teardown.ts b/packages/service-core/src/runner/teardown.ts index 49e2bc66d..583635642 100644 --- a/packages/service-core/src/runner/teardown.ts +++ b/packages/service-core/src/runner/teardown.ts @@ -51,7 +51,7 @@ async function terminateSyncRules(storageFactory: storage.BucketStorageFactory, // Mark the sync rules as terminated for (let syncRules of combinedSyncRules) { - const syncRulesStorage = storageFactory.getInstance(syncRules.parsed()); + const syncRulesStorage = storageFactory.getInstance(syncRules); // The storage will be dropped at the end of the teardown, so we don't need to clear it here await syncRulesStorage.terminate({ clearStorage: false }); } diff --git a/packages/service-core/src/storage/BucketStorage.ts b/packages/service-core/src/storage/BucketStorage.ts index 2cf706528..15aaf61d5 100644 --- a/packages/service-core/src/storage/BucketStorage.ts +++ b/packages/service-core/src/storage/BucketStorage.ts @@ -24,7 +24,7 @@ export interface BucketStorageFactory { /** * Get a storage instance to query sync data for specific sync rules. */ - getInstance(options: PersistedSyncRules): SyncRulesBucketStorage; + getInstance(options: PersistedSyncRulesContent): SyncRulesBucketStorage; /** * Deploy new sync rules. @@ -48,7 +48,7 @@ export interface BucketStorageFactory { /** * Get the sync rules used for querying. */ - getActiveSyncRules(): Promise; + getActiveSyncRules(options: ParseSyncRulesOptions): Promise; /** * Get the sync rules used for querying. @@ -58,7 +58,7 @@ export interface BucketStorageFactory { /** * Get the sync rules that will be active next once done with initial replicatino. */ - getNextSyncRules(): Promise; + getNextSyncRules(options: ParseSyncRulesOptions): Promise; /** * Get the sync rules that will be active next once done with initial replicatino. @@ -131,6 +131,10 @@ export interface StorageMetrics { replication_size_bytes: number; } +export interface ParseSyncRulesOptions { + defaultSchema: string; +} + export interface PersistedSyncRulesContent { readonly id: number; readonly sync_rules_content: string; @@ -140,7 +144,7 @@ export interface PersistedSyncRulesContent { readonly last_keepalive_ts?: Date | null; readonly last_checkpoint_ts?: Date | null; - parsed(): PersistedSyncRules; + parsed(options: ParseSyncRulesOptions): PersistedSyncRules; lock(): Promise; } @@ -186,12 +190,11 @@ export interface BucketDataBatchOptions { chunkLimitBytes?: number; } -export interface StartBatchOptions { +export interface StartBatchOptions extends ParseSyncRulesOptions { zeroLSN: string; } export interface SyncRulesBucketStorage { - readonly sync_rules: SqlSyncRules; readonly group_id: number; readonly slot_name: string; @@ -206,6 +209,8 @@ export interface SyncRulesBucketStorage { getCheckpoint(): Promise<{ checkpoint: util.OpId }>; + getParsedSyncRules(options: ParseSyncRulesOptions): SqlSyncRules; + getParameterSets(checkpoint: util.OpId, lookups: SqliteJsonValue[][]): Promise; /** @@ -359,7 +364,7 @@ export interface SaveBucketData { export type SaveOptions = SaveInsert | SaveUpdate | SaveDelete; -export type ReplicaId = string | bson.UUID | bson.Document; +export type ReplicaId = string | bson.UUID | bson.Document | any; export interface SaveInsert { tag: 'insert'; diff --git a/packages/service-core/src/storage/MongoBucketStorage.ts b/packages/service-core/src/storage/MongoBucketStorage.ts index 2cf45c436..c3e59d0b7 100644 --- a/packages/service-core/src/storage/MongoBucketStorage.ts +++ b/packages/service-core/src/storage/MongoBucketStorage.ts @@ -13,6 +13,7 @@ import { v4 as uuid } from 'uuid'; import { ActiveCheckpoint, BucketStorageFactory, + ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent, StorageMetrics, @@ -47,7 +48,7 @@ export class MongoBucketStorage implements BucketStorageFactory { return undefined; } const rules = new MongoPersistedSyncRulesContent(this.db, doc2); - return this.getInstance(rules.parsed()); + return this.getInstance(rules); } }); @@ -60,12 +61,12 @@ export class MongoBucketStorage implements BucketStorageFactory { this.slot_name_prefix = options.slot_name_prefix; } - getInstance(options: PersistedSyncRules): MongoSyncBucketStorage { - let { id, sync_rules, slot_name } = options; + getInstance(options: PersistedSyncRulesContent): MongoSyncBucketStorage { + let { id, slot_name } = options; if ((typeof id as any) == 'bigint') { id = Number(id); } - return new MongoSyncBucketStorage(this, id, sync_rules, slot_name); + return new MongoSyncBucketStorage(this, id, options, slot_name); } async configureSyncRules(sync_rules: string, options?: { lock?: boolean }) { @@ -135,7 +136,12 @@ export class MongoBucketStorage implements BucketStorageFactory { async updateSyncRules(options: UpdateSyncRulesOptions): Promise { // Parse and validate before applying any changes - const parsed = SqlSyncRules.fromYaml(options.content); + const parsed = SqlSyncRules.fromYaml(options.content, { + // No schema-based validation at this point + schema: undefined, + defaultSchema: 'not_applicable', // Not needed for validation + throwOnError: true + }); let rules: MongoPersistedSyncRulesContent | undefined = undefined; @@ -203,9 +209,9 @@ export class MongoBucketStorage implements BucketStorageFactory { return new MongoPersistedSyncRulesContent(this.db, doc); } - async getActiveSyncRules(): Promise { + async getActiveSyncRules(options: ParseSyncRulesOptions): Promise { const content = await this.getActiveSyncRulesContent(); - return content?.parsed() ?? null; + return content?.parsed(options) ?? null; } async getNextSyncRulesContent(): Promise { @@ -222,9 +228,9 @@ export class MongoBucketStorage implements BucketStorageFactory { return new MongoPersistedSyncRulesContent(this.db, doc); } - async getNextSyncRules(): Promise { + async getNextSyncRules(options: ParseSyncRulesOptions): Promise { const content = await this.getNextSyncRulesContent(); - return content?.parsed() ?? null; + return content?.parsed(options) ?? null; } async getReplicatingSyncRules(): Promise { @@ -293,14 +299,6 @@ export class MongoBucketStorage implements BucketStorageFactory { } async getStorageMetrics(): Promise { - const active_sync_rules = await this.getActiveSyncRules(); - if (active_sync_rules == null) { - return { - operations_size_bytes: 0, - parameters_size_bytes: 0, - replication_size_bytes: 0 - }; - } const operations_aggregate = await this.db.bucket_data .aggregate([ diff --git a/packages/service-core/src/storage/SourceTable.ts b/packages/service-core/src/storage/SourceTable.ts index 6379732f5..2f5364633 100644 --- a/packages/service-core/src/storage/SourceTable.ts +++ b/packages/service-core/src/storage/SourceTable.ts @@ -1,9 +1,8 @@ -import { DEFAULT_SCHEMA, DEFAULT_TAG } from '@powersync/service-sync-rules'; +import { DEFAULT_TAG } from '@powersync/service-sync-rules'; import * as util from '../util/util-index.js'; import { ColumnDescriptor } from './SourceEntity.js'; export class SourceTable { - static readonly DEFAULT_SCHEMA = DEFAULT_SCHEMA; static readonly DEFAULT_TAG = DEFAULT_TAG; /** diff --git a/packages/service-core/src/storage/mongo/MongoPersistedSyncRulesContent.ts b/packages/service-core/src/storage/mongo/MongoPersistedSyncRulesContent.ts index a32cf6fc1..363766df4 100644 --- a/packages/service-core/src/storage/mongo/MongoPersistedSyncRulesContent.ts +++ b/packages/service-core/src/storage/mongo/MongoPersistedSyncRulesContent.ts @@ -1,7 +1,7 @@ import { SqlSyncRules } from '@powersync/service-sync-rules'; import * as mongo from 'mongodb'; -import { PersistedSyncRulesContent } from '../BucketStorage.js'; +import { ParseSyncRulesOptions, PersistedSyncRulesContent } from '../BucketStorage.js'; import { MongoPersistedSyncRules } from './MongoPersistedSyncRules.js'; import { MongoSyncRulesLock } from './MongoSyncRulesLock.js'; import { PowerSyncMongo } from './db.js'; @@ -30,10 +30,10 @@ export class MongoPersistedSyncRulesContent implements PersistedSyncRulesContent this.last_keepalive_ts = doc.last_keepalive_ts; } - parsed() { + parsed(options: ParseSyncRulesOptions) { return new MongoPersistedSyncRules( this.id, - SqlSyncRules.fromYaml(this.sync_rules_content), + SqlSyncRules.fromYaml(this.sync_rules_content, options), this.last_checkpoint_lsn, this.slot_name ); diff --git a/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts b/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts index 95715d45c..771f5093d 100644 --- a/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts +++ b/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts @@ -11,6 +11,9 @@ import { DEFAULT_DOCUMENT_BATCH_LIMIT, DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES, FlushedResult, + ParseSyncRulesOptions, + PersistedSyncRules, + PersistedSyncRulesContent, ResolveTableOptions, ResolveTableResult, StartBatchOptions, @@ -36,15 +39,22 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage { } }); + private parsedSyncRulesCache: SqlSyncRules | undefined; + constructor( public readonly factory: MongoBucketStorage, public readonly group_id: number, - public readonly sync_rules: SqlSyncRules, + private readonly sync_rules: PersistedSyncRulesContent, public readonly slot_name: string ) { this.db = factory.db; } + getParsedSyncRules(options: ParseSyncRulesOptions): SqlSyncRules { + this.parsedSyncRulesCache ??= this.sync_rules.parsed(options).sync_rules; + return this.parsedSyncRulesCache; + } + async getCheckpoint() { const doc = await this.db.sync_rules.findOne( { _id: this.group_id }, @@ -71,7 +81,7 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage { const batch = new MongoBucketBatch( this.db, - this.sync_rules, + this.sync_rules.parsed(options).sync_rules, this.group_id, this.slot_name, checkpoint_lsn, diff --git a/packages/service-core/src/storage/mongo/OperationBatch.ts b/packages/service-core/src/storage/mongo/OperationBatch.ts index 75babb5a6..74b4b651f 100644 --- a/packages/service-core/src/storage/mongo/OperationBatch.ts +++ b/packages/service-core/src/storage/mongo/OperationBatch.ts @@ -2,6 +2,7 @@ import { ToastableSqliteRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { ReplicaId, SaveOptions } from '../BucketStorage.js'; +import { isUUID } from './util.js'; /** * Maximum number of operations in a batch. @@ -80,13 +81,16 @@ export class RecordOperation { } } +/** + * In-memory cache key - must not be persisted. + */ export function cacheKey(table: bson.ObjectId, id: ReplicaId) { - if (id instanceof bson.UUID) { + if (isUUID(id)) { return `${table.toHexString()}.${id.toHexString()}`; } else if (typeof id == 'string') { return `${table.toHexString()}.${id}`; } else { - return `${table.toHexString()}.${(bson.serialize(id) as Buffer).toString('base64')}`; + return `${table.toHexString()}.${(bson.serialize({ id: id }) as Buffer).toString('base64')}`; } } diff --git a/packages/service-core/src/storage/mongo/util.ts b/packages/service-core/src/storage/mongo/util.ts index 8dc5ac40b..58a9bc36c 100644 --- a/packages/service-core/src/storage/mongo/util.ts +++ b/packages/service-core/src/storage/mongo/util.ts @@ -115,9 +115,11 @@ export function mapOpEntry(row: BucketDataDocument): OplogEntry { } export function replicaIdEquals(a: ReplicaId, b: ReplicaId) { - if (typeof a == 'string' && typeof b == 'string') { + if (a === b) { + return true; + } else if (typeof a == 'string' && typeof b == 'string') { return a == b; - } else if (a instanceof bson.UUID && b instanceof bson.UUID) { + } else if (isUUID(a) && isUUID(b)) { return a.equals(b); } else if (a == null && b == null) { return true; @@ -129,13 +131,22 @@ export function replicaIdEquals(a: ReplicaId, b: ReplicaId) { } export function replicaIdToSubkey(id: ReplicaId) { - if (id instanceof bson.UUID) { + if (isUUID(id)) { + // Special case for UUID for backwards-compatiblity return id.toHexString(); } else if (typeof id == 'string') { return id; } else { - const repr = bson.serialize(id); + const repr = bson.serialize({ id: id }); return uuid.v5(repr, ID_NAMESPACE); return; } } + +export function isUUID(value: any): value is bson.UUID { + if (value == null || typeof value != 'object') { + return false; + } + const uuid = value as bson.UUID; + return uuid._bsontype == 'Binary' && uuid.sub_type == bson.Binary.SUBTYPE_UUID; +} diff --git a/packages/service-core/src/sync/sync.ts b/packages/service-core/src/sync/sync.ts index e439888f8..7a71540e6 100644 --- a/packages/service-core/src/sync/sync.ts +++ b/packages/service-core/src/sync/sync.ts @@ -1,5 +1,5 @@ import { JSONBig, JsonContainer } from '@powersync/service-jsonbig'; -import { RequestParameters } from '@powersync/service-sync-rules'; +import { RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules'; import { Semaphore } from 'async-mutex'; import { AbortError } from 'ix/aborterror.js'; @@ -23,6 +23,7 @@ export interface SyncStreamParameters { params: util.StreamingSyncRequest; syncParams: RequestParameters; token: auth.JwtPayload; + parseOptions: storage.ParseSyncRulesOptions; /** * If this signal is aborted, the stream response ends as soon as possible, without error. */ @@ -35,7 +36,7 @@ export interface SyncStreamParameters { export async function* streamResponse( options: SyncStreamParameters ): AsyncIterable { - const { storage, params, syncParams, token, tokenStreamOptions, tracker, signal } = options; + const { storage, params, syncParams, token, tokenStreamOptions, tracker, signal, parseOptions } = options; // We also need to be able to abort, so we create our own controller. const controller = new AbortController(); if (signal) { @@ -51,7 +52,7 @@ export async function* streamResponse( } } const ki = tokenStream(token, controller.signal, tokenStreamOptions); - const stream = streamResponseInner(storage, params, syncParams, tracker, controller.signal); + const stream = streamResponseInner(storage, params, syncParams, tracker, parseOptions, controller.signal); // Merge the two streams, and abort as soon as one of the streams end. const merged = mergeAsyncIterables([stream, ki], controller.signal); @@ -75,6 +76,7 @@ async function* streamResponseInner( params: util.StreamingSyncRequest, syncParams: RequestParameters, tracker: RequestTracker, + parseOptions: storage.ParseSyncRulesOptions, signal: AbortSignal ): AsyncGenerator { // Bucket state of bucket id -> op_id. @@ -103,9 +105,9 @@ async function* streamResponseInner( // Sync rules deleted in the meantime - try again with the next checkpoint. continue; } - const sync_rules = storage.sync_rules; + const syncRules = storage.getParsedSyncRules(parseOptions); - const allBuckets = await sync_rules.queryBucketIds({ + const allBuckets = await syncRules.queryBucketIds({ getParameterSets(lookups) { return storage.getParameterSets(checkpoint, lookups); }, diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index 459178ee1..a140a1fc2 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -2,6 +2,7 @@ import { IdSequence } from './IdSequence.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { SqlDataQuery } from './SqlDataQuery.js'; import { SqlParameterQuery } from './SqlParameterQuery.js'; +import { SyncRulesOptions } from './SqlSyncRules.js'; import { StaticSqlParameterQuery } from './StaticSqlParameterQuery.js'; import { TablePattern } from './TablePattern.js'; import { SqlRuleError } from './errors.js'; @@ -42,11 +43,11 @@ export class SqlBucketDescriptor { parameterIdSequence = new IdSequence(); - addDataQuery(sql: string, schema?: SourceSchema): QueryParseResult { + addDataQuery(sql: string, options: SyncRulesOptions): QueryParseResult { if (this.bucket_parameters == null) { throw new Error('Bucket parameters must be defined'); } - const dataRows = SqlDataQuery.fromSql(this.name, this.bucket_parameters, sql, schema); + const dataRows = SqlDataQuery.fromSql(this.name, this.bucket_parameters, sql, options); dataRows.ruleId = this.idSequence.nextId(); @@ -58,8 +59,8 @@ export class SqlBucketDescriptor { }; } - addParameterQuery(sql: string, schema: SourceSchema | undefined, options: QueryParseOptions): QueryParseResult { - const parameterQuery = SqlParameterQuery.fromSql(this.name, sql, schema, options); + addParameterQuery(sql: string, options: QueryParseOptions): QueryParseResult { + const parameterQuery = SqlParameterQuery.fromSql(this.name, sql, options); if (this.bucket_parameters == null) { this.bucket_parameters = parameterQuery.bucket_parameters; } else { diff --git a/packages/sync-rules/src/SqlDataQuery.ts b/packages/sync-rules/src/SqlDataQuery.ts index c1176bc9b..b5377d436 100644 --- a/packages/sync-rules/src/SqlDataQuery.ts +++ b/packages/sync-rules/src/SqlDataQuery.ts @@ -19,6 +19,7 @@ import { } from './types.js'; import { filterJsonRow, getBucketId, isSelectStatement } from './utils.js'; import { TableQuerySchema } from './TableQuerySchema.js'; +import { SyncRulesOptions } from './SqlSyncRules.js'; interface RowValueExtractor { extract(tables: QueryParameters, into: SqliteRow): void; @@ -26,9 +27,10 @@ interface RowValueExtractor { } export class SqlDataQuery { - static fromSql(descriptor_name: string, bucket_parameters: string[], sql: string, schema?: SourceSchema) { + static fromSql(descriptor_name: string, bucket_parameters: string[], sql: string, options: SyncRulesOptions) { const parsed = parse(sql, { locationTracking: true }); const rows = new SqlDataQuery(); + const schema = options.schema; if (parsed.length > 1) { throw new SqlRuleError('Only a single SELECT statement is supported', sql, parsed[1]?._location); @@ -50,7 +52,7 @@ export class SqlDataQuery { } const alias: string = tableRef.alias ?? tableRef.name; - const sourceTable = new TablePattern(tableRef.schema, tableRef.name); + const sourceTable = new TablePattern(tableRef.schema ?? options.defaultSchema, tableRef.name); let querySchema: QuerySchema | undefined = undefined; if (schema) { const tables = schema.getTables(sourceTable); diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index f13352177..2b5a4d14a 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -23,6 +23,7 @@ import { SqliteRow } from './types.js'; import { filterJsonRow, getBucketId, isJsonValue, isSelectStatement } from './utils.js'; +import { SyncRulesOptions } from './SqlSyncRules.js'; /** * Represents a parameter query, such as: @@ -34,11 +35,11 @@ export class SqlParameterQuery { static fromSql( descriptor_name: string, sql: string, - schema?: SourceSchema, - options?: QueryParseOptions + options: QueryParseOptions ): SqlParameterQuery | StaticSqlParameterQuery { const parsed = parse(sql, { locationTracking: true }); const rows = new SqlParameterQuery(); + const schema = options?.schema; if (parsed.length > 1) { throw new SqlRuleError('Only a single SELECT statement is supported', sql, parsed[1]?._location); @@ -70,7 +71,7 @@ export class SqlParameterQuery { new SqlRuleError('Table aliases not supported in parameter queries', sql, q.from?.[0]._location) ); } - const sourceTable = new TablePattern(tableRef.schema, tableRef.name); + const sourceTable = new TablePattern(tableRef.schema ?? options.defaultSchema, tableRef.name); let querySchema: QuerySchema | undefined = undefined; if (schema) { const tables = schema.getTables(sourceTable); @@ -139,7 +140,7 @@ export class SqlParameterQuery { rows.tools = tools; rows.errors.push(...tools.errors); - if (rows.usesDangerousRequestParameters && !options?.accept_potentially_dangerous_queries) { + if (rows.usesDangerousRequestParameters && !options.accept_potentially_dangerous_queries) { let err = new SqlRuleError( "Potentially dangerous query based on parameters set by the client. The client can send any value for these parameters so it's not a good place to do authorization.", sql diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index 15b84adc6..d93292b49 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -25,6 +25,16 @@ import { const ACCEPT_POTENTIALLY_DANGEROUS_QUERIES = Symbol('ACCEPT_POTENTIALLY_DANGEROUS_QUERIES'); +export interface SyncRulesOptions { + schema?: SourceSchema; + /** + * 'public' for Postgres, default database for MongoDB/MySQL. + */ + defaultSchema: string; + + throwOnError?: boolean; +} + export class SqlSyncRules implements SyncRules { bucket_descriptors: SqlBucketDescriptor[] = []; idSequence = new IdSequence(); @@ -33,7 +43,7 @@ export class SqlSyncRules implements SyncRules { errors: YamlError[] = []; - static validate(yaml: string, options?: { schema?: SourceSchema }): YamlError[] { + static validate(yaml: string, options: SyncRulesOptions): YamlError[] { try { const rules = this.fromYaml(yaml, options); return rules.errors; @@ -48,9 +58,9 @@ export class SqlSyncRules implements SyncRules { } } - static fromYaml(yaml: string, options?: { throwOnError?: boolean; schema?: SourceSchema }) { - const throwOnError = options?.throwOnError ?? true; - const schema = options?.schema; + static fromYaml(yaml: string, options: SyncRulesOptions) { + const throwOnError = options.throwOnError ?? true; + const schema = options.schema; const lineCounter = new LineCounter(); const parsed = parseDocument(yaml, { @@ -98,7 +108,8 @@ export class SqlSyncRules implements SyncRules { const accept_potentially_dangerous_queries = value.get('accept_potentially_dangerous_queries', true)?.value == true; - const options: QueryParseOptions = { + const queryOptions: QueryParseOptions = { + ...options, accept_potentially_dangerous_queries }; const parameters = value.get('parameters', true) as unknown; @@ -108,16 +119,16 @@ export class SqlSyncRules implements SyncRules { if (parameters instanceof Scalar) { rules.withScalar(parameters, (q) => { - return descriptor.addParameterQuery(q, schema, options); + return descriptor.addParameterQuery(q, queryOptions); }); } else if (parameters instanceof YAMLSeq) { for (let item of parameters.items) { rules.withScalar(item, (q) => { - return descriptor.addParameterQuery(q, schema, options); + return descriptor.addParameterQuery(q, queryOptions); }); } } else { - descriptor.addParameterQuery('SELECT', schema, options); + descriptor.addParameterQuery('SELECT', queryOptions); } if (!(dataQueries instanceof YAMLSeq)) { @@ -126,7 +137,7 @@ export class SqlSyncRules implements SyncRules { } for (let query of dataQueries.items) { rules.withScalar(query, (q) => { - return descriptor.addDataQuery(q, schema); + return descriptor.addDataQuery(q, queryOptions); }); } rules.bucket_descriptors.push(descriptor); diff --git a/packages/sync-rules/src/TablePattern.ts b/packages/sync-rules/src/TablePattern.ts index d6d3494ba..55c90ec9e 100644 --- a/packages/sync-rules/src/TablePattern.ts +++ b/packages/sync-rules/src/TablePattern.ts @@ -1,7 +1,6 @@ import { SourceTableInterface } from './SourceTableInterface.js'; export const DEFAULT_TAG = 'default'; -export const DEFAULT_SCHEMA = 'public'; /** * Some pattern matching SourceTables. @@ -12,8 +11,7 @@ export class TablePattern { public readonly schema: string; public readonly tablePattern: string; - constructor(schema: string | undefined, tablePattern: string) { - schema ??= DEFAULT_SCHEMA; + constructor(schema: string, tablePattern: string) { const splitSchema = schema.split('.'); if (splitSchema.length > 2) { throw new Error(`Invalid schema: ${schema}`); diff --git a/packages/sync-rules/src/types.ts b/packages/sync-rules/src/types.ts index e54d508df..e27489435 100644 --- a/packages/sync-rules/src/types.ts +++ b/packages/sync-rules/src/types.ts @@ -3,6 +3,7 @@ import { SourceTableInterface } from './SourceTableInterface.js'; import { ColumnDefinition, ExpressionType } from './ExpressionType.js'; import { TablePattern } from './TablePattern.js'; import { toSyncRulesParameters } from './utils.js'; +import { SyncRulesOptions } from './SqlSyncRules.js'; export interface SyncRules { evaluateRow(options: EvaluateRowOptions): EvaluationResult[]; @@ -10,7 +11,7 @@ export interface SyncRules { evaluateParameterRow(table: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[]; } -export interface QueryParseOptions { +export interface QueryParseOptions extends SyncRulesOptions { accept_potentially_dangerous_queries?: boolean; } From e36f306336d5708d253188147ff0fa3bfdc3103f Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Sep 2024 16:07:29 +0200 Subject: [PATCH 07/35] Fix merge conflicts. --- packages/service-core/src/storage/mongo/PersistedBatch.ts | 1 - packages/service-core/src/storage/mongo/models.ts | 1 - pnpm-lock.yaml | 8 ++++---- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/service-core/src/storage/mongo/PersistedBatch.ts b/packages/service-core/src/storage/mongo/PersistedBatch.ts index 2d7654687..58b0be659 100644 --- a/packages/service-core/src/storage/mongo/PersistedBatch.ts +++ b/packages/service-core/src/storage/mongo/PersistedBatch.ts @@ -18,7 +18,6 @@ import { } from './models.js'; import { replicaIdToSubkey, serializeLookup } from './util.js'; import { logger } from '@powersync/lib-services-framework'; -import { ReplicaId } from '../BucketStorage.js'; /** * Maximum size of operations we write in a single transaction. diff --git a/packages/service-core/src/storage/mongo/models.ts b/packages/service-core/src/storage/mongo/models.ts index aeaffbebd..fa52a37da 100644 --- a/packages/service-core/src/storage/mongo/models.ts +++ b/packages/service-core/src/storage/mongo/models.ts @@ -1,6 +1,5 @@ import * as bson from 'bson'; import { SqliteJsonValue } from '@powersync/service-sync-rules'; -import { ReplicaId } from '../BucketStorage.js'; /** * Replica id uniquely identifying a row on the source database. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 738908a92..9a1f108ca 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -113,7 +113,7 @@ importers: version: link:../../packages/types mongodb: specifier: ^6.7.0 - version: 6.8.0(socks@2.8.3) + version: 6.7.0(socks@2.8.3) ts-codec: specifier: ^1.2.2 version: 1.2.2 @@ -132,7 +132,7 @@ importers: version: 5.2.2 vite-tsconfig-paths: specifier: ^4.3.2 - version: 4.3.2(typescript@5.2.2)(vite@5.3.3(@types/node@18.11.11)) + version: 4.3.2(typescript@5.2.2)(vite@5.2.11(@types/node@18.11.11)) vitest: specifier: ^0.34.6 version: 0.34.6 @@ -5267,7 +5267,7 @@ snapshots: '@opentelemetry/semantic-conventions': 1.25.0 '@prisma/instrumentation': 5.15.0 '@sentry/core': 8.9.2 - '@sentry/opentelemetry': 8.9.2(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.52.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.0(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.0) + '@sentry/opentelemetry': 8.9.2(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.0(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.0(@opentelemetry/api@1.6.0))(@opentelemetry/sdk-trace-base@1.25.0(@opentelemetry/api@1.6.0))(@opentelemetry/semantic-conventions@1.25.0) '@sentry/types': 8.9.2 '@sentry/utils': 8.9.2 optionalDependencies: @@ -5275,7 +5275,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@sentry/opentelemetry@8.9.2(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.52.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.0(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.0)': + '@sentry/opentelemetry@8.9.2(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.0(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.0(@opentelemetry/api@1.6.0))(@opentelemetry/sdk-trace-base@1.25.0(@opentelemetry/api@1.6.0))(@opentelemetry/semantic-conventions@1.25.0)': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 1.25.0(@opentelemetry/api@1.9.0) From 3facc2f3522523f298a1fa839f96a5637d9567b3 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Sep 2024 16:10:21 +0200 Subject: [PATCH 08/35] Fix Dockerfile. --- service/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/service/Dockerfile b/service/Dockerfile index 369abcc59..715931b11 100644 --- a/service/Dockerfile +++ b/service/Dockerfile @@ -18,6 +18,7 @@ COPY packages/types/package.json packages/types/tsconfig.json packages/types/ COPY libs/lib-services/package.json libs/lib-services/tsconfig.json libs/lib-services/ COPY modules/module-postgres/package.json modules/module-postgres/tsconfig.json modules/module-postgres/ +COPY modules/module-mongodb/package.json modules/module-mongodb/tsconfig.json modules/module-mongodb/ RUN pnpm install --frozen-lockfile @@ -34,6 +35,7 @@ COPY packages/types/src packages/types/src/ COPY libs/lib-services/src libs/lib-services/src/ COPY modules/module-postgres/src modules/module-postgres/src/ +COPY modules/module-mongodb/src modules/module-mongodb/src/ RUN pnpm build:production && \ rm -rf node_modules **/node_modules && \ From 505e728d142d48780dff0becc65b0061365e4ca7 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Sep 2024 18:03:32 +0200 Subject: [PATCH 09/35] Fix and test mongo data type conversion. --- .../src/replication/ChangeStream.ts | 2 +- .../src/replication/MongoRelation.ts | 21 +- modules/module-mongodb/test/src/env.ts | 7 + .../test/src/mongo_test.test.ts | 235 ++++++++++++++++++ modules/module-mongodb/test/src/setup.ts | 7 + modules/module-mongodb/test/src/util.ts | 52 ++++ 6 files changed, 322 insertions(+), 2 deletions(-) create mode 100644 modules/module-mongodb/test/src/env.ts create mode 100644 modules/module-mongodb/test/src/mongo_test.test.ts create mode 100644 modules/module-mongodb/test/src/setup.ts create mode 100644 modules/module-mongodb/test/src/util.ts diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 150cb9970..5bc08ba63 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -182,7 +182,7 @@ export class ChangeStream { static *getQueryData(results: Iterable): Generator { for (let row of results) { - yield toSyncRulesRow(row); + yield constructAfterRecord(row); } } diff --git a/modules/module-mongodb/src/replication/MongoRelation.ts b/modules/module-mongodb/src/replication/MongoRelation.ts index d64e531f2..d3c48ea2b 100644 --- a/modules/module-mongodb/src/replication/MongoRelation.ts +++ b/modules/module-mongodb/src/replication/MongoRelation.ts @@ -56,6 +56,12 @@ export function toMongoSyncRulesValue(data: any): SqliteValue { return data.toHexString(); } else if (data instanceof mongo.UUID) { return data.toHexString(); + } else if (data instanceof Date) { + return data.toISOString().replace('T', ' '); + } else if (data instanceof mongo.Binary) { + return new Uint8Array(data.buffer); + } else if (data instanceof mongo.Long) { + return data.toBigInt(); } else if (Array.isArray(data)) { // We may be able to avoid some parse + stringify cycles here for JsonSqliteContainer. return JSONBig.stringify(data.map((element) => filterJsonData(element))); @@ -77,22 +83,35 @@ export function toMongoSyncRulesValue(data: any): SqliteValue { const DEPTH_LIMIT = 20; function filterJsonData(data: any, depth = 0): any { + const autoBigNum = true; if (depth > DEPTH_LIMIT) { // This is primarily to prevent infinite recursion throw new Error(`json nested object depth exceeds the limit of ${DEPTH_LIMIT}`); } if (data == null) { return data; // null or undefined - } else if (typeof data == 'string' || typeof data == 'number') { + } else if (typeof data == 'string') { return data; + } else if (typeof data == 'number') { + if (autoBigNum && Number.isInteger(data)) { + return BigInt(data); + } else { + return data; + } } else if (typeof data == 'boolean') { return data ? 1n : 0n; } else if (typeof data == 'bigint') { return data; + } else if (data instanceof Date) { + return data.toISOString().replace('T', ' '); } else if (data instanceof mongo.ObjectId) { return data.toHexString(); } else if (data instanceof mongo.UUID) { return data.toHexString(); + } else if (data instanceof mongo.Binary) { + return undefined; + } else if (data instanceof mongo.Long) { + return data.toBigInt(); } else if (Array.isArray(data)) { return data.map((element) => filterJsonData(element, depth + 1)); } else if (ArrayBuffer.isView(data)) { diff --git a/modules/module-mongodb/test/src/env.ts b/modules/module-mongodb/test/src/env.ts new file mode 100644 index 000000000..7d72df3a9 --- /dev/null +++ b/modules/module-mongodb/test/src/env.ts @@ -0,0 +1,7 @@ +import { utils } from '@powersync/lib-services-framework'; + +export const env = utils.collectEnvironmentVariables({ + MONGO_TEST_DATA_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'), + CI: utils.type.boolean.default('false'), + SLOW_TESTS: utils.type.boolean.default('false') +}); diff --git a/modules/module-mongodb/test/src/mongo_test.test.ts b/modules/module-mongodb/test/src/mongo_test.test.ts new file mode 100644 index 000000000..e4126c484 --- /dev/null +++ b/modules/module-mongodb/test/src/mongo_test.test.ts @@ -0,0 +1,235 @@ +import { ChangeStream } from '@module/replication/ChangeStream.js'; +import * as mongo from 'mongodb'; +import { describe, expect, test } from 'vitest'; +import { clearTestDb, connectMongoData } from './util.js'; + +describe('mongo data types', () => { + async function setupTable(db: mongo.Db) { + await clearTestDb(db); + } + + async function insert(collection: mongo.Collection) { + await collection.insertMany([ + { + _id: 1 as any, + null: null, + text: 'text', + uuid: new mongo.UUID('baeb2514-4c57-436d-b3cc-c1256211656d'), + bool: true, + bytea: Buffer.from('test'), + int2: 1000, + int4: 1000000, + int8: 9007199254740993n, + float: 3.14 + }, + { _id: 2 as any, nested: { test: 'thing' } }, + { _id: 3 as any, date: new Date('2023-03-06 15:47+02') }, + { + _id: 4 as any, + timestamp: mongo.Timestamp.fromBits(123, 456), + objectId: mongo.ObjectId.createFromHexString('66e834cc91d805df11fa0ecb') + } + ]); + } + + async function insertNested(collection: mongo.Collection) { + await collection.insertMany([ + { + _id: 1 as any, + null: [null], + text: ['text'], + uuid: [new mongo.UUID('baeb2514-4c57-436d-b3cc-c1256211656d')], + bool: [true], + bytea: [Buffer.from('test')], + int2: [1000], + int4: [1000000], + int8: [9007199254740993n], + float: [3.14] + }, + { _id: 2 as any, nested: [{ test: 'thing' }] }, + { _id: 3 as any, date: [new Date('2023-03-06 15:47+02')] }, + { + _id: 10 as any, + timestamp: [mongo.Timestamp.fromBits(123, 456)], + objectId: [mongo.ObjectId.createFromHexString('66e834cc91d805df11fa0ecb')] + } + ]); + } + + function checkResults(transformed: Record[]) { + expect(transformed[0]).toMatchObject({ + _id: 1n, + text: 'text', + uuid: 'baeb2514-4c57-436d-b3cc-c1256211656d', + bool: 1n, + bytea: new Uint8Array([116, 101, 115, 116]), + int2: 1000n, + int4: 1000000n, + int8: 9007199254740993n, + float: 3.14, + null: null + }); + expect(transformed[1]).toMatchObject({ + _id: 2n, + nested: '{"test":"thing"}' + }); + + expect(transformed[2]).toMatchObject({ + _id: 3n, + date: '2023-03-06 13:47:00.000Z' + }); + + expect(transformed[3]).toMatchObject({ + _id: 4n, + objectId: '66e834cc91d805df11fa0ecb', + timestamp: 1958505087099n + }); + } + + function checkResultsNested(transformed: Record[]) { + expect(transformed[0]).toMatchObject({ + _id: 1n, + text: `["text"]`, + uuid: '["baeb2514-4c57-436d-b3cc-c1256211656d"]', + bool: '[1]', + bytea: '[null]', + int2: '[1000]', + int4: '[1000000]', + int8: `[9007199254740993]`, + float: '[3.14]', + null: '[null]' + }); + + // Note: Depending on to what extent we use the original postgres value, the whitespace may change, and order may change. + // We do expect that decimals and big numbers are preserved. + expect(transformed[1]).toMatchObject({ + _id: 2n, + nested: '[{"test":"thing"}]' + }); + + expect(transformed[2]).toMatchObject({ + _id: 3n, + date: '["2023-03-06 13:47:00.000Z"]' + }); + + expect(transformed[3]).toMatchObject({ + _id: 10n, + objectId: '["66e834cc91d805df11fa0ecb"]', + timestamp: '[1958505087099]' + }); + } + + test('test direct queries', async () => { + const { db, client } = await connectMongoData(); + const collection = db.collection('test_data'); + try { + await setupTable(db); + + await insert(collection); + + const transformed = [...ChangeStream.getQueryData(await db.collection('test_data').find().toArray())]; + + checkResults(transformed); + } finally { + await client.close(); + } + }); + + test('test direct queries - arrays', async () => { + const { db, client } = await connectMongoData(); + const collection = db.collection('test_data_arrays'); + try { + await setupTable(db); + + await insertNested(collection); + + const transformed = [...ChangeStream.getQueryData(await db.collection('test_data_arrays').find().toArray())]; + + checkResultsNested(transformed); + } finally { + await client.close(); + } + }); + + // test('test replication', async () => { + // const db = await connectPgPool(); + // try { + // await setupTable(db); + + // const slotName = 'test_slot'; + + // await db.query({ + // statement: 'SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = $1', + // params: [{ type: 'varchar', value: slotName }] + // }); + + // await db.query({ + // statement: `SELECT slot_name, lsn FROM pg_catalog.pg_create_logical_replication_slot($1, 'pgoutput')`, + // params: [{ type: 'varchar', value: slotName }] + // }); + + // await insert(db); + + // const pg: pgwire.PgConnection = await pgwire.pgconnect({ replication: 'database' }, TEST_URI); + // const replicationStream = await pg.logicalReplication({ + // slot: slotName, + // options: { + // proto_version: '1', + // publication_names: 'powersync' + // } + // }); + + // const transformed = await getReplicationTx(replicationStream); + // await pg.end(); + + // checkResults(transformed); + // } finally { + // await db.end(); + // } + // }); + + // test('test replication - arrays', async () => { + // const db = await connectPgPool(); + // try { + // await setupTable(db); + + // const slotName = 'test_slot'; + + // await db.query({ + // statement: 'SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = $1', + // params: [{ type: 'varchar', value: slotName }] + // }); + + // await db.query({ + // statement: `SELECT slot_name, lsn FROM pg_catalog.pg_create_logical_replication_slot($1, 'pgoutput')`, + // params: [{ type: 'varchar', value: slotName }] + // }); + + // await insertArrays(db); + + // const pg: pgwire.PgConnection = await pgwire.pgconnect({ replication: 'database' }, TEST_URI); + // const replicationStream = await pg.logicalReplication({ + // slot: slotName, + // options: { + // proto_version: '1', + // publication_names: 'powersync' + // } + // }); + + // const transformed = await getReplicationTx(replicationStream); + // await pg.end(); + + // checkResultArrays(transformed); + // } finally { + // await db.end(); + // } + // }); + + test.skip('schema', async function () { + // const db = await connectPgWire(); + // await setupTable(db); + // TODO need a test for adapter + // const schema = await api.getConnectionsSchema(db); + // expect(schema).toMatchSnapshot(); + }); +}); diff --git a/modules/module-mongodb/test/src/setup.ts b/modules/module-mongodb/test/src/setup.ts new file mode 100644 index 000000000..b924cf736 --- /dev/null +++ b/modules/module-mongodb/test/src/setup.ts @@ -0,0 +1,7 @@ +import { container } from '@powersync/lib-services-framework'; +import { beforeAll } from 'vitest'; + +beforeAll(() => { + // Executes for every test file + container.registerDefaults(); +}); diff --git a/modules/module-mongodb/test/src/util.ts b/modules/module-mongodb/test/src/util.ts new file mode 100644 index 000000000..a101f77a5 --- /dev/null +++ b/modules/module-mongodb/test/src/util.ts @@ -0,0 +1,52 @@ +import * as types from '@module/types/types.js'; +import { BucketStorageFactory, Metrics, MongoBucketStorage, OpId } from '@powersync/service-core'; + +import { env } from './env.js'; +import { logger } from '@powersync/lib-services-framework'; +import { connectMongo } from '@core-tests/util.js'; +import * as mongo from 'mongodb'; + +// The metrics need to be initialized before they can be used +await Metrics.initialise({ + disable_telemetry_sharing: true, + powersync_instance_id: 'test', + internal_metrics_endpoint: 'unused.for.tests.com' +}); +Metrics.getInstance().resetCounters(); + +export const TEST_URI = env.MONGO_TEST_DATA_URL; + +export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({ + type: 'mongodb', + uri: TEST_URI +}); + +export type StorageFactory = () => Promise; + +export const INITIALIZED_MONGO_STORAGE_FACTORY: StorageFactory = async () => { + const db = await connectMongo(); + + // None of the PG tests insert data into this collection, so it was never created + if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) { + await db.db.createCollection('bucket_parameters'); + } + + await db.clear(); + + return new MongoBucketStorage(db, { slot_name_prefix: 'test_' }); +}; + +export async function clearTestDb(db: mongo.Db) { + await db.dropDatabase(); +} + +export async function connectMongoData() { + const client = new mongo.MongoClient(env.MONGO_TEST_DATA_URL, { + connectTimeoutMS: env.CI ? 15_000 : 5_000, + socketTimeoutMS: env.CI ? 15_000 : 5_000, + serverSelectionTimeoutMS: env.CI ? 15_000 : 2_500, + useBigInt64: true + }); + const dbname = new URL(env.MONGO_TEST_DATA_URL).pathname.substring(1); + return { client, db: client.db(dbname) }; +} From ae89a0d45da8443fc49d644c8ab15f21447162cb Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 17 Sep 2024 08:40:59 +0200 Subject: [PATCH 10/35] More mongo format tests. --- .../test/src/mongo_test.test.ts | 144 ++++++++---------- 1 file changed, 64 insertions(+), 80 deletions(-) diff --git a/modules/module-mongodb/test/src/mongo_test.test.ts b/modules/module-mongodb/test/src/mongo_test.test.ts index e4126c484..1b36dab90 100644 --- a/modules/module-mongodb/test/src/mongo_test.test.ts +++ b/modules/module-mongodb/test/src/mongo_test.test.ts @@ -2,6 +2,8 @@ import { ChangeStream } from '@module/replication/ChangeStream.js'; import * as mongo from 'mongodb'; import { describe, expect, test } from 'vitest'; import { clearTestDb, connectMongoData } from './util.js'; +import { SqliteRow } from '@powersync/service-sync-rules'; +import { constructAfterRecord } from '@module/replication/MongoRelation.js'; describe('mongo data types', () => { async function setupTable(db: mongo.Db) { @@ -151,85 +153,67 @@ describe('mongo data types', () => { } }); - // test('test replication', async () => { - // const db = await connectPgPool(); - // try { - // await setupTable(db); - - // const slotName = 'test_slot'; - - // await db.query({ - // statement: 'SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = $1', - // params: [{ type: 'varchar', value: slotName }] - // }); - - // await db.query({ - // statement: `SELECT slot_name, lsn FROM pg_catalog.pg_create_logical_replication_slot($1, 'pgoutput')`, - // params: [{ type: 'varchar', value: slotName }] - // }); - - // await insert(db); - - // const pg: pgwire.PgConnection = await pgwire.pgconnect({ replication: 'database' }, TEST_URI); - // const replicationStream = await pg.logicalReplication({ - // slot: slotName, - // options: { - // proto_version: '1', - // publication_names: 'powersync' - // } - // }); - - // const transformed = await getReplicationTx(replicationStream); - // await pg.end(); - - // checkResults(transformed); - // } finally { - // await db.end(); - // } - // }); - - // test('test replication - arrays', async () => { - // const db = await connectPgPool(); - // try { - // await setupTable(db); - - // const slotName = 'test_slot'; - - // await db.query({ - // statement: 'SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = $1', - // params: [{ type: 'varchar', value: slotName }] - // }); - - // await db.query({ - // statement: `SELECT slot_name, lsn FROM pg_catalog.pg_create_logical_replication_slot($1, 'pgoutput')`, - // params: [{ type: 'varchar', value: slotName }] - // }); - - // await insertArrays(db); - - // const pg: pgwire.PgConnection = await pgwire.pgconnect({ replication: 'database' }, TEST_URI); - // const replicationStream = await pg.logicalReplication({ - // slot: slotName, - // options: { - // proto_version: '1', - // publication_names: 'powersync' - // } - // }); - - // const transformed = await getReplicationTx(replicationStream); - // await pg.end(); - - // checkResultArrays(transformed); - // } finally { - // await db.end(); - // } - // }); - - test.skip('schema', async function () { - // const db = await connectPgWire(); - // await setupTable(db); - // TODO need a test for adapter - // const schema = await api.getConnectionsSchema(db); - // expect(schema).toMatchSnapshot(); + test('test replication', async () => { + // With MongoDB, replication uses the exact same document format + // as normal queries. We test it anyway. + const { db, client } = await connectMongoData(); + const collection = db.collection('test_data'); + try { + await setupTable(db); + + const stream = db.watch([], { + useBigInt64: true, + maxAwaitTimeMS: 50, + fullDocument: 'updateLookup' + }); + + await stream.tryNext(); + + await insert(collection); + + const transformed = await getReplicationTx(stream, 4); + + checkResults(transformed); + } finally { + await client.close(); + } + }); + + test('test replication - arrays', async () => { + const { db, client } = await connectMongoData(); + const collection = db.collection('test_data'); + try { + await setupTable(db); + + const stream = db.watch([], { + useBigInt64: true, + maxAwaitTimeMS: 50, + fullDocument: 'updateLookup' + }); + + await stream.tryNext(); + + await insertNested(collection); + + const transformed = await getReplicationTx(stream, 4); + + checkResultsNested(transformed); + } finally { + await client.close(); + } }); }); + +/** + * Return all the inserts from the first transaction in the replication stream. + */ +async function getReplicationTx(replicationStream: mongo.ChangeStream, count: number) { + let transformed: SqliteRow[] = []; + for await (const doc of replicationStream) { + transformed.push(constructAfterRecord((doc as any).fullDocument)); + if (transformed.length == count) { + break; + } + } + return transformed; +} From a2857bf82d9b4df1d48b72bc1cda5ee174c8badd Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 17 Sep 2024 11:59:58 +0200 Subject: [PATCH 11/35] Fix initial snapshot; initial change-stream tests. --- .../src/api/MongoRouteAPIAdapter.ts | 6 +- .../src/replication/ChangeStream.ts | 11 +- .../src/replication/MongoRelation.ts | 23 ++ .../test/src/change_stream.test.ts | 301 ++++++++++++++++++ .../test/src/change_stream_utils.ts | 145 +++++++++ .../service-core/src/storage/BucketStorage.ts | 11 +- .../src/storage/mongo/MongoBucketBatch.ts | 27 +- 7 files changed, 502 insertions(+), 22 deletions(-) create mode 100644 modules/module-mongodb/test/src/change_stream.test.ts create mode 100644 modules/module-mongodb/test/src/change_stream_utils.ts diff --git a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts index 3d862db5e..3389b2eb1 100644 --- a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts +++ b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts @@ -5,9 +5,11 @@ import * as sync_rules from '@powersync/service-sync-rules'; import * as service_types from '@powersync/service-types'; import * as types from '../types/types.js'; import { MongoManager } from '../replication/MongoManager.js'; +import { createCheckpoint, getMongoLsn } from '../replication/MongoRelation.js'; export class MongoRouteAPIAdapter implements api.RouteAPI { protected client: mongo.MongoClient; + private db: mongo.Db; connectionTag: string; defaultSchema: string; @@ -15,6 +17,7 @@ export class MongoRouteAPIAdapter implements api.RouteAPI { constructor(protected config: types.ResolvedConnectionConfig) { const manager = new MongoManager(config); this.client = manager.client; + this.db = manager.db; this.defaultSchema = manager.db.databaseName; this.connectionTag = config.tag ?? sync_rules.DEFAULT_TAG; } @@ -72,8 +75,7 @@ export class MongoRouteAPIAdapter implements api.RouteAPI { } async getReplicationHead(): Promise { - // TODO: implement - return ''; + return createCheckpoint(this.db); } async getConnectionSchema(): Promise { diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 5bc08ba63..03a267afd 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -7,7 +7,7 @@ import { constructAfterRecord, getMongoLsn, getMongoRelation, mongoLsnToTimestam export const ZERO_LSN = '0000000000000000'; -export interface WalStreamOptions { +export interface ChangeStreamOptions { connections: MongoManager; storage: storage.SyncRulesBucketStorage; abort_signal: AbortSignal; @@ -39,7 +39,7 @@ export class ChangeStream { private relation_cache = new Map(); - constructor(options: WalStreamOptions) { + constructor(options: ChangeStreamOptions) { this.storage = options.storage; this.group_id = options.storage.group_id; this.connections = options.connections; @@ -147,7 +147,7 @@ export class ChangeStream { if (time != null) { const lsn = getMongoLsn(time.clusterTime); logger.info(`Snapshot commit at ${time.clusterTime.inspect()} / ${lsn}`); - await batch.commit(lsn); + await batch.commit(lsn, { forceCommit: true }); } else { logger.info(`No snapshot clusterTime (no snapshot data?) - skipping commit.`); } @@ -161,7 +161,7 @@ export class ChangeStream { private getSourceNamespaceFilters() { const sourceTables = this.sync_rules.getSourceTables(); - let filters: any[] = []; + let filters: any[] = [{ db: this.defaultDb.databaseName, collection: '_powersync_checkpoints' }]; for (let tablePattern of sourceTables) { if (tablePattern.connectionTag != this.connections.connectionTag) { continue; @@ -337,9 +337,6 @@ export class ChangeStream { // TODO: Use changeStreamSplitLargeEvent - const nsFilter = this.getSourceNamespaceFilters(); - nsFilter.$in.push({ ns: nsFilter }); - const pipeline: mongo.Document[] = [ { $match: { diff --git a/modules/module-mongodb/src/replication/MongoRelation.ts b/modules/module-mongodb/src/replication/MongoRelation.ts index d3c48ea2b..8a83f5803 100644 --- a/modules/module-mongodb/src/replication/MongoRelation.ts +++ b/modules/module-mongodb/src/replication/MongoRelation.ts @@ -129,3 +129,26 @@ function filterJsonData(data: any, depth = 0): any { return undefined; } } + +export async function createCheckpoint(db: mongo.Db): Promise { + const pingResult = await db.command({ ping: 1 }); + + const time: mongo.Timestamp = pingResult.$clusterTime.clusterTime; + + const result = await db.collection('_powersync_checkpoints').findOneAndUpdate( + { + _id: 'checkpoint' as any + }, + { + $inc: { i: 1 } + }, + { + upsert: true, + returnDocument: 'after' + } + ); + + // TODO: Use the above when we support custom write checkpoints + + return getMongoLsn(time); +} diff --git a/modules/module-mongodb/test/src/change_stream.test.ts b/modules/module-mongodb/test/src/change_stream.test.ts new file mode 100644 index 000000000..c0f144a1b --- /dev/null +++ b/modules/module-mongodb/test/src/change_stream.test.ts @@ -0,0 +1,301 @@ +import { putOp, removeOp } from '@core-tests/stream_utils.js'; +import { MONGO_STORAGE_FACTORY } from '@core-tests/util.js'; +import { BucketStorageFactory, Metrics } from '@powersync/service-core'; +import * as crypto from 'crypto'; +import { describe, expect, test } from 'vitest'; +import { walStreamTest } from './change_stream_utils.js'; + +type StorageFactory = () => Promise; + +const BASIC_SYNC_RULES = ` +bucket_definitions: + global: + data: + - SELECT id, description FROM "test_data" +`; + +describe( + 'wal stream - mongodb', + function () { + defineWalStreamTests(MONGO_STORAGE_FACTORY); + }, + { timeout: 20_000 } +); + +function defineWalStreamTests(factory: StorageFactory) { + test.only( + 'replicating basic values', + walStreamTest(factory, async (context) => { + const { db } = context; + await context.updateSyncRules(` +bucket_definitions: + global: + data: + - SELECT _id as id, description, num FROM "test_data"`); + + await context.replicateSnapshot(); + + context.startStreaming(); + + const collection = db.collection('test_data'); + const result = await collection.insertOne({ description: 'test1', num: 1152921504606846976n }); + const test_id = result.insertedId; + + const data = await context.getBucketData('global[]'); + + expect(data).toMatchObject([ + putOp('test_data', { id: test_id.toHexString(), description: 'test1', num: 1152921504606846976n }) + ]); + }) + ); + + test( + 'replicating case sensitive table', + walStreamTest(factory, async (context) => { + const { pool } = context; + await context.updateSyncRules(` + bucket_definitions: + global: + data: + - SELECT id, description FROM "test_DATA" + `); + + await pool.query(`DROP TABLE IF EXISTS "test_DATA"`); + await pool.query(`CREATE TABLE "test_DATA"(id uuid primary key default uuid_generate_v4(), description text)`); + + await context.replicateSnapshot(); + + const startRowCount = + (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; + const startTxCount = + (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + + context.startStreaming(); + + const [{ test_id }] = pgwireRows( + await pool.query(`INSERT INTO "test_DATA"(description) VALUES('test1') returning id as test_id`) + ); + + const data = await context.getBucketData('global[]'); + + expect(data).toMatchObject([putOp('test_DATA', { id: test_id, description: 'test1' })]); + const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; + const endTxCount = + (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + expect(endRowCount - startRowCount).toEqual(1); + expect(endTxCount - startTxCount).toEqual(1); + }) + ); + + test( + 'replicating TOAST values', + walStreamTest(factory, async (context) => { + const { pool } = context; + await context.updateSyncRules(` + bucket_definitions: + global: + data: + - SELECT id, name, description FROM "test_data" + `); + + await pool.query(`DROP TABLE IF EXISTS test_data`); + await pool.query( + `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), name text, description text)` + ); + + await context.replicateSnapshot(); + context.startStreaming(); + + // Must be > 8kb after compression + const largeDescription = crypto.randomBytes(20_000).toString('hex'); + const [{ test_id }] = pgwireRows( + await pool.query({ + statement: `INSERT INTO test_data(name, description) VALUES('test1', $1) returning id as test_id`, + params: [{ type: 'varchar', value: largeDescription }] + }) + ); + + await pool.query(`UPDATE test_data SET name = 'test2' WHERE id = '${test_id}'`); + + const data = await context.getBucketData('global[]'); + expect(data.slice(0, 1)).toMatchObject([ + putOp('test_data', { id: test_id, name: 'test1', description: largeDescription }) + ]); + expect(data.slice(1)).toMatchObject([ + putOp('test_data', { id: test_id, name: 'test2', description: largeDescription }) + ]); + }) + ); + + test( + 'replicating TRUNCATE', + walStreamTest(factory, async (context) => { + const { pool } = context; + const syncRuleContent = ` +bucket_definitions: + global: + data: + - SELECT id, description FROM "test_data" + by_test_data: + parameters: SELECT id FROM test_data WHERE id = token_parameters.user_id + data: [] +`; + await context.updateSyncRules(syncRuleContent); + await pool.query(`DROP TABLE IF EXISTS test_data`); + await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`); + + await context.replicateSnapshot(); + context.startStreaming(); + + const [{ test_id }] = pgwireRows( + await pool.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`) + ); + await pool.query(`TRUNCATE test_data`); + + const data = await context.getBucketData('global[]'); + + expect(data).toMatchObject([ + putOp('test_data', { id: test_id, description: 'test1' }), + removeOp('test_data', test_id) + ]); + }) + ); + + test( + 'replicating changing primary key', + walStreamTest(factory, async (context) => { + const { pool } = context; + await context.updateSyncRules(BASIC_SYNC_RULES); + await pool.query(`DROP TABLE IF EXISTS test_data`); + await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`); + + await context.replicateSnapshot(); + context.startStreaming(); + + const [{ test_id }] = pgwireRows( + await pool.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`) + ); + + const [{ test_id: test_id2 }] = pgwireRows( + await pool.query( + `UPDATE test_data SET id = uuid_generate_v4(), description = 'test2a' WHERE id = '${test_id}' returning id as test_id` + ) + ); + + // This update may fail replicating with: + // Error: Update on missing record public.test_data:074a601e-fc78-4c33-a15d-f89fdd4af31d :: {"g":1,"t":"651e9fbe9fec6155895057ec","k":"1a0b34da-fb8c-5e6f-8421-d7a3c5d4df4f"} + await pool.query(`UPDATE test_data SET description = 'test2b' WHERE id = '${test_id2}'`); + + // Re-use old id again + await pool.query(`INSERT INTO test_data(id, description) VALUES('${test_id}', 'test1b')`); + await pool.query(`UPDATE test_data SET description = 'test1c' WHERE id = '${test_id}'`); + + const data = await context.getBucketData('global[]'); + expect(data).toMatchObject([ + // Initial insert + putOp('test_data', { id: test_id, description: 'test1' }), + // Update id, then description + removeOp('test_data', test_id), + putOp('test_data', { id: test_id2, description: 'test2a' }), + putOp('test_data', { id: test_id2, description: 'test2b' }), + // Re-use old id + putOp('test_data', { id: test_id, description: 'test1b' }), + putOp('test_data', { id: test_id, description: 'test1c' }) + ]); + }) + ); + + test( + 'initial sync', + walStreamTest(factory, async (context) => { + const { pool } = context; + await context.updateSyncRules(BASIC_SYNC_RULES); + + await pool.query(`DROP TABLE IF EXISTS test_data`); + await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`); + + const [{ test_id }] = pgwireRows( + await pool.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`) + ); + + await context.replicateSnapshot(); + context.startStreaming(); + + const data = await context.getBucketData('global[]'); + expect(data).toMatchObject([putOp('test_data', { id: test_id, description: 'test1' })]); + }) + ); + + test( + 'record too large', + walStreamTest(factory, async (context) => { + await context.updateSyncRules(`bucket_definitions: + global: + data: + - SELECT id, description, other FROM "test_data"`); + const { pool } = context; + + await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`); + + await context.replicateSnapshot(); + + // 4MB + const largeDescription = crypto.randomBytes(2_000_000).toString('hex'); + // 18MB + const tooLargeDescription = crypto.randomBytes(9_000_000).toString('hex'); + + await pool.query({ + statement: `INSERT INTO test_data(id, description, other) VALUES('t1', $1, 'foo')`, + params: [{ type: 'varchar', value: tooLargeDescription }] + }); + await pool.query({ + statement: `UPDATE test_data SET description = $1 WHERE id = 't1'`, + params: [{ type: 'varchar', value: largeDescription }] + }); + + context.startStreaming(); + + const data = await context.getBucketData('global[]'); + expect(data.length).toEqual(1); + const row = JSON.parse(data[0].data as string); + delete row.description; + expect(row).toEqual({ id: 't1', other: 'foo' }); + delete data[0].data; + expect(data[0]).toMatchObject({ object_id: 't1', object_type: 'test_data', op: 'PUT', op_id: '1' }); + }) + ); + + test( + 'table not in sync rules', + walStreamTest(factory, async (context) => { + const { pool } = context; + await context.updateSyncRules(BASIC_SYNC_RULES); + + await pool.query(`CREATE TABLE test_donotsync(id uuid primary key default uuid_generate_v4(), description text)`); + + await context.replicateSnapshot(); + + const startRowCount = + (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; + const startTxCount = + (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + + context.startStreaming(); + + const [{ test_id }] = pgwireRows( + await pool.query(`INSERT INTO test_donotsync(description) VALUES('test1') returning id as test_id`) + ); + + const data = await context.getBucketData('global[]'); + + expect(data).toMatchObject([]); + const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; + const endTxCount = + (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + + // There was a transaction, but we should not replicate any actual data + expect(endRowCount - startRowCount).toEqual(0); + expect(endTxCount - startTxCount).toEqual(1); + }) + ); +} diff --git a/modules/module-mongodb/test/src/change_stream_utils.ts b/modules/module-mongodb/test/src/change_stream_utils.ts new file mode 100644 index 000000000..fedfb536c --- /dev/null +++ b/modules/module-mongodb/test/src/change_stream_utils.ts @@ -0,0 +1,145 @@ +import { BucketStorageFactory, OpId, SyncRulesBucketStorage } from '@powersync/service-core'; + +import { TEST_CONNECTION_OPTIONS, clearTestDb } from './util.js'; +import { fromAsync } from '@core-tests/stream_utils.js'; +import { MongoManager } from '@module/replication/MongoManager.js'; +import { ChangeStream, ChangeStreamOptions } from '@module/replication/ChangeStream.js'; +import * as mongo from 'mongodb'; +import { createCheckpoint } from '@module/replication/MongoRelation.js'; + +/** + * Tests operating on the wal stream need to configure the stream and manage asynchronous + * replication, which gets a little tricky. + * + * This wraps a test in a function that configures all the context, and tears it down afterwards. + */ +export function walStreamTest( + factory: () => Promise, + test: (context: ChangeStreamTestContext) => Promise +): () => Promise { + return async () => { + const f = await factory(); + const connectionManager = new MongoManager(TEST_CONNECTION_OPTIONS); + + await clearTestDb(connectionManager.db); + const context = new ChangeStreamTestContext(f, connectionManager); + try { + await test(context); + } finally { + await context.dispose(); + } + }; +} + +export class ChangeStreamTestContext { + private _walStream?: ChangeStream; + private abortController = new AbortController(); + private streamPromise?: Promise; + public storage?: SyncRulesBucketStorage; + + constructor(public factory: BucketStorageFactory, public connectionManager: MongoManager) {} + + async dispose() { + this.abortController.abort(); + await this.streamPromise?.catch((e) => e); + await this.connectionManager.destroy(); + } + + get client() { + return this.connectionManager.client; + } + + get db() { + return this.connectionManager.db; + } + + get connectionTag() { + return this.connectionManager.connectionTag; + } + + async updateSyncRules(content: string) { + const syncRules = await this.factory.updateSyncRules({ content: content }); + this.storage = this.factory.getInstance(syncRules); + return this.storage!; + } + + get walStream() { + if (this.storage == null) { + throw new Error('updateSyncRules() first'); + } + if (this._walStream) { + return this._walStream; + } + const options: ChangeStreamOptions = { + storage: this.storage, + connections: this.connectionManager, + abort_signal: this.abortController.signal + }; + this._walStream = new ChangeStream(options); + return this._walStream!; + } + + async replicateSnapshot() { + await this.walStream.initReplication(); + await this.storage!.autoActivate(); + } + + startStreaming() { + this.streamPromise = this.walStream.streamChanges(); + } + + async getCheckpoint(options?: { timeout?: number }) { + let checkpoint = await Promise.race([ + getClientCheckpoint(this.db, this.factory, { timeout: options?.timeout ?? 15_000 }), + this.streamPromise + ]); + if (typeof checkpoint == undefined) { + // This indicates an issue with the test setup - streamingPromise completed instead + // of getClientCheckpoint() + throw new Error('Test failure - streamingPromise completed'); + } + return checkpoint as string; + } + + async getBucketsDataBatch(buckets: Record, options?: { timeout?: number }) { + let checkpoint = await this.getCheckpoint(options); + const map = new Map(Object.entries(buckets)); + return fromAsync(this.storage!.getBucketDataBatch(checkpoint, map)); + } + + async getBucketData(bucket: string, start?: string, options?: { timeout?: number }) { + start ??= '0'; + let checkpoint = await this.getCheckpoint(options); + const map = new Map([[bucket, start]]); + const batch = this.storage!.getBucketDataBatch(checkpoint, map); + const batches = await fromAsync(batch); + return batches[0]?.batch.data ?? []; + } +} + +export async function getClientCheckpoint( + db: mongo.Db, + bucketStorage: BucketStorageFactory, + options?: { timeout?: number } +): Promise { + const start = Date.now(); + const lsn = await createCheckpoint(db); + // This old API needs a persisted checkpoint id. + // Since we don't use LSNs anymore, the only way to get that is to wait. + + const timeout = options?.timeout ?? 5_00; + + while (Date.now() - start < timeout) { + const cp = await bucketStorage.getActiveCheckpoint(); + if (!cp.hasSyncRules()) { + throw new Error('No sync rules available'); + } + if (cp.lsn && cp.lsn >= lsn) { + return cp.checkpoint; + } + + await new Promise((resolve) => setTimeout(resolve, 30)); + } + + throw new Error('Timeout while waiting for checkpoint'); +} diff --git a/packages/service-core/src/storage/BucketStorage.ts b/packages/service-core/src/storage/BucketStorage.ts index 373be8f33..aface2594 100644 --- a/packages/service-core/src/storage/BucketStorage.ts +++ b/packages/service-core/src/storage/BucketStorage.ts @@ -328,7 +328,7 @@ export interface BucketStorageBatch { * * Only call this after a transaction. */ - commit(lsn: string): Promise; + commit(lsn: string, options?: CommitOptions): Promise; /** * Advance the checkpoint LSN position, without any associated op. @@ -347,6 +347,15 @@ export interface BucketStorageBatch { markSnapshotDone(tables: SourceTable[], no_checkpoint_before_lsn: string): Promise; } +export interface CommitOptions { + /** + * Usually, a commit only takes effect if there are operations to commit. + * + * Setting this to true forces the commit to take place. + */ + forceCommit?: boolean; +} + export interface SaveParameterData { sourceTable: SourceTable; /** UUID */ diff --git a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts index d82759029..1d42a6494 100644 --- a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts +++ b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts @@ -4,10 +4,10 @@ import * as mongo from 'mongodb'; import { container, errors, logger } from '@powersync/lib-services-framework'; import * as util from '../../util/util-index.js'; -import { BucketStorageBatch, FlushedResult, mergeToast, SaveOptions } from '../BucketStorage.js'; +import { BucketStorageBatch, CommitOptions, FlushedResult, mergeToast, SaveOptions } from '../BucketStorage.js'; import { SourceTable } from '../SourceTable.js'; import { PowerSyncMongo } from './db.js'; -import { CurrentBucket, CurrentDataDocument, SourceKey } from './models.js'; +import { CurrentBucket, CurrentDataDocument, SourceKey, SyncRuleDocument } from './models.js'; import { MongoIdSequence } from './MongoIdSequence.js'; import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js'; import { PersistedBatch } from './PersistedBatch.js'; @@ -536,7 +536,7 @@ export class MongoBucketBatch implements BucketStorageBatch { await this.session.endSession(); } - async commit(lsn: string): Promise { + async commit(lsn: string, options?: CommitOptions): Promise { await this.flush(); if (this.last_checkpoint_lsn != null && lsn < this.last_checkpoint_lsn) { @@ -550,21 +550,24 @@ export class MongoBucketBatch implements BucketStorageBatch { return false; } - if (this.persisted_op != null) { + if (this.persisted_op != null || options?.forceCommit) { const now = new Date(); + let setValues: mongo.MatchKeysAndValues = { + last_checkpoint_lsn: lsn, + last_checkpoint_ts: now, + last_keepalive_ts: now, + snapshot_done: true, + last_fatal_error: null + }; + if (this.persisted_op != null) { + (setValues as any).last_checkpoint = this.persisted_op; + } await this.db.sync_rules.updateOne( { _id: this.group_id }, { - $set: { - last_checkpoint: this.persisted_op, - last_checkpoint_lsn: lsn, - last_checkpoint_ts: now, - last_keepalive_ts: now, - snapshot_done: true, - last_fatal_error: null - } + $set: setValues }, { session: this.session } ); From a9da5217f91f8d944f3fbdfb15e61a6cde42ece5 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 17 Sep 2024 12:18:51 +0200 Subject: [PATCH 12/35] Record keepalive events. --- .../src/replication/ChangeStream.ts | 10 +- .../src/replication/MongoRelation.ts | 1 - .../test/src/change_stream.test.ts | 133 +++++------------- .../service-core/src/storage/BucketStorage.ts | 11 +- .../src/storage/mongo/MongoBucketBatch.ts | 25 ++-- 5 files changed, 51 insertions(+), 129 deletions(-) diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 03a267afd..0cfbd8a6e 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -147,7 +147,8 @@ export class ChangeStream { if (time != null) { const lsn = getMongoLsn(time.clusterTime); logger.info(`Snapshot commit at ${time.clusterTime.inspect()} / ${lsn}`); - await batch.commit(lsn, { forceCommit: true }); + await batch.commit(lsn); + await batch.keepalive(lsn); } else { logger.info(`No snapshot clusterTime (no snapshot data?) - skipping commit.`); } @@ -161,7 +162,7 @@ export class ChangeStream { private getSourceNamespaceFilters() { const sourceTables = this.sync_rules.getSourceTables(); - let filters: any[] = [{ db: this.defaultDb.databaseName, collection: '_powersync_checkpoints' }]; + let filters: any[] = [{ db: this.defaultDb.databaseName, coll: '_powersync_checkpoints' }]; for (let tablePattern of sourceTables) { if (tablePattern.connectionTag != this.connections.connectionTag) { continue; @@ -396,7 +397,10 @@ export class ChangeStream { // console.log('event', changeDocument); - if ( + if (changeDocument.operationType == 'insert' && changeDocument.ns.coll == '_powersync_checkpoints') { + const lsn = getMongoLsn(changeDocument.clusterTime!); + await batch.keepalive(lsn); + } else if ( changeDocument.operationType == 'insert' || changeDocument.operationType == 'update' || changeDocument.operationType == 'delete' diff --git a/modules/module-mongodb/src/replication/MongoRelation.ts b/modules/module-mongodb/src/replication/MongoRelation.ts index 8a83f5803..6b129d9da 100644 --- a/modules/module-mongodb/src/replication/MongoRelation.ts +++ b/modules/module-mongodb/src/replication/MongoRelation.ts @@ -134,7 +134,6 @@ export async function createCheckpoint(db: mongo.Db): Promise { const pingResult = await db.command({ ping: 1 }); const time: mongo.Timestamp = pingResult.$clusterTime.clusterTime; - const result = await db.collection('_powersync_checkpoints').findOneAndUpdate( { _id: 'checkpoint' as any diff --git a/modules/module-mongodb/test/src/change_stream.test.ts b/modules/module-mongodb/test/src/change_stream.test.ts index c0f144a1b..9578ae506 100644 --- a/modules/module-mongodb/test/src/change_stream.test.ts +++ b/modules/module-mongodb/test/src/change_stream.test.ts @@ -11,7 +11,7 @@ const BASIC_SYNC_RULES = ` bucket_definitions: global: data: - - SELECT id, description FROM "test_data" + - SELECT _id as id, description FROM "test_data" `; describe( @@ -23,7 +23,7 @@ describe( ); function defineWalStreamTests(factory: StorageFactory) { - test.only( + test( 'replicating basic values', walStreamTest(factory, async (context) => { const { db } = context; @@ -52,105 +52,83 @@ bucket_definitions: test( 'replicating case sensitive table', walStreamTest(factory, async (context) => { - const { pool } = context; + const { db } = context; await context.updateSyncRules(` bucket_definitions: global: data: - - SELECT id, description FROM "test_DATA" + - SELECT _id as id, description FROM "test_DATA" `); - await pool.query(`DROP TABLE IF EXISTS "test_DATA"`); - await pool.query(`CREATE TABLE "test_DATA"(id uuid primary key default uuid_generate_v4(), description text)`); - await context.replicateSnapshot(); - const startRowCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; - const startTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; - context.startStreaming(); - const [{ test_id }] = pgwireRows( - await pool.query(`INSERT INTO "test_DATA"(description) VALUES('test1') returning id as test_id`) - ); + const collection = db.collection('test_DATA'); + const result = await collection.insertOne({ description: 'test1' }); + const test_id = result.insertedId.toHexString(); const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_DATA', { id: test_id, description: 'test1' })]); - const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; - const endTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; - expect(endRowCount - startRowCount).toEqual(1); - expect(endTxCount - startTxCount).toEqual(1); }) ); test( - 'replicating TOAST values', + 'replicating large values', walStreamTest(factory, async (context) => { - const { pool } = context; + const { db } = context; await context.updateSyncRules(` bucket_definitions: global: data: - - SELECT id, name, description FROM "test_data" + - SELECT _id as id, name, description FROM "test_data" `); - await pool.query(`DROP TABLE IF EXISTS test_data`); - await pool.query( - `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), name text, description text)` - ); - await context.replicateSnapshot(); context.startStreaming(); - // Must be > 8kb after compression const largeDescription = crypto.randomBytes(20_000).toString('hex'); - const [{ test_id }] = pgwireRows( - await pool.query({ - statement: `INSERT INTO test_data(name, description) VALUES('test1', $1) returning id as test_id`, - params: [{ type: 'varchar', value: largeDescription }] - }) - ); - await pool.query(`UPDATE test_data SET name = 'test2' WHERE id = '${test_id}'`); + const collection = db.collection('test_data'); + const result = await collection.insertOne({ name: 'test1', description: largeDescription }); + const test_id = result.insertedId; + + await collection.updateOne({ _id: test_id }, { $set: { name: 'test2' } }); const data = await context.getBucketData('global[]'); expect(data.slice(0, 1)).toMatchObject([ - putOp('test_data', { id: test_id, name: 'test1', description: largeDescription }) + putOp('test_data', { id: test_id.toHexString(), name: 'test1', description: largeDescription }) ]); expect(data.slice(1)).toMatchObject([ - putOp('test_data', { id: test_id, name: 'test2', description: largeDescription }) + putOp('test_data', { id: test_id.toHexString(), name: 'test2', description: largeDescription }) ]); }) ); - test( - 'replicating TRUNCATE', + // Not implemented yet + test.skip( + 'replicating dropCollection', walStreamTest(factory, async (context) => { - const { pool } = context; + const { db } = context; const syncRuleContent = ` bucket_definitions: global: data: - - SELECT id, description FROM "test_data" + - SELECT _id as id, description FROM "test_data" by_test_data: - parameters: SELECT id FROM test_data WHERE id = token_parameters.user_id + parameters: SELECT _id as id FROM test_data WHERE id = token_parameters.user_id data: [] `; await context.updateSyncRules(syncRuleContent); - await pool.query(`DROP TABLE IF EXISTS test_data`); - await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`); - await context.replicateSnapshot(); context.startStreaming(); - const [{ test_id }] = pgwireRows( - await pool.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`) - ); - await pool.query(`TRUNCATE test_data`); + const collection = db.collection('test_data'); + const result = await collection.insertOne({ description: 'test1' }); + const test_id = result.insertedId.toHexString(); + + await collection.drop(); const data = await context.getBucketData('global[]'); @@ -161,62 +139,15 @@ bucket_definitions: }) ); - test( - 'replicating changing primary key', - walStreamTest(factory, async (context) => { - const { pool } = context; - await context.updateSyncRules(BASIC_SYNC_RULES); - await pool.query(`DROP TABLE IF EXISTS test_data`); - await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`); - - await context.replicateSnapshot(); - context.startStreaming(); - - const [{ test_id }] = pgwireRows( - await pool.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`) - ); - - const [{ test_id: test_id2 }] = pgwireRows( - await pool.query( - `UPDATE test_data SET id = uuid_generate_v4(), description = 'test2a' WHERE id = '${test_id}' returning id as test_id` - ) - ); - - // This update may fail replicating with: - // Error: Update on missing record public.test_data:074a601e-fc78-4c33-a15d-f89fdd4af31d :: {"g":1,"t":"651e9fbe9fec6155895057ec","k":"1a0b34da-fb8c-5e6f-8421-d7a3c5d4df4f"} - await pool.query(`UPDATE test_data SET description = 'test2b' WHERE id = '${test_id2}'`); - - // Re-use old id again - await pool.query(`INSERT INTO test_data(id, description) VALUES('${test_id}', 'test1b')`); - await pool.query(`UPDATE test_data SET description = 'test1c' WHERE id = '${test_id}'`); - - const data = await context.getBucketData('global[]'); - expect(data).toMatchObject([ - // Initial insert - putOp('test_data', { id: test_id, description: 'test1' }), - // Update id, then description - removeOp('test_data', test_id), - putOp('test_data', { id: test_id2, description: 'test2a' }), - putOp('test_data', { id: test_id2, description: 'test2b' }), - // Re-use old id - putOp('test_data', { id: test_id, description: 'test1b' }), - putOp('test_data', { id: test_id, description: 'test1c' }) - ]); - }) - ); - test( 'initial sync', walStreamTest(factory, async (context) => { - const { pool } = context; + const { db } = context; await context.updateSyncRules(BASIC_SYNC_RULES); - await pool.query(`DROP TABLE IF EXISTS test_data`); - await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`); - - const [{ test_id }] = pgwireRows( - await pool.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`) - ); + const collection = db.collection('test_data'); + const result = await collection.insertOne({ description: 'test1' }); + const test_id = result.insertedId.toHexString(); await context.replicateSnapshot(); context.startStreaming(); diff --git a/packages/service-core/src/storage/BucketStorage.ts b/packages/service-core/src/storage/BucketStorage.ts index aface2594..373be8f33 100644 --- a/packages/service-core/src/storage/BucketStorage.ts +++ b/packages/service-core/src/storage/BucketStorage.ts @@ -328,7 +328,7 @@ export interface BucketStorageBatch { * * Only call this after a transaction. */ - commit(lsn: string, options?: CommitOptions): Promise; + commit(lsn: string): Promise; /** * Advance the checkpoint LSN position, without any associated op. @@ -347,15 +347,6 @@ export interface BucketStorageBatch { markSnapshotDone(tables: SourceTable[], no_checkpoint_before_lsn: string): Promise; } -export interface CommitOptions { - /** - * Usually, a commit only takes effect if there are operations to commit. - * - * Setting this to true forces the commit to take place. - */ - forceCommit?: boolean; -} - export interface SaveParameterData { sourceTable: SourceTable; /** UUID */ diff --git a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts index 1d42a6494..4ac36a385 100644 --- a/packages/service-core/src/storage/mongo/MongoBucketBatch.ts +++ b/packages/service-core/src/storage/mongo/MongoBucketBatch.ts @@ -4,7 +4,7 @@ import * as mongo from 'mongodb'; import { container, errors, logger } from '@powersync/lib-services-framework'; import * as util from '../../util/util-index.js'; -import { BucketStorageBatch, CommitOptions, FlushedResult, mergeToast, SaveOptions } from '../BucketStorage.js'; +import { BucketStorageBatch, FlushedResult, mergeToast, SaveOptions } from '../BucketStorage.js'; import { SourceTable } from '../SourceTable.js'; import { PowerSyncMongo } from './db.js'; import { CurrentBucket, CurrentDataDocument, SourceKey, SyncRuleDocument } from './models.js'; @@ -536,7 +536,7 @@ export class MongoBucketBatch implements BucketStorageBatch { await this.session.endSession(); } - async commit(lsn: string, options?: CommitOptions): Promise { + async commit(lsn: string): Promise { await this.flush(); if (this.last_checkpoint_lsn != null && lsn < this.last_checkpoint_lsn) { @@ -550,24 +550,21 @@ export class MongoBucketBatch implements BucketStorageBatch { return false; } - if (this.persisted_op != null || options?.forceCommit) { + if (this.persisted_op != null) { const now = new Date(); - let setValues: mongo.MatchKeysAndValues = { - last_checkpoint_lsn: lsn, - last_checkpoint_ts: now, - last_keepalive_ts: now, - snapshot_done: true, - last_fatal_error: null - }; - if (this.persisted_op != null) { - (setValues as any).last_checkpoint = this.persisted_op; - } await this.db.sync_rules.updateOne( { _id: this.group_id }, { - $set: setValues + $set: { + last_checkpoint: this.persisted_op, + last_checkpoint_lsn: lsn, + last_checkpoint_ts: now, + last_keepalive_ts: now, + snapshot_done: true, + last_fatal_error: null + } }, { session: this.session } ); From aac810188e931264814eafd99bb046d5bfec9000 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 17 Sep 2024 12:36:51 +0200 Subject: [PATCH 13/35] Another fix; basic replication tests passing. --- .../src/replication/ChangeStream.ts | 2 +- .../test/src/change_stream.test.ts | 65 +++++++------------ 2 files changed, 25 insertions(+), 42 deletions(-) diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 0cfbd8a6e..0c46c0c00 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -147,7 +147,7 @@ export class ChangeStream { if (time != null) { const lsn = getMongoLsn(time.clusterTime); logger.info(`Snapshot commit at ${time.clusterTime.inspect()} / ${lsn}`); - await batch.commit(lsn); + // keepalive() does an auto-commit if there is data await batch.keepalive(lsn); } else { logger.info(`No snapshot clusterTime (no snapshot data?) - skipping commit.`); diff --git a/modules/module-mongodb/test/src/change_stream.test.ts b/modules/module-mongodb/test/src/change_stream.test.ts index 9578ae506..83e713470 100644 --- a/modules/module-mongodb/test/src/change_stream.test.ts +++ b/modules/module-mongodb/test/src/change_stream.test.ts @@ -1,6 +1,6 @@ import { putOp, removeOp } from '@core-tests/stream_utils.js'; import { MONGO_STORAGE_FACTORY } from '@core-tests/util.js'; -import { BucketStorageFactory, Metrics } from '@powersync/service-core'; +import { BucketStorageFactory } from '@powersync/service-core'; import * as crypto from 'crypto'; import { describe, expect, test } from 'vitest'; import { walStreamTest } from './change_stream_utils.js'; @@ -15,7 +15,7 @@ bucket_definitions: `; describe( - 'wal stream - mongodb', + 'change stream - mongodb', function () { defineWalStreamTests(MONGO_STORAGE_FACTORY); }, @@ -157,76 +157,59 @@ bucket_definitions: }) ); - test( - 'record too large', + // Not correctly implemented yet + test.skip( + 'large record', walStreamTest(factory, async (context) => { await context.updateSyncRules(`bucket_definitions: global: data: - - SELECT id, description, other FROM "test_data"`); - const { pool } = context; - - await pool.query(`CREATE TABLE test_data(id text primary key, description text, other text)`); + - SELECT _id as id, description, other FROM "test_data"`); + const { db } = context; await context.replicateSnapshot(); - // 4MB - const largeDescription = crypto.randomBytes(2_000_000).toString('hex'); - // 18MB - const tooLargeDescription = crypto.randomBytes(9_000_000).toString('hex'); + // 16MB + const largeDescription = crypto.randomBytes(8_000_000 - 100).toString('hex'); - await pool.query({ - statement: `INSERT INTO test_data(id, description, other) VALUES('t1', $1, 'foo')`, - params: [{ type: 'varchar', value: tooLargeDescription }] - }); - await pool.query({ - statement: `UPDATE test_data SET description = $1 WHERE id = 't1'`, - params: [{ type: 'varchar', value: largeDescription }] - }); + const collection = db.collection('test_data'); + const result = await collection.insertOne({ description: largeDescription }); + const test_id = result.insertedId; + await collection.updateOne({ _id: test_id }, { $set: { name: 't2' } }); context.startStreaming(); const data = await context.getBucketData('global[]'); - expect(data.length).toEqual(1); + expect(data.length).toEqual(2); const row = JSON.parse(data[0].data as string); delete row.description; - expect(row).toEqual({ id: 't1', other: 'foo' }); + expect(row).toEqual({ id: test_id.toHexString() }); delete data[0].data; - expect(data[0]).toMatchObject({ object_id: 't1', object_type: 'test_data', op: 'PUT', op_id: '1' }); + expect(data[0]).toMatchObject({ + object_id: test_id.toHexString(), + object_type: 'test_data', + op: 'PUT', + op_id: '1' + }); }) ); test( 'table not in sync rules', walStreamTest(factory, async (context) => { - const { pool } = context; + const { db } = context; await context.updateSyncRules(BASIC_SYNC_RULES); - await pool.query(`CREATE TABLE test_donotsync(id uuid primary key default uuid_generate_v4(), description text)`); - await context.replicateSnapshot(); - const startRowCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; - const startTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; - context.startStreaming(); - const [{ test_id }] = pgwireRows( - await pool.query(`INSERT INTO test_donotsync(description) VALUES('test1') returning id as test_id`) - ); + const collection = db.collection('test_donotsync'); + const result = await collection.insertOne({ description: 'test' }); const data = await context.getBucketData('global[]'); expect(data).toMatchObject([]); - const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; - const endTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; - - // There was a transaction, but we should not replicate any actual data - expect(endRowCount - startRowCount).toEqual(0); - expect(endTxCount - startTxCount).toEqual(1); }) ); } From 6a3b1e41669ac3680a8a181be5b06ab2724de4ff Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 17 Sep 2024 13:33:53 +0200 Subject: [PATCH 14/35] Handle collection drop and rename events. --- .../src/replication/ChangeStream.ts | 74 ++++++++++++++----- .../test/src/change_stream.test.ts | 35 ++++++++- modules/module-mongodb/test/src/env.ts | 2 +- 3 files changed, 90 insertions(+), 21 deletions(-) diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 0c46c0c00..10efb1942 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -80,18 +80,33 @@ export class ChangeStream { const name = tablePattern.name; - const table = await this.handleRelation( - batch, - { - name, - schema, - objectId: name, - replicationColumns: [{ name: '_id' }] - } as SourceEntityDescriptor, - false - ); + // Check if the collection exists + const collections = await this.client + .db(schema) + .listCollections( + { + name: name + }, + { nameOnly: true } + ) + .toArray(); + + if (collections[0]?.name == name) { + const table = await this.handleRelation( + batch, + { + name, + schema, + objectId: name, + replicationColumns: [{ name: '_id' }] + } as SourceEntityDescriptor, + // This is done as part of the initial setup - snapshot is handled elsewhere + { snapshot: false } + ); + + result.push(table); + } - result.push(table); return result; } @@ -126,6 +141,8 @@ export class ChangeStream { const sourceTables = this.sync_rules.getSourceTables(); await this.client.connect(); + const ping = await this.defaultDb.command({ ping: 1 }); + const startTime = ping.$clusterTime.clusterTime as mongo.Timestamp; const session = await this.client.startSession({ snapshot: true }); @@ -142,11 +159,12 @@ export class ChangeStream { await touch(); } } - const time = session.clusterTime; - if (time != null) { - const lsn = getMongoLsn(time.clusterTime); - logger.info(`Snapshot commit at ${time.clusterTime.inspect()} / ${lsn}`); + const snapshotTime = session.clusterTime?.clusterTime ?? startTime; + + if (snapshotTime != null) { + const lsn = getMongoLsn(snapshotTime); + logger.info(`Snapshot commit at ${snapshotTime.inspect()} / ${lsn}`); // keepalive() does an auto-commit if there is data await batch.keepalive(lsn); } else { @@ -228,7 +246,12 @@ export class ChangeStream { await batch.flush(); } - async handleRelation(batch: storage.BucketStorageBatch, descriptor: SourceEntityDescriptor, snapshot: boolean) { + async handleRelation( + batch: storage.BucketStorageBatch, + descriptor: SourceEntityDescriptor, + options: { snapshot: boolean } + ) { + const snapshot = options.snapshot; if (!descriptor.objectId && typeof descriptor.objectId != 'string') { throw new Error('objectId expected'); } @@ -249,7 +272,6 @@ export class ChangeStream { // 2. Snapshot is not already done, AND: // 3. The table is used in sync rules. const shouldSnapshot = snapshot && !result.table.snapshotComplete && result.table.syncAny; - if (shouldSnapshot) { // Truncate this table, in case a previous snapshot was interrupted. await batch.truncate([result.table]); @@ -406,13 +428,29 @@ export class ChangeStream { changeDocument.operationType == 'delete' ) { const rel = getMongoRelation(changeDocument.ns); - const table = await this.handleRelation(batch, rel, true); + // Don't snapshot tables here - we get all the data as insert events + const table = await this.handleRelation(batch, rel, { snapshot: false }); if (table.syncAny) { await this.writeChange(batch, table, changeDocument); if (changeDocument.clusterTime) { lastEventTimestamp = changeDocument.clusterTime; } } + } else if (changeDocument.operationType == 'drop') { + const rel = getMongoRelation(changeDocument.ns); + const table = await this.handleRelation(batch, rel, { snapshot: false }); + if (table.syncAny) { + await batch.drop([table]); + } + } else if (changeDocument.operationType == 'rename') { + const relFrom = getMongoRelation(changeDocument.ns); + const relTo = getMongoRelation(changeDocument.to); + const tableFrom = await this.handleRelation(batch, relFrom, { snapshot: false }); + if (tableFrom.syncAny) { + await batch.drop([tableFrom]); + } + // Here we do need to snapshot the new table + await this.handleRelation(batch, relTo, { snapshot: true }); } } }); diff --git a/modules/module-mongodb/test/src/change_stream.test.ts b/modules/module-mongodb/test/src/change_stream.test.ts index 83e713470..618a0856c 100644 --- a/modules/module-mongodb/test/src/change_stream.test.ts +++ b/modules/module-mongodb/test/src/change_stream.test.ts @@ -106,8 +106,7 @@ bucket_definitions: }) ); - // Not implemented yet - test.skip( + test( 'replicating dropCollection', walStreamTest(factory, async (context) => { const { db } = context; @@ -139,6 +138,38 @@ bucket_definitions: }) ); + test( + 'replicating renameCollection', + walStreamTest(factory, async (context) => { + const { db } = context; + const syncRuleContent = ` +bucket_definitions: + global: + data: + - SELECT _id as id, description FROM "test_data1" + - SELECT _id as id, description FROM "test_data2" +`; + await context.updateSyncRules(syncRuleContent); + await context.replicateSnapshot(); + context.startStreaming(); + + console.log('insert1', db.databaseName); + const collection = db.collection('test_data1'); + const result = await collection.insertOne({ description: 'test1' }); + const test_id = result.insertedId.toHexString(); + + await collection.rename('test_data2'); + + const data = await context.getBucketData('global[]'); + + expect(data).toMatchObject([ + putOp('test_data1', { id: test_id, description: 'test1' }), + removeOp('test_data1', test_id), + putOp('test_data2', { id: test_id, description: 'test1' }) + ]); + }) + ); + test( 'initial sync', walStreamTest(factory, async (context) => { diff --git a/modules/module-mongodb/test/src/env.ts b/modules/module-mongodb/test/src/env.ts index 7d72df3a9..e460c80b3 100644 --- a/modules/module-mongodb/test/src/env.ts +++ b/modules/module-mongodb/test/src/env.ts @@ -1,7 +1,7 @@ import { utils } from '@powersync/lib-services-framework'; export const env = utils.collectEnvironmentVariables({ - MONGO_TEST_DATA_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'), + MONGO_TEST_DATA_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test_data'), CI: utils.type.boolean.default('false'), SLOW_TESTS: utils.type.boolean.default('false') }); From 9f1de16225bf5c02a2cfa21ee3eb88d067452c1e Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 17 Sep 2024 13:40:30 +0200 Subject: [PATCH 15/35] Handle replace event. --- .../module-mongodb/src/replication/ChangeStream.ts | 3 ++- .../module-mongodb/test/src/change_stream.test.ts | 14 ++++++++++++-- .../module-mongodb/test/src/change_stream_utils.ts | 6 ++++-- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 10efb1942..73f32bcb8 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -307,7 +307,7 @@ export class ChangeStream { after: baseRecord, afterReplicaId: change.documentKey._id }); - } else if (change.operationType == 'update') { + } else if (change.operationType == 'update' || change.operationType == 'replace') { const after = constructAfterRecord(change.fullDocument ?? {}); return await batch.save({ tag: 'update', @@ -425,6 +425,7 @@ export class ChangeStream { } else if ( changeDocument.operationType == 'insert' || changeDocument.operationType == 'update' || + changeDocument.operationType == 'replace' || changeDocument.operationType == 'delete' ) { const rel = getMongoRelation(changeDocument.ns); diff --git a/modules/module-mongodb/test/src/change_stream.test.ts b/modules/module-mongodb/test/src/change_stream.test.ts index 618a0856c..889ddda8b 100644 --- a/modules/module-mongodb/test/src/change_stream.test.ts +++ b/modules/module-mongodb/test/src/change_stream.test.ts @@ -33,18 +33,28 @@ bucket_definitions: data: - SELECT _id as id, description, num FROM "test_data"`); + db.createCollection('test_data', { + changeStreamPreAndPostImages: { enabled: true } + }); + const collection = db.collection('test_data'); + await context.replicateSnapshot(); context.startStreaming(); - const collection = db.collection('test_data'); const result = await collection.insertOne({ description: 'test1', num: 1152921504606846976n }); const test_id = result.insertedId; + await collection.updateOne({ _id: test_id }, { $set: { description: 'test2' } }); + await collection.replaceOne({ _id: test_id }, { description: 'test3' }); + await collection.deleteOne({ _id: test_id }); const data = await context.getBucketData('global[]'); expect(data).toMatchObject([ - putOp('test_data', { id: test_id.toHexString(), description: 'test1', num: 1152921504606846976n }) + putOp('test_data', { id: test_id.toHexString(), description: 'test1', num: 1152921504606846976n }), + putOp('test_data', { id: test_id.toHexString(), description: 'test2', num: 1152921504606846976n }), + putOp('test_data', { id: test_id.toHexString(), description: 'test3' }), + removeOp('test_data', test_id.toHexString()) ]); }) ); diff --git a/modules/module-mongodb/test/src/change_stream_utils.ts b/modules/module-mongodb/test/src/change_stream_utils.ts index fedfb536c..73f71f292 100644 --- a/modules/module-mongodb/test/src/change_stream_utils.ts +++ b/modules/module-mongodb/test/src/change_stream_utils.ts @@ -1,4 +1,4 @@ -import { BucketStorageFactory, OpId, SyncRulesBucketStorage } from '@powersync/service-core'; +import { ActiveCheckpoint, BucketStorageFactory, OpId, SyncRulesBucketStorage } from '@powersync/service-core'; import { TEST_CONNECTION_OPTIONS, clearTestDb } from './util.js'; import { fromAsync } from '@core-tests/stream_utils.js'; @@ -128,9 +128,11 @@ export async function getClientCheckpoint( // Since we don't use LSNs anymore, the only way to get that is to wait. const timeout = options?.timeout ?? 5_00; + let lastCp: ActiveCheckpoint | null = null; while (Date.now() - start < timeout) { const cp = await bucketStorage.getActiveCheckpoint(); + lastCp = cp; if (!cp.hasSyncRules()) { throw new Error('No sync rules available'); } @@ -141,5 +143,5 @@ export async function getClientCheckpoint( await new Promise((resolve) => setTimeout(resolve, 30)); } - throw new Error('Timeout while waiting for checkpoint'); + throw new Error(`Timeout while waiting for checkpoint. Last checkpoint: ${lastCp?.lsn}`); } From 64bb4f48de312a887424f9f33c511a9ca17be297 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 17 Sep 2024 13:46:37 +0200 Subject: [PATCH 16/35] Handle missing fullDocument correctly. --- .../src/replication/ChangeStream.ts | 11 ++++- .../test/src/change_stream.test.ts | 47 ++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 73f32bcb8..a92842617 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -308,7 +308,16 @@ export class ChangeStream { afterReplicaId: change.documentKey._id }); } else if (change.operationType == 'update' || change.operationType == 'replace') { - const after = constructAfterRecord(change.fullDocument ?? {}); + if (change.fullDocument == null) { + // Treat as delete + return await batch.save({ + tag: 'delete', + sourceTable: table, + before: undefined, + beforeReplicaId: change.documentKey._id + }); + } + const after = constructAfterRecord(change.fullDocument!); return await batch.save({ tag: 'update', sourceTable: table, diff --git a/modules/module-mongodb/test/src/change_stream.test.ts b/modules/module-mongodb/test/src/change_stream.test.ts index 889ddda8b..1abefe69b 100644 --- a/modules/module-mongodb/test/src/change_stream.test.ts +++ b/modules/module-mongodb/test/src/change_stream.test.ts @@ -4,7 +4,7 @@ import { BucketStorageFactory } from '@powersync/service-core'; import * as crypto from 'crypto'; import { describe, expect, test } from 'vitest'; import { walStreamTest } from './change_stream_utils.js'; - +import * as mongo from 'mongodb'; type StorageFactory = () => Promise; const BASIC_SYNC_RULES = ` @@ -59,6 +59,51 @@ bucket_definitions: }) ); + test( + 'no fullDocument available', + walStreamTest(factory, async (context) => { + const { db, client } = context; + await context.updateSyncRules(` +bucket_definitions: + global: + data: + - SELECT _id as id, description, num FROM "test_data"`); + + db.createCollection('test_data', { + changeStreamPreAndPostImages: { enabled: false } + }); + const collection = db.collection('test_data'); + + await context.replicateSnapshot(); + + context.startStreaming(); + + const session = client.startSession(); + let test_id: mongo.ObjectId | undefined; + try { + await session.withTransaction(async () => { + const result = await collection.insertOne({ description: 'test1', num: 1152921504606846976n }, { session }); + test_id = result.insertedId; + await collection.updateOne({ _id: test_id }, { $set: { description: 'test2' } }, { session }); + await collection.replaceOne({ _id: test_id }, { description: 'test3' }, { session }); + await collection.deleteOne({ _id: test_id }, { session }); + }); + } finally { + await session.endSession(); + } + + const data = await context.getBucketData('global[]'); + + expect(data).toMatchObject([ + putOp('test_data', { id: test_id!.toHexString(), description: 'test1', num: 1152921504606846976n }), + // fullDocument is not available at the point this is replicated, resulting in it treated as a remove + removeOp('test_data', test_id!.toHexString()), + putOp('test_data', { id: test_id!.toHexString(), description: 'test3' }), + removeOp('test_data', test_id!.toHexString()) + ]); + }) + ); + test( 'replicating case sensitive table', walStreamTest(factory, async (context) => { From 834aa18998f90aa5456d0547d1f864ec91898a04 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 17 Sep 2024 14:21:15 +0200 Subject: [PATCH 17/35] Improve keepalive stability. --- .../src/api/MongoRouteAPIAdapter.ts | 2 +- .../src/replication/ChangeStream.ts | 21 ++++++--- .../src/replication/MongoRelation.ts | 43 ++++++++++--------- .../test/src/change_stream.test.ts | 5 +++ .../test/src/change_stream_utils.ts | 9 ++-- 5 files changed, 50 insertions(+), 30 deletions(-) diff --git a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts index 3389b2eb1..e3a3eea72 100644 --- a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts +++ b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts @@ -75,7 +75,7 @@ export class MongoRouteAPIAdapter implements api.RouteAPI { } async getReplicationHead(): Promise { - return createCheckpoint(this.db); + return createCheckpoint(this.client, this.db); } async getConnectionSchema(): Promise { diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index a92842617..d3552dc96 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -3,7 +3,13 @@ import { Metrics, SourceEntityDescriptor, storage } from '@powersync/service-cor import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern, toSyncRulesRow } from '@powersync/service-sync-rules'; import * as mongo from 'mongodb'; import { MongoManager } from './MongoManager.js'; -import { constructAfterRecord, getMongoLsn, getMongoRelation, mongoLsnToTimestamp } from './MongoRelation.js'; +import { + constructAfterRecord, + createCheckpoint, + getMongoLsn, + getMongoRelation, + mongoLsnToTimestamp +} from './MongoRelation.js'; export const ZERO_LSN = '0000000000000000'; @@ -276,10 +282,10 @@ export class ChangeStream { // Truncate this table, in case a previous snapshot was interrupted. await batch.truncate([result.table]); - let lsn: string = ZERO_LSN; - // TODO: Transaction / consistency await this.snapshotTable(batch, result.table); - const [table] = await batch.markSnapshotDone([result.table], lsn); + const no_checkpoint_before_lsn = await createCheckpoint(this.client, this.defaultDb); + + const [table] = await batch.markSnapshotDone([result.table], no_checkpoint_before_lsn); return table; } @@ -428,7 +434,12 @@ export class ChangeStream { // console.log('event', changeDocument); - if (changeDocument.operationType == 'insert' && changeDocument.ns.coll == '_powersync_checkpoints') { + if ( + (changeDocument.operationType == 'insert' || + changeDocument.operationType == 'update' || + changeDocument.operationType == 'replace') && + changeDocument.ns.coll == '_powersync_checkpoints' + ) { const lsn = getMongoLsn(changeDocument.clusterTime!); await batch.keepalive(lsn); } else if ( diff --git a/modules/module-mongodb/src/replication/MongoRelation.ts b/modules/module-mongodb/src/replication/MongoRelation.ts index 6b129d9da..267674895 100644 --- a/modules/module-mongodb/src/replication/MongoRelation.ts +++ b/modules/module-mongodb/src/replication/MongoRelation.ts @@ -130,24 +130,27 @@ function filterJsonData(data: any, depth = 0): any { } } -export async function createCheckpoint(db: mongo.Db): Promise { - const pingResult = await db.command({ ping: 1 }); - - const time: mongo.Timestamp = pingResult.$clusterTime.clusterTime; - const result = await db.collection('_powersync_checkpoints').findOneAndUpdate( - { - _id: 'checkpoint' as any - }, - { - $inc: { i: 1 } - }, - { - upsert: true, - returnDocument: 'after' - } - ); - - // TODO: Use the above when we support custom write checkpoints - - return getMongoLsn(time); +export async function createCheckpoint(client: mongo.MongoClient, db: mongo.Db): Promise { + const session = client.startSession(); + try { + const result = await db.collection('_powersync_checkpoints').findOneAndUpdate( + { + _id: 'checkpoint' as any + }, + { + $inc: { i: 1 } + }, + { + upsert: true, + returnDocument: 'after', + session + } + ); + const time = session.operationTime!; + // console.log('marked checkpoint at', time, getMongoLsn(time)); + // TODO: Use the above when we support custom write checkpoints + return getMongoLsn(time); + } finally { + await session.endSession(); + } } diff --git a/modules/module-mongodb/test/src/change_stream.test.ts b/modules/module-mongodb/test/src/change_stream.test.ts index 1abefe69b..f950e4f35 100644 --- a/modules/module-mongodb/test/src/change_stream.test.ts +++ b/modules/module-mongodb/test/src/change_stream.test.ts @@ -5,6 +5,8 @@ import * as crypto from 'crypto'; import { describe, expect, test } from 'vitest'; import { walStreamTest } from './change_stream_utils.js'; import * as mongo from 'mongodb'; +import { setTimeout } from 'node:timers/promises'; + type StorageFactory = () => Promise; const BASIC_SYNC_RULES = ` @@ -44,8 +46,11 @@ bucket_definitions: const result = await collection.insertOne({ description: 'test1', num: 1152921504606846976n }); const test_id = result.insertedId; + await setTimeout(10); await collection.updateOne({ _id: test_id }, { $set: { description: 'test2' } }); + await setTimeout(10); await collection.replaceOne({ _id: test_id }, { description: 'test3' }); + await setTimeout(10); await collection.deleteOne({ _id: test_id }); const data = await context.getBucketData('global[]'); diff --git a/modules/module-mongodb/test/src/change_stream_utils.ts b/modules/module-mongodb/test/src/change_stream_utils.ts index 73f71f292..02a70e6b4 100644 --- a/modules/module-mongodb/test/src/change_stream_utils.ts +++ b/modules/module-mongodb/test/src/change_stream_utils.ts @@ -90,7 +90,7 @@ export class ChangeStreamTestContext { async getCheckpoint(options?: { timeout?: number }) { let checkpoint = await Promise.race([ - getClientCheckpoint(this.db, this.factory, { timeout: options?.timeout ?? 15_000 }), + getClientCheckpoint(this.client, this.db, this.factory, { timeout: options?.timeout ?? 15_000 }), this.streamPromise ]); if (typeof checkpoint == undefined) { @@ -118,16 +118,17 @@ export class ChangeStreamTestContext { } export async function getClientCheckpoint( + client: mongo.MongoClient, db: mongo.Db, bucketStorage: BucketStorageFactory, options?: { timeout?: number } ): Promise { const start = Date.now(); - const lsn = await createCheckpoint(db); + const lsn = await createCheckpoint(client, db); // This old API needs a persisted checkpoint id. // Since we don't use LSNs anymore, the only way to get that is to wait. - const timeout = options?.timeout ?? 5_00; + const timeout = options?.timeout ?? 50_000; let lastCp: ActiveCheckpoint | null = null; while (Date.now() - start < timeout) { @@ -143,5 +144,5 @@ export async function getClientCheckpoint( await new Promise((resolve) => setTimeout(resolve, 30)); } - throw new Error(`Timeout while waiting for checkpoint. Last checkpoint: ${lastCp?.lsn}`); + throw new Error(`Timeout while waiting for checkpoint ${lsn}. Last checkpoint: ${lastCp?.lsn}`); } From a31b727be4c99fc17efc3f7a24c0d354042f719e Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 17 Sep 2024 16:29:44 +0200 Subject: [PATCH 18/35] Use checkpoints for standard replication. --- .../src/replication/ChangeStream.ts | 58 +++++++++---------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index d3552dc96..fc092f4ca 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -1,5 +1,5 @@ import { container, logger } from '@powersync/lib-services-framework'; -import { Metrics, SourceEntityDescriptor, storage } from '@powersync/service-core'; +import { Metrics, SourceEntityDescriptor, SourceTable, storage } from '@powersync/service-core'; import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern, toSyncRulesRow } from '@powersync/service-sync-rules'; import * as mongo from 'mongodb'; import { MongoManager } from './MongoManager.js'; @@ -172,6 +172,7 @@ export class ChangeStream { const lsn = getMongoLsn(snapshotTime); logger.info(`Snapshot commit at ${snapshotTime.inspect()} / ${lsn}`); // keepalive() does an auto-commit if there is data + await batch.flush(); await batch.keepalive(lsn); } else { logger.info(`No snapshot clusterTime (no snapshot data?) - skipping commit.`); @@ -252,6 +253,17 @@ export class ChangeStream { await batch.flush(); } + private async getRelation( + batch: storage.BucketStorageBatch, + descriptor: SourceEntityDescriptor + ): Promise { + const existing = this.relation_cache.get(descriptor.objectId); + if (existing != null) { + return existing; + } + return this.handleRelation(batch, descriptor, { snapshot: false }); + } + async handleRelation( batch: storage.BucketStorageBatch, descriptor: SourceEntityDescriptor, @@ -388,28 +400,17 @@ export class ChangeStream { showExpandedEvents: true, useBigInt64: true, maxAwaitTimeMS: 200, - fullDocument: 'updateLookup' // FIXME: figure this one out + fullDocument: 'updateLookup' }); this.abort_signal.addEventListener('abort', () => { stream.close(); }); - let lastEventTimestamp: mongo.Timestamp | null = null; + let waitForCheckpointLsn: string | null = null; while (true) { const changeDocument = await stream.tryNext(); if (changeDocument == null) { - // We don't get events for transaction commit. - // So if no events are available in the stream within maxAwaitTimeMS, - // we assume the transaction is complete. - // This is not foolproof - we may end up with a commit in the middle - // of a transaction. - if (lastEventTimestamp != null) { - const lsn = getMongoLsn(lastEventTimestamp); - await batch.commit(lsn); - lastEventTimestamp = null; - } - continue; } await touch(); @@ -422,16 +423,6 @@ export class ChangeStream { continue; } - if ( - lastEventTimestamp != null && - changeDocument.clusterTime != null && - changeDocument.clusterTime.neq(lastEventTimestamp) - ) { - const lsn = getMongoLsn(lastEventTimestamp); - await batch.commit(lsn); - lastEventTimestamp = null; - } - // console.log('event', changeDocument); if ( @@ -441,6 +432,10 @@ export class ChangeStream { changeDocument.ns.coll == '_powersync_checkpoints' ) { const lsn = getMongoLsn(changeDocument.clusterTime!); + if (waitForCheckpointLsn != null && lsn >= waitForCheckpointLsn) { + waitForCheckpointLsn = null; + } + await batch.flush(); await batch.keepalive(lsn); } else if ( changeDocument.operationType == 'insert' || @@ -448,27 +443,28 @@ export class ChangeStream { changeDocument.operationType == 'replace' || changeDocument.operationType == 'delete' ) { + if (waitForCheckpointLsn == null) { + waitForCheckpointLsn = await createCheckpoint(this.client, this.defaultDb); + } const rel = getMongoRelation(changeDocument.ns); - // Don't snapshot tables here - we get all the data as insert events - const table = await this.handleRelation(batch, rel, { snapshot: false }); + const table = await this.getRelation(batch, rel); if (table.syncAny) { await this.writeChange(batch, table, changeDocument); - if (changeDocument.clusterTime) { - lastEventTimestamp = changeDocument.clusterTime; - } } } else if (changeDocument.operationType == 'drop') { const rel = getMongoRelation(changeDocument.ns); - const table = await this.handleRelation(batch, rel, { snapshot: false }); + const table = await this.getRelation(batch, rel); if (table.syncAny) { await batch.drop([table]); + this.relation_cache.delete(table.objectId); } } else if (changeDocument.operationType == 'rename') { const relFrom = getMongoRelation(changeDocument.ns); const relTo = getMongoRelation(changeDocument.to); - const tableFrom = await this.handleRelation(batch, relFrom, { snapshot: false }); + const tableFrom = await this.getRelation(batch, relFrom); if (tableFrom.syncAny) { await batch.drop([tableFrom]); + this.relation_cache.delete(tableFrom.objectId); } // Here we do need to snapshot the new table await this.handleRelation(batch, relTo, { snapshot: true }); From 594acc45ef9149e403fe1da5e9592449577d00f0 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 17 Sep 2024 16:42:00 +0200 Subject: [PATCH 19/35] Support wildcard table patterns. --- .../src/replication/ChangeStream.ts | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index fc092f4ca..f993ec171 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -78,32 +78,32 @@ export class ChangeStream { return []; } + let nameFilter: RegExp | string; if (tablePattern.isWildcard) { - // TODO: Implement - throw new Error('Wildcard collections not supported yet'); + nameFilter = new RegExp('^' + escapeRegExp(tablePattern.tablePrefix)); + } else { + nameFilter = tablePattern.name; } let result: storage.SourceTable[] = []; - const name = tablePattern.name; - // Check if the collection exists const collections = await this.client .db(schema) .listCollections( { - name: name + name: nameFilter }, { nameOnly: true } ) .toArray(); - if (collections[0]?.name == name) { + for (let collection of collections) { const table = await this.handleRelation( batch, { - name, + name: collection.name, schema, - objectId: name, + objectId: collection.name, replicationColumns: [{ name: '_id' }] } as SourceEntityDescriptor, // This is done as part of the initial setup - snapshot is handled elsewhere @@ -187,23 +187,26 @@ export class ChangeStream { private getSourceNamespaceFilters() { const sourceTables = this.sync_rules.getSourceTables(); - let filters: any[] = [{ db: this.defaultDb.databaseName, coll: '_powersync_checkpoints' }]; + let $inFilters: any[] = [{ db: this.defaultDb.databaseName, coll: '_powersync_checkpoints' }]; + let $refilters: any[] = []; for (let tablePattern of sourceTables) { if (tablePattern.connectionTag != this.connections.connectionTag) { continue; } if (tablePattern.isWildcard) { - // TODO: Implement - throw new Error('wildcard collections not supported yet'); + $refilters.push({ db: tablePattern.schema, coll: new RegExp('^' + escapeRegExp(tablePattern.tablePrefix)) }); + } else { + $inFilters.push({ + db: tablePattern.schema, + coll: tablePattern.name + }); } - - filters.push({ - db: tablePattern.schema, - coll: tablePattern.name - }); } - return { $in: filters }; + if ($refilters.length > 0) { + return { $or: [{ ns: { $in: $inFilters } }, ...$refilters] }; + } + return { ns: { $in: $inFilters } }; } static *getQueryData(results: Iterable): Generator { @@ -389,9 +392,7 @@ export class ChangeStream { const pipeline: mongo.Document[] = [ { - $match: { - ns: this.getSourceNamespaceFilters() - } + $match: this.getSourceNamespaceFilters() } ]; @@ -480,3 +481,8 @@ async function touch() { // or reduce PING_INTERVAL here. return container.probes.touch(); } + +function escapeRegExp(string: string) { + // https://stackoverflow.com/a/3561711/214837 + return string.replace(/[/\-\\^$*+?.()|[\]{}]/g, '\\$&'); +} From ef65356009b1ddb5c49b80c95fc3f14d8d2ac901 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 17 Sep 2024 16:43:34 +0200 Subject: [PATCH 20/35] Fix merge issue. --- .../module-mongodb/src/replication/ChangeStreamReplicator.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/module-mongodb/src/replication/ChangeStreamReplicator.ts b/modules/module-mongodb/src/replication/ChangeStreamReplicator.ts index d4c1314dd..84b2c7f68 100644 --- a/modules/module-mongodb/src/replication/ChangeStreamReplicator.ts +++ b/modules/module-mongodb/src/replication/ChangeStreamReplicator.ts @@ -1,6 +1,7 @@ import { storage, replication } from '@powersync/service-core'; import { ChangeStreamReplicationJob } from './ChangeStreamReplicationJob.js'; import { ConnectionManagerFactory } from './ConnectionManagerFactory.js'; +import { MongoErrorRateLimiter } from './MongoErrorRateLimiter.js'; export interface WalStreamReplicatorOptions extends replication.AbstractReplicatorOptions { connectionFactory: ConnectionManagerFactory; @@ -19,7 +20,8 @@ export class ChangeStreamReplicator extends replication.AbstractReplicator Date: Thu, 19 Sep 2024 17:14:40 +0200 Subject: [PATCH 21/35] Add changeset. --- .changeset/green-peas-roll.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/green-peas-roll.md diff --git a/.changeset/green-peas-roll.md b/.changeset/green-peas-roll.md new file mode 100644 index 000000000..61762c082 --- /dev/null +++ b/.changeset/green-peas-roll.md @@ -0,0 +1,6 @@ +--- +'@powersync/service-module-mongodb': minor +'@powersync/service-image': minor +--- + +Add MongoDB support (Alpha) From e9bf63ec86dad63c2bf9c6795a26d9cdf97df07d Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 23 Sep 2024 12:24:10 +0200 Subject: [PATCH 22/35] Fix aborting logic. --- .../src/replication/ChangeStream.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index f993ec171..7638b6bdf 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -403,6 +403,12 @@ export class ChangeStream { maxAwaitTimeMS: 200, fullDocument: 'updateLookup' }); + + if (this.abort_signal.aborted) { + stream.close(); + return; + } + this.abort_signal.addEventListener('abort', () => { stream.close(); }); @@ -410,16 +416,17 @@ export class ChangeStream { let waitForCheckpointLsn: string | null = null; while (true) { + if (this.abort_signal.aborted) { + break; + } + const changeDocument = await stream.tryNext(); - if (changeDocument == null) { + + if (changeDocument == null || this.abort_signal.aborted) { continue; } await touch(); - if (this.abort_signal.aborted) { - break; - } - if (startAfter != null && changeDocument.clusterTime?.lte(startAfter)) { continue; } From 03a18c78c920cc03036bf01b2c7968e0a5676374 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 23 Sep 2024 12:24:46 +0200 Subject: [PATCH 23/35] Normalize mongo connection parameters. --- modules/module-mongodb/src/types/types.ts | 60 ++----------------- packages/service-core/package.json | 1 + packages/service-core/src/db/mongo.ts | 8 ++- .../service-core/src/storage/mongo/config.ts | 40 +++++++++++++ .../service-core/src/storage/storage-index.ts | 1 + pnpm-lock.yaml | 3 + 6 files changed, 55 insertions(+), 58 deletions(-) create mode 100644 packages/service-core/src/storage/mongo/config.ts diff --git a/modules/module-mongodb/src/types/types.ts b/modules/module-mongodb/src/types/types.ts index 4aeef79d3..572a8b4dd 100644 --- a/modules/module-mongodb/src/types/types.ts +++ b/modules/module-mongodb/src/types/types.ts @@ -1,6 +1,6 @@ +import { normalizeMongoConfig } from '@powersync/service-core'; import * as service_types from '@powersync/service-types'; import * as t from 'ts-codec'; -import * as urijs from 'uri-js'; export const MONGO_CONNECTION_TYPE = 'mongodb' as const; @@ -22,20 +22,10 @@ export const MongoConnectionConfig = service_types.configFile.dataSourceConfig.a id: t.string.optional(), /** Tag used as reference in sync rules. Defaults to "default". Does not have to be unique. */ tag: t.string.optional(), - uri: t.string.optional(), - hostname: t.string.optional(), - port: service_types.configFile.portCodec.optional(), + uri: t.string, username: t.string.optional(), password: t.string.optional(), - database: t.string.optional(), - - /** Defaults to verify-full */ - sslmode: t.literal('verify-full').or(t.literal('verify-ca')).or(t.literal('disable')).optional(), - /** Required for verify-ca, optional for verify-full */ - cacert: t.string.optional(), - - client_certificate: t.string.optional(), - client_private_key: t.string.optional() + database: t.string.optional() }) ); @@ -55,56 +45,16 @@ export type ResolvedConnectionConfig = MongoConnectionConfig & NormalizedMongoCo * Returns destructured options. */ export function normalizeConnectionConfig(options: MongoConnectionConfig): NormalizedMongoConnectionConfig { - let uri: urijs.URIComponents; - if (options.uri) { - uri = urijs.parse(options.uri); - if (uri.scheme != 'mongodb') { - `Invalid URI - protocol must be postgresql, got ${uri.scheme}`; - } - } else { - uri = urijs.parse('mongodb:///'); - } - - const database = options.database ?? uri.path?.substring(1) ?? ''; - - const [uri_username, uri_password] = (uri.userinfo ?? '').split(':'); - - const username = options.username ?? uri_username; - const password = options.password ?? uri_password; - - if (database == '') { - throw new Error(`database required`); - } + const base = normalizeMongoConfig(options); return { id: options.id ?? 'default', tag: options.tag ?? 'default', - // TODO: remove username & password from uri - uri: options.uri ?? '', - database, - - username, - password + ...base }; } -/** - * Check whether the port is in a "safe" range. - * - * We do not support connecting to "privileged" ports. - */ -export function validatePort(port: string | number): number { - if (typeof port == 'string') { - port = parseInt(port); - } - if (port >= 1024 && port <= 49151) { - return port; - } else { - throw new Error(`Port ${port} not supported`); - } -} - /** * Construct a mongodb URI, without username, password or ssl options. * diff --git a/packages/service-core/package.json b/packages/service-core/package.json index 58f691586..025489ead 100644 --- a/packages/service-core/package.json +++ b/packages/service-core/package.json @@ -40,6 +40,7 @@ "mongodb": "^6.7.0", "node-fetch": "^3.3.2", "ts-codec": "^1.2.2", + "uri-js": "^4.4.1", "uuid": "^9.0.1", "winston": "^3.13.0", "yaml": "^2.3.2" diff --git a/packages/service-core/src/db/mongo.ts b/packages/service-core/src/db/mongo.ts index be915bdc3..33d6d5b86 100644 --- a/packages/service-core/src/db/mongo.ts +++ b/packages/service-core/src/db/mongo.ts @@ -2,6 +2,7 @@ import * as mongo from 'mongodb'; import * as timers from 'timers/promises'; import { configFile } from '@powersync/service-types'; +import { normalizeMongoConfig } from '../storage/storage-index.js'; /** * Time for new connection to timeout. @@ -23,10 +24,11 @@ export const MONGO_SOCKET_TIMEOUT_MS = 60_000; export const MONGO_OPERATION_TIMEOUT_MS = 30_000; export function createMongoClient(config: configFile.PowerSyncConfig['storage']) { - return new mongo.MongoClient(config.uri, { + const normalized = normalizeMongoConfig(config); + return new mongo.MongoClient(normalized.uri, { auth: { - username: config.username, - password: config.password + username: normalized.username, + password: normalized.password }, // Time for connection to timeout connectTimeoutMS: MONGO_CONNECT_TIMEOUT_MS, diff --git a/packages/service-core/src/storage/mongo/config.ts b/packages/service-core/src/storage/mongo/config.ts new file mode 100644 index 000000000..8ff241e25 --- /dev/null +++ b/packages/service-core/src/storage/mongo/config.ts @@ -0,0 +1,40 @@ +import * as urijs from 'uri-js'; + +export interface MongoConnectionConfig { + uri: string; + username?: string; + password?: string; + database?: string; +} + +/** + * Validate and normalize connection options. + * + * Returns destructured options. + * + * For use by both storage and mongo module. + */ +export function normalizeMongoConfig(options: MongoConnectionConfig) { + let uri = urijs.parse(options.uri); + + const database = options.database ?? uri.path?.substring(1) ?? ''; + + const userInfo = uri.userinfo?.split(':'); + + const username = options.username ?? userInfo?.[0]; + const password = options.password ?? userInfo?.[1]; + + if (database == '') { + throw new Error(`database required`); + } + + delete uri.userinfo; + + return { + uri: urijs.serialize(uri), + database, + + username, + password + }; +} diff --git a/packages/service-core/src/storage/storage-index.ts b/packages/service-core/src/storage/storage-index.ts index 3c137b8d1..42aec416d 100644 --- a/packages/service-core/src/storage/storage-index.ts +++ b/packages/service-core/src/storage/storage-index.ts @@ -16,3 +16,4 @@ export * from './mongo/MongoSyncRulesLock.js'; export * from './mongo/OperationBatch.js'; export * from './mongo/PersistedBatch.js'; export * from './mongo/util.js'; +export * from './mongo/config.js'; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b8dd719e4..75f8ae93f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -321,6 +321,9 @@ importers: ts-codec: specifier: ^1.2.2 version: 1.2.2 + uri-js: + specifier: ^4.4.1 + version: 4.4.1 uuid: specifier: ^9.0.1 version: 9.0.1 From f42873655d54c25a27f474a7be224ea93852ff0c Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 23 Sep 2024 15:07:55 +0200 Subject: [PATCH 24/35] Remove error message when closing a ChangStreamReplicationJob. --- .../src/replication/ChangeStreamReplicationJob.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts b/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts index 05b6c1e26..fb60e7aa7 100644 --- a/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts +++ b/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts @@ -79,6 +79,9 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ }); await stream.replicate(); } catch (e) { + if (this.abortController.signal.aborted) { + return; + } this.logger.error(`Replication error`, e); if (e.cause != null) { // Without this additional log, the cause may not be visible in the logs. From 9de2d35d128b364c09ab5b3d543005f40095bb64 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 23 Sep 2024 15:08:11 +0200 Subject: [PATCH 25/35] Support M0 clusters; better error message on unsupported clusters. --- .../module-mongodb/src/replication/ChangeStream.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 7638b6bdf..323288001 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -147,8 +147,15 @@ export class ChangeStream { const sourceTables = this.sync_rules.getSourceTables(); await this.client.connect(); - const ping = await this.defaultDb.command({ ping: 1 }); - const startTime = ping.$clusterTime.clusterTime as mongo.Timestamp; + const hello = await this.defaultDb.command({ hello: 1 }); + const startTime = hello.lastWrite?.majorityOpTime as mongo.Timestamp; + if (hello.isdbgrid) { + throw new Error('Sharded MongoDB Clusters are not supported yet.'); + } else if (hello.setName == null) { + throw new Error('Standalone MongoDB instances are not supported - use a replicaset.'); + } else if (startTime == null) { + throw new Error('MongoDB lastWrite timestamp not found.'); + } const session = await this.client.startSession({ snapshot: true }); @@ -175,7 +182,7 @@ export class ChangeStream { await batch.flush(); await batch.keepalive(lsn); } else { - logger.info(`No snapshot clusterTime (no snapshot data?) - skipping commit.`); + throw new Error(`No snapshot clusterTime available.`); } } ); From 3185e026f71134e0c5c448bdff44198bf7afc3ea Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 23 Sep 2024 15:16:56 +0200 Subject: [PATCH 26/35] Fix sharded cluster check. --- modules/module-mongodb/src/replication/ChangeStream.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 323288001..9b8f9d5aa 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -149,11 +149,12 @@ export class ChangeStream { const hello = await this.defaultDb.command({ hello: 1 }); const startTime = hello.lastWrite?.majorityOpTime as mongo.Timestamp; - if (hello.isdbgrid) { - throw new Error('Sharded MongoDB Clusters are not supported yet.'); + if (hello.msg == 'isdbgrid') { + throw new Error('Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).'); } else if (hello.setName == null) { throw new Error('Standalone MongoDB instances are not supported - use a replicaset.'); } else if (startTime == null) { + // Not known where this would happen apart from the above cases throw new Error('MongoDB lastWrite timestamp not found.'); } const session = await this.client.startSession({ From 14b83eb4004f15390adf194e26f09b89c0e3959a Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 23 Sep 2024 17:47:36 +0200 Subject: [PATCH 27/35] Fix initial replication regression. --- modules/module-mongodb/src/replication/ChangeStream.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 9b8f9d5aa..92392d863 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -148,7 +148,7 @@ export class ChangeStream { await this.client.connect(); const hello = await this.defaultDb.command({ hello: 1 }); - const startTime = hello.lastWrite?.majorityOpTime as mongo.Timestamp; + const startTime = hello.lastWrite?.majorityOpTime?.ts as mongo.Timestamp; if (hello.msg == 'isdbgrid') { throw new Error('Sharded MongoDB Clusters are not supported yet (including MongoDB Serverless instances).'); } else if (hello.setName == null) { From bfea46165bbef4f49b3c4990f68070f8fe43d3e6 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 1 Oct 2024 16:49:13 +0200 Subject: [PATCH 28/35] remove mongodb publish config restriction in order to publish dev packages --- modules/module-mongodb/package.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/module-mongodb/package.json b/modules/module-mongodb/package.json index dfcd52e0e..b42949353 100644 --- a/modules/module-mongodb/package.json +++ b/modules/module-mongodb/package.json @@ -2,9 +2,6 @@ "name": "@powersync/service-module-mongodb", "repository": "https://github.com/powersync-ja/powersync-service", "types": "dist/index.d.ts", - "publishConfig": { - "access": "restricted" - }, "version": "0.0.1", "main": "dist/index.js", "license": "FSL-1.1-Apache-2.0", From fb52b595e4dc4feaa35a17e1ed537ac572028c64 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 1 Oct 2024 17:03:25 +0200 Subject: [PATCH 29/35] need to explicitly set MongDB package access to public for publish. --- modules/module-mongodb/package.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/module-mongodb/package.json b/modules/module-mongodb/package.json index b42949353..5809bed99 100644 --- a/modules/module-mongodb/package.json +++ b/modules/module-mongodb/package.json @@ -6,6 +6,9 @@ "main": "dist/index.js", "license": "FSL-1.1-Apache-2.0", "type": "module", + "publishConfig": { + "access": "public" + }, "scripts": { "build": "tsc -b", "build:tests": "tsc -b test/tsconfig.json", From 354f7624a099b4ebf04728b1f99e619f7467e39a Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 2 Oct 2024 11:38:37 +0200 Subject: [PATCH 30/35] Fix pnpm-lock. --- pnpm-lock.yaml | 312 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 308 insertions(+), 4 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2a346a0ea..bf6e89cf5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -129,10 +129,10 @@ importers: version: 9.0.8 typescript: specifier: ^5.2.2 - version: 5.2.2 + version: 5.6.2 vite-tsconfig-paths: specifier: ^4.3.2 - version: 4.3.2(typescript@5.2.2)(vite@5.2.11(@types/node@18.11.11)) + version: 4.3.2(typescript@5.6.2)(vite@5.2.11(@types/node@22.5.5)) vitest: specifier: ^0.34.6 version: 0.34.6 @@ -789,6 +789,10 @@ packages: resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} + '@jest/schemas@29.6.3': + resolution: {integrity: sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + '@jridgewell/resolve-uri@3.1.2': resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} engines: {node: '>=6.0.0'} @@ -1258,6 +1262,9 @@ packages: resolution: {integrity: sha512-2bRovzs0nJZFlCN3rXirE4gwxCn97JNjMmwpecqlbgV9WcxX7WRuIrgzx/X7Ib7MYRbyUTpBYE0s2x6AmZXnlg==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} + '@sinclair/typebox@0.27.8': + resolution: {integrity: sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==} + '@sindresorhus/is@5.6.0': resolution: {integrity: sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==} engines: {node: '>=14.16'} @@ -1302,6 +1309,12 @@ packages: '@types/body-parser@1.19.5': resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==} + '@types/chai-subset@1.3.5': + resolution: {integrity: sha512-c2mPnw+xHtXDoHmdtcCXGwyLMiauiAyxWMzhGpqHC4nqI/Y5G2XhTampslK2rb59kpcuHon03UH8W6iYUzw88A==} + + '@types/chai@4.3.20': + resolution: {integrity: sha512-/pC9HAB5I/xMlc5FP77qjCnI16ChlJfW0tGa0IUcFn38VJrTV6DeZ60NU5KZBtaOZqjdpwTWohz5HU1RrhiYxQ==} + '@types/connect@3.4.36': resolution: {integrity: sha512-P63Zd/JUGq+PdrM1lv0Wv5SBYeA2+CORvbrXbngriYY0jzLUWfQMQQxOhjONEz/wlHOAxOdY7CY65rgQdTjq2w==} @@ -1413,6 +1426,9 @@ packages: '@types/ws@8.2.3': resolution: {integrity: sha512-ahRJZquUYCdOZf/rCsWg88S0/+cb9wazUBHv6HZEe3XdYaBe2zr/slM8J28X07Hn88Pnm4ezo7N8/ofnOgrPVQ==} + '@vitest/expect@0.34.6': + resolution: {integrity: sha512-QUzKpUQRc1qC7qdGo7rMK3AkETI7w18gTCUrsNnyjjJKYiuUB9+TQK3QnR1unhCnWRC0AbKv2omLGQDF/mIjOw==} + '@vitest/expect@2.1.1': resolution: {integrity: sha512-YeueunS0HiHiQxk+KEOnq/QMzlUuOzbU1Go+PgAsHvvv3tUkJPm9xWt+6ITNTlzsMXUjmgm5T+U7KBPK2qQV6w==} @@ -1431,15 +1447,27 @@ packages: '@vitest/pretty-format@2.1.1': resolution: {integrity: sha512-SjxPFOtuINDUW8/UkElJYQSFtnWX7tMksSGW0vfjxMneFqxVr8YJ979QpMbDW7g+BIiq88RAGDjf7en6rvLPPQ==} + '@vitest/runner@0.34.6': + resolution: {integrity: sha512-1CUQgtJSLF47NnhN+F9X2ycxUP0kLHQ/JWvNHbeBfwW8CzEGgeskzNnHDyv1ieKTltuR6sdIHV+nmR6kPxQqzQ==} + '@vitest/runner@2.1.1': resolution: {integrity: sha512-uTPuY6PWOYitIkLPidaY5L3t0JJITdGTSwBtwMjKzo5O6RCOEncz9PUN+0pDidX8kTHYjO0EwUIvhlGpnGpxmA==} + '@vitest/snapshot@0.34.6': + resolution: {integrity: sha512-B3OZqYn6k4VaN011D+ve+AA4whM4QkcwcrwaKwAbyyvS/NB1hCWjFIBQxAQQSQir9/RtyAAGuq+4RJmbn2dH4w==} + '@vitest/snapshot@2.1.1': resolution: {integrity: sha512-BnSku1WFy7r4mm96ha2FzN99AZJgpZOWrAhtQfoxjUU5YMRpq1zmHRq7a5K9/NjqonebO7iVDla+VvZS8BOWMw==} + '@vitest/spy@0.34.6': + resolution: {integrity: sha512-xaCvneSaeBw/cz8ySmF7ZwGvL0lBjfvqc1LpQ/vcdHEvpLn3Ff1vAvjw+CoGn0802l++5L/pxb7whwcWAw+DUQ==} + '@vitest/spy@2.1.1': resolution: {integrity: sha512-ZM39BnZ9t/xZ/nF4UwRH5il0Sw93QnZXd9NAZGRpIgj0yvVwPpLd702s/Cx955rGaMlyBQkZJ2Ir7qyY48VZ+g==} + '@vitest/utils@0.34.6': + resolution: {integrity: sha512-IG5aDD8S6zlvloDsnzHw0Ut5xczlF+kv2BOTo+iXfPr54Yhi5qbVOgGB1hZaVq4iJ4C/MZ2J0y15IlsV/ZcI0A==} + '@vitest/utils@2.1.1': resolution: {integrity: sha512-Y6Q9TsI+qJ2CC0ZKj6VBb+T8UPz593N113nnUykqwANqhgf3QkZeHFlusgKLTqrnVHbj/XDKZcDHol+dxVT+rQ==} @@ -1530,6 +1558,10 @@ packages: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} engines: {node: '>=8'} + ansi-styles@5.2.0: + resolution: {integrity: sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==} + engines: {node: '>=10'} + ansi-styles@6.2.1: resolution: {integrity: sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==} engines: {node: '>=12'} @@ -1559,6 +1591,9 @@ packages: resolution: {integrity: sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==} engines: {node: '>=8'} + assertion-error@1.1.0: + resolution: {integrity: sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==} + assertion-error@2.0.1: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} @@ -1654,6 +1689,10 @@ packages: resolution: {integrity: sha512-xlx1yCK2Oc1APsPXDL2LdlNP6+uu8OCDdhOBSVT279M/S+y75O30C2VuD8T2ogdePBBl7PfPF4504tnLgX3zfw==} engines: {node: '>=14.16'} + chai@4.5.0: + resolution: {integrity: sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==} + engines: {node: '>=4'} + chai@5.1.1: resolution: {integrity: sha512-pT1ZgP8rPNqUgieVaEY+ryQr6Q4HXNg8Ei9UnLUrjN4IA7dvQC5JB+/kxVcPNDHyBcc/26CXPkbNzq3qwrOEKA==} engines: {node: '>=12'} @@ -1673,6 +1712,9 @@ packages: chardet@0.7.0: resolution: {integrity: sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==} + check-error@1.0.3: + resolution: {integrity: sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==} + check-error@2.1.1: resolution: {integrity: sha512-OAlb+T7V4Op9OwdkjmguYRqncdlx5JiofwOAUkmTF+jNdHwzTaTs4sRAGpzLF3oOz5xAyDGrPgeIDFQmDOTiJw==} engines: {node: '>= 16'} @@ -1772,6 +1814,9 @@ packages: engines: {node: ^14.13.0 || >=16.0.0} hasBin: true + confbox@0.1.7: + resolution: {integrity: sha512-uJcB/FKZtBMCJpK8MQji6bJHgu1tixKPxRLeGkNzBoOZzpnZUJm0jm2/sBDWcuBx1dYgxV4JU+g5hmNxCyAmdA==} + config-chain@1.1.13: resolution: {integrity: sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==} @@ -1844,6 +1889,10 @@ packages: resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} engines: {node: '>=10'} + deep-eql@4.1.4: + resolution: {integrity: sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg==} + engines: {node: '>=6'} + deep-eql@5.0.2: resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} engines: {node: '>=6'} @@ -1870,6 +1919,10 @@ packages: resolution: {integrity: sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA==} engines: {node: '>=8'} + diff-sequences@29.6.3: + resolution: {integrity: sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + diff@4.0.2: resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==} engines: {node: '>=0.3.1'} @@ -2462,6 +2515,10 @@ packages: light-my-request@5.13.0: resolution: {integrity: sha512-9IjUN9ZyCS9pTG+KqTDEQo68Sui2lHsYBrfMyVUTTZ3XhH8PMZq7xO94Kr+eP9dhi/kcKsx4N41p2IXEBil1pQ==} + local-pkg@0.4.3: + resolution: {integrity: sha512-SFppqq5p42fe2qcZQqqEOiVRXl+WCP1MdT6k7BDEW1j++sp5fIY+/fdRQitvKgB5BrBcmrs5m/L0v2FrU5MY1g==} + engines: {node: '>=14'} + locate-path@5.0.0: resolution: {integrity: sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==} engines: {node: '>=8'} @@ -2490,6 +2547,9 @@ packages: lossless-json@2.0.11: resolution: {integrity: sha512-BP0vn+NGYvzDielvBZaFain/wgeJ1hTvURCqtKvhr1SCPePdaaTanmmcplrHfEJSJOUql7hk4FHwToNJjWRY3g==} + loupe@2.3.7: + resolution: {integrity: sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA==} + loupe@3.1.1: resolution: {integrity: sha512-edNu/8D5MKVfGVFRhFf8aAxiTM6Wumfz5XsaatSxlD3w4R1d/WEKUTydCdPGbl9K7QG/Ca3GnDV2sIKIpXRQcw==} @@ -2610,6 +2670,9 @@ packages: engines: {node: '>=10'} hasBin: true + mlly@1.7.1: + resolution: {integrity: sha512-rrVRZRELyQzrIUAVMHxP97kv+G786pHmOKzuFII8zDYahFBS7qnHh2AlYSl1GAHhaMPCz6/oHjVMcfFYgFYHgA==} + mnemonist@0.39.5: resolution: {integrity: sha512-FPUtkhtJ0efmEFGpU14x7jGbTB+s18LrzRL2KgoWz9YvcY3cPomz8tih01GbHwnGk/OmkOKfqd/RAQoc8Lm7DQ==} @@ -2812,6 +2875,10 @@ packages: resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} engines: {node: '>=10'} + p-limit@4.0.0: + resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + p-locate@4.1.0: resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==} engines: {node: '>=8'} @@ -2875,6 +2942,9 @@ packages: pathe@1.1.2: resolution: {integrity: sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==} + pathval@1.1.1: + resolution: {integrity: sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==} + pathval@2.0.0: resolution: {integrity: sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA==} engines: {node: '>= 14.16'} @@ -2925,6 +2995,9 @@ packages: resolution: {integrity: sha512-ip4qdzjkAyDDZklUaZkcRFb2iA118H9SgRh8yzTkSQK8HilsOJF7rSY8HoW5+I0M46AZgX/pxbprf2vvzQCE0Q==} hasBin: true + pkg-types@1.2.0: + resolution: {integrity: sha512-+ifYuSSqOQ8CqP4MbZA5hDpb97n3E8SVWdJe+Wms9kj745lmd3b7EZJiqvmLwAlmRfjrI7Hi5z3kdBJ93lFNPA==} + postcss@8.4.38: resolution: {integrity: sha512-Wglpdk03BSfXkHoQa3b/oulrotAkwrlLDRSOb9D0bN86FdRyE9lppSp33aHNPgBa0JKCoB+drFLZkQoRRYae5A==} engines: {node: ^10 || ^12 || >=14} @@ -2955,6 +3028,10 @@ packages: engines: {node: '>=14'} hasBin: true + pretty-format@29.7.0: + resolution: {integrity: sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + proc-log@3.0.0: resolution: {integrity: sha512-++Vn7NS4Xf9NacaU9Xq3URUuqZETPsf8L4j5/ckhaRYsfPeRyzGw+iDjFhV/Jr3uNmTvvddEJFWh5R1gRgUH8A==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -3042,6 +3119,9 @@ packages: resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} hasBin: true + react-is@18.3.1: + resolution: {integrity: sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==} + read-package-json-fast@3.0.2: resolution: {integrity: sha512-0J+Msgym3vrLOUB3hzQCuZHII0xkNGCtz/HJH9xZshwv9DbDwkw1KaE3gx/e2J5rpEY5rtOy6cyhKOPrkP7FZw==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -3395,6 +3475,9 @@ packages: resolution: {integrity: sha512-0fk9zBqO67Nq5M/m45qHCJxylV/DhBlIOVExqgOMiCCrzrhU6tCibRXNqE3jwJLftzE9SNuZtYbpzcO+i9FiKw==} engines: {node: '>=14.16'} + strip-literal@1.3.0: + resolution: {integrity: sha512-PugKzOsyXpArk0yWmUwqOZecSO0GH0bPoctLcqNDH9J04pVW3lflYE0ujElBGTloevcxF5MofAOZ7C5l2b+wLg==} + supports-color@5.5.0: resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} engines: {node: '>=4'} @@ -3437,6 +3520,10 @@ packages: tinyexec@0.3.0: resolution: {integrity: sha512-tVGE0mVJPGb0chKhqmsoosjsS+qUnJVGJpZgsHYQcGoPlG3B51R3PouqTgEGH2Dc9jjFyOqOpix6ZHNMXp1FZg==} + tinypool@0.7.0: + resolution: {integrity: sha512-zSYNUlYSMhJ6Zdou4cJwo/p7w5nmAH17GRfU/ui3ctvjXFErXXkruT4MWW6poDeXgCaIBlGLrfU6TbTXxyGMww==} + engines: {node: '>=14.0.0'} + tinypool@1.0.1: resolution: {integrity: sha512-URZYihUbRPcGv95En+sz6MfghfIc2OJ1sv/RmhWZLouPY0/8Vo80viwPvg3dlaS9fuq7fQMEfgRRK7BBZThBEA==} engines: {node: ^18.0.0 || >=20.0.0} @@ -3445,6 +3532,10 @@ packages: resolution: {integrity: sha512-weEDEq7Z5eTHPDh4xjX789+fHfF+P8boiFB+0vbWzpbnbsEr/GRaohi/uMKxg8RZMXnl1ItAi/IUHWMsjDV7kQ==} engines: {node: '>=14.0.0'} + tinyspy@2.2.1: + resolution: {integrity: sha512-KYad6Vy5VDWV4GH3fjpseMQ/XU2BhIYP7Vzd0LG44qRWm/Yt2WCOTicFdvmgo6gWaqooMQCawTtILVQJupKu7A==} + engines: {node: '>=14.0.0'} + tinyspy@3.0.0: resolution: {integrity: sha512-q5nmENpTHgiPVd1cJDDc9cVoYN5x4vCvwT3FMilvKPKneCBZAxn2YWQjDF0UMcE9k0Cay1gBiDfTMU0g+mPMQA==} engines: {node: '>=14.0.0'} @@ -3532,6 +3623,10 @@ packages: resolution: {integrity: sha512-i3P9Kgw3ytjELUfpuKVDNBJvk4u5bXL6gskv572mcevPbSKCV3zt3djhmlEQ65yERjIbOSncy7U4cQJaB1CBCg==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} + type-detect@4.1.0: + resolution: {integrity: sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==} + engines: {node: '>=4'} + type-fest@0.21.3: resolution: {integrity: sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==} engines: {node: '>=10'} @@ -3552,6 +3647,9 @@ packages: engines: {node: '>=14.17'} hasBin: true + ufo@1.5.4: + resolution: {integrity: sha512-UsUk3byDzKd04EyoZ7U4DOlxQaD14JUKQl6/P7wiX4FNvUfm3XL246n9W5AmqwW5RSFJ27NAuM0iLscAOYUiGQ==} + undefsafe@2.0.5: resolution: {integrity: sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==} @@ -3614,6 +3712,11 @@ packages: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} + vite-node@0.34.6: + resolution: {integrity: sha512-nlBMJ9x6n7/Amaz6F3zJ97EBwR2FkzhBRxF5e+jE6LA3yi6Wtc2lyTij1OnDMIr34v5g/tVQtsVAzhT0jc5ygA==} + engines: {node: '>=v14.18.0'} + hasBin: true + vite-node@2.1.1: resolution: {integrity: sha512-N/mGckI1suG/5wQI35XeR9rsMsPqKXzq1CdUndzVstBj/HvyxxGctwnK6WX43NGt5L3Z5tcRf83g4TITKJhPrA==} engines: {node: ^18.0.0 || >=20.0.0} @@ -3655,6 +3758,37 @@ packages: terser: optional: true + vitest@0.34.6: + resolution: {integrity: sha512-+5CALsOvbNKnS+ZHMXtuUC7nL8/7F1F2DnHGjSsszX8zCjWSSviphCb/NuS9Nzf4Q03KyyDRBAXhF/8lffME4Q==} + engines: {node: '>=v14.18.0'} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@vitest/browser': '*' + '@vitest/ui': '*' + happy-dom: '*' + jsdom: '*' + playwright: '*' + safaridriver: '*' + webdriverio: '*' + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@vitest/browser': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + playwright: + optional: true + safaridriver: + optional: true + webdriverio: + optional: true + vitest@2.1.1: resolution: {integrity: sha512-97We7/VC0e9X5zBVkvt7SGQMGrRtn3KtySFQG5fpaMlS+l62eeXRQO633AYhSTC3z7IMebnPPNjGXVGNRFlxBA==} engines: {node: ^18.0.0 || >=20.0.0} @@ -3823,6 +3957,10 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + yocto-queue@1.1.1: + resolution: {integrity: sha512-b4JR1PFR10y1mKjhHY9LaGo6tmrgjit7hxVIeAmyMw3jegXR4dhYqLaQF5zMXZxY7tLpMyJeLjr1C4rLmkVe8g==} + engines: {node: '>=12.20'} + zod@3.23.8: resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==} @@ -4110,6 +4248,10 @@ snapshots: wrap-ansi: 8.1.0 wrap-ansi-cjs: wrap-ansi@7.0.0 + '@jest/schemas@29.6.3': + dependencies: + '@sinclair/typebox': 0.27.8 + '@jridgewell/resolve-uri@3.1.2': {} '@jridgewell/sourcemap-codec@1.4.15': {} @@ -4675,7 +4817,7 @@ snapshots: '@opentelemetry/semantic-conventions': 1.25.0 '@prisma/instrumentation': 5.15.0 '@sentry/core': 8.9.2 - '@sentry/opentelemetry': 8.9.2(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.52.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.0(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.0) + '@sentry/opentelemetry': 8.9.2(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.0(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.0(@opentelemetry/api@1.6.0))(@opentelemetry/sdk-trace-base@1.25.0(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.0) '@sentry/types': 8.9.2 '@sentry/utils': 8.9.2 optionalDependencies: @@ -4683,7 +4825,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@sentry/opentelemetry@8.9.2(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.52.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.0(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.0)': + '@sentry/opentelemetry@8.9.2(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.0(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.0(@opentelemetry/api@1.6.0))(@opentelemetry/sdk-trace-base@1.25.0(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.0)': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 1.25.0(@opentelemetry/api@1.9.0) @@ -4721,6 +4863,8 @@ snapshots: transitivePeerDependencies: - supports-color + '@sinclair/typebox@0.27.8': {} + '@sindresorhus/is@5.6.0': {} '@syncpoint/wkx@0.5.2': @@ -4759,6 +4903,12 @@ snapshots: '@types/connect': 3.4.38 '@types/node': 22.5.5 + '@types/chai-subset@1.3.5': + dependencies: + '@types/chai': 4.3.20 + + '@types/chai@4.3.20': {} + '@types/connect@3.4.36': dependencies: '@types/node': 22.5.5 @@ -4886,6 +5036,12 @@ snapshots: dependencies: '@types/node': 22.5.5 + '@vitest/expect@0.34.6': + dependencies: + '@vitest/spy': 0.34.6 + '@vitest/utils': 0.34.6 + chai: 4.5.0 + '@vitest/expect@2.1.1': dependencies: '@vitest/spy': 2.1.1 @@ -4905,21 +5061,43 @@ snapshots: dependencies: tinyrainbow: 1.2.0 + '@vitest/runner@0.34.6': + dependencies: + '@vitest/utils': 0.34.6 + p-limit: 4.0.0 + pathe: 1.1.2 + '@vitest/runner@2.1.1': dependencies: '@vitest/utils': 2.1.1 pathe: 1.1.2 + '@vitest/snapshot@0.34.6': + dependencies: + magic-string: 0.30.11 + pathe: 1.1.2 + pretty-format: 29.7.0 + '@vitest/snapshot@2.1.1': dependencies: '@vitest/pretty-format': 2.1.1 magic-string: 0.30.11 pathe: 1.1.2 + '@vitest/spy@0.34.6': + dependencies: + tinyspy: 2.2.1 + '@vitest/spy@2.1.1': dependencies: tinyspy: 3.0.0 + '@vitest/utils@0.34.6': + dependencies: + diff-sequences: 29.6.3 + loupe: 2.3.7 + pretty-format: 29.7.0 + '@vitest/utils@2.1.1': dependencies: '@vitest/pretty-format': 2.1.1 @@ -4999,6 +5177,8 @@ snapshots: dependencies: color-convert: 2.0.1 + ansi-styles@5.2.0: {} + ansi-styles@6.2.1: {} anymatch@3.1.3: @@ -5023,6 +5203,8 @@ snapshots: array-union@2.1.0: {} + assertion-error@1.1.0: {} + assertion-error@2.0.1: {} async-mutex@0.5.0: @@ -5163,6 +5345,16 @@ snapshots: camelcase@7.0.1: {} + chai@4.5.0: + dependencies: + assertion-error: 1.1.0 + check-error: 1.0.3 + deep-eql: 4.1.4 + get-func-name: 2.0.2 + loupe: 2.3.7 + pathval: 1.1.1 + type-detect: 4.1.0 + chai@5.1.1: dependencies: assertion-error: 2.0.1 @@ -5186,6 +5378,10 @@ snapshots: chardet@0.7.0: {} + check-error@1.0.3: + dependencies: + get-func-name: 2.0.2 + check-error@2.1.1: {} chokidar@3.6.0: @@ -5287,6 +5483,8 @@ snapshots: tree-kill: 1.2.2 yargs: 17.7.2 + confbox@0.1.7: {} + config-chain@1.1.13: dependencies: ini: 1.3.8 @@ -5361,6 +5559,10 @@ snapshots: dependencies: mimic-response: 3.1.0 + deep-eql@4.1.4: + dependencies: + type-detect: 4.1.0 + deep-eql@5.0.2: {} deep-extend@0.6.0: {} @@ -5381,6 +5583,8 @@ snapshots: detect-indent@6.1.0: {} + diff-sequences@29.6.3: {} + diff@4.0.2: {} dir-glob@3.0.1: @@ -5994,6 +6198,8 @@ snapshots: process-warning: 3.0.0 set-cookie-parser: 2.6.0 + local-pkg@0.4.3: {} + locate-path@5.0.0: dependencies: p-locate: 4.1.0 @@ -6024,6 +6230,10 @@ snapshots: lossless-json@2.0.11: {} + loupe@2.3.7: + dependencies: + get-func-name: 2.0.2 + loupe@3.1.1: dependencies: get-func-name: 2.0.2 @@ -6170,6 +6380,13 @@ snapshots: mkdirp@1.0.4: {} + mlly@1.7.1: + dependencies: + acorn: 8.11.3 + pathe: 1.1.2 + pkg-types: 1.2.0 + ufo: 1.5.4 + mnemonist@0.39.5: dependencies: obliterator: 2.0.4 @@ -6415,6 +6632,10 @@ snapshots: dependencies: yocto-queue: 0.1.0 + p-limit@4.0.0: + dependencies: + yocto-queue: 1.1.1 + p-locate@4.1.0: dependencies: p-limit: 2.3.0 @@ -6483,6 +6704,8 @@ snapshots: pathe@1.1.2: {} + pathval@1.1.1: {} + pathval@2.0.0: {} pause-stream@0.0.11: @@ -6537,6 +6760,12 @@ snapshots: sonic-boom: 3.8.1 thread-stream: 2.7.0 + pkg-types@1.2.0: + dependencies: + confbox: 0.1.7 + mlly: 1.7.1 + pathe: 1.1.2 + postcss@8.4.38: dependencies: nanoid: 3.3.7 @@ -6557,6 +6786,12 @@ snapshots: prettier@3.3.3: {} + pretty-format@29.7.0: + dependencies: + '@jest/schemas': 29.6.3 + ansi-styles: 5.2.0 + react-is: 18.3.1 + proc-log@3.0.0: {} process-nextick-args@2.0.1: {} @@ -6631,6 +6866,8 @@ snapshots: minimist: 1.2.8 strip-json-comments: 2.0.1 + react-is@18.3.1: {} + read-package-json-fast@3.0.2: dependencies: json-parse-even-better-errors: 3.0.2 @@ -6992,6 +7229,10 @@ snapshots: strip-json-comments@5.0.1: {} + strip-literal@1.3.0: + dependencies: + acorn: 8.11.3 + supports-color@5.5.0: dependencies: has-flag: 3.0.0 @@ -7034,10 +7275,14 @@ snapshots: tinyexec@0.3.0: {} + tinypool@0.7.0: {} + tinypool@1.0.1: {} tinyrainbow@1.2.0: {} + tinyspy@2.2.1: {} + tinyspy@3.0.0: {} tmp@0.0.33: @@ -7127,6 +7372,8 @@ snapshots: transitivePeerDependencies: - supports-color + type-detect@4.1.0: {} + type-fest@0.21.3: {} type-fest@1.4.0: {} @@ -7139,6 +7386,8 @@ snapshots: typescript@5.6.2: {} + ufo@1.5.4: {} + undefsafe@2.0.5: {} undici-types@6.19.8: {} @@ -7203,6 +7452,24 @@ snapshots: vary@1.1.2: {} + vite-node@0.34.6(@types/node@22.5.5): + dependencies: + cac: 6.7.14 + debug: 4.3.7 + mlly: 1.7.1 + pathe: 1.1.2 + picocolors: 1.1.0 + vite: 5.2.11(@types/node@22.5.5) + transitivePeerDependencies: + - '@types/node' + - less + - lightningcss + - sass + - stylus + - sugarss + - supports-color + - terser + vite-node@2.1.1(@types/node@22.5.5): dependencies: cac: 6.7.14 @@ -7239,6 +7506,41 @@ snapshots: '@types/node': 22.5.5 fsevents: 2.3.3 + vitest@0.34.6: + dependencies: + '@types/chai': 4.3.20 + '@types/chai-subset': 1.3.5 + '@types/node': 22.5.5 + '@vitest/expect': 0.34.6 + '@vitest/runner': 0.34.6 + '@vitest/snapshot': 0.34.6 + '@vitest/spy': 0.34.6 + '@vitest/utils': 0.34.6 + acorn: 8.11.3 + acorn-walk: 8.3.2 + cac: 6.7.14 + chai: 4.5.0 + debug: 4.3.7 + local-pkg: 0.4.3 + magic-string: 0.30.11 + pathe: 1.1.2 + picocolors: 1.1.0 + std-env: 3.7.0 + strip-literal: 1.3.0 + tinybench: 2.9.0 + tinypool: 0.7.0 + vite: 5.2.11(@types/node@22.5.5) + vite-node: 0.34.6(@types/node@22.5.5) + why-is-node-running: 2.3.0 + transitivePeerDependencies: + - less + - lightningcss + - sass + - stylus + - sugarss + - supports-color + - terser + vitest@2.1.1(@types/node@22.5.5): dependencies: '@vitest/expect': 2.1.1 @@ -7403,4 +7705,6 @@ snapshots: yocto-queue@0.1.0: {} + yocto-queue@1.1.1: {} + zod@3.23.8: {} From b85150cfbfca0b5c18e8b6d28660277948f6dead Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 2 Oct 2024 12:05:09 +0200 Subject: [PATCH 31/35] Implement basic diagnostics apis for MongoDB. --- .../src/api/MongoRouteAPIAdapter.ts | 113 ++++++++++++++++-- .../src/replication/ChangeStream.ts | 6 +- modules/module-mongodb/src/utils.ts | 4 + packages/service-core/src/api/RouteAPI.ts | 2 +- 4 files changed, 111 insertions(+), 14 deletions(-) create mode 100644 modules/module-mongodb/src/utils.ts diff --git a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts index e3a3eea72..bcf53b195 100644 --- a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts +++ b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts @@ -1,4 +1,4 @@ -import { api, ParseSyncRulesOptions } from '@powersync/service-core'; +import { api, ParseSyncRulesOptions, SourceTable } from '@powersync/service-core'; import * as mongo from 'mongodb'; import * as sync_rules from '@powersync/service-sync-rules'; @@ -6,6 +6,7 @@ import * as service_types from '@powersync/service-types'; import * as types from '../types/types.js'; import { MongoManager } from '../replication/MongoManager.js'; import { createCheckpoint, getMongoLsn } from '../replication/MongoRelation.js'; +import { escapeRegExp } from '../utils.js'; export class MongoRouteAPIAdapter implements api.RouteAPI { protected client: mongo.MongoClient; @@ -37,11 +38,21 @@ export class MongoRouteAPIAdapter implements api.RouteAPI { } async getConnectionStatus(): Promise { - // TODO: Implement const base = { id: this.config.id, uri: types.baseUri(this.config) }; + + try { + await this.client.connect(); + await this.db.command({ hello: 1 }); + } catch (e) { + return { + ...base, + connected: false, + errors: [{ level: 'fatal', message: e.message }] + }; + } return { ...base, connected: true, @@ -64,14 +75,100 @@ export class MongoRouteAPIAdapter implements api.RouteAPI { tablePatterns: sync_rules.TablePattern[], sqlSyncRules: sync_rules.SqlSyncRules ): Promise { - // TODO: Implement - return []; - } + let result: api.PatternResult[] = []; + for (let tablePattern of tablePatterns) { + const schema = tablePattern.schema; - async getReplicationLag(syncRulesId: string): Promise { - // TODO: Implement + let patternResult: api.PatternResult = { + schema: schema, + pattern: tablePattern.tablePattern, + wildcard: tablePattern.isWildcard + }; + result.push(patternResult); + + let nameFilter: RegExp | string; + if (tablePattern.isWildcard) { + nameFilter = new RegExp('^' + escapeRegExp(tablePattern.tablePrefix)); + } else { + nameFilter = tablePattern.name; + } + + // Check if the collection exists + const collections = await this.client + .db(schema) + .listCollections( + { + name: nameFilter + }, + { nameOnly: true } + ) + .toArray(); + + if (tablePattern.isWildcard) { + patternResult.tables = []; + for (let collection of collections) { + const sourceTable = new SourceTable( + 0, + this.connectionTag, + collection.name, + schema, + collection.name, + [], + true + ); + const syncData = sqlSyncRules.tableSyncsData(sourceTable); + const syncParameters = sqlSyncRules.tableSyncsParameters(sourceTable); + patternResult.tables.push({ + schema, + name: collection.name, + replication_id: ['_id'], + data_queries: syncData, + parameter_queries: syncParameters, + errors: [] + }); + } + } else { + const sourceTable = new SourceTable( + 0, + this.connectionTag, + tablePattern.name, + schema, + tablePattern.name, + [], + true + ); + + const syncData = sqlSyncRules.tableSyncsData(sourceTable); + const syncParameters = sqlSyncRules.tableSyncsParameters(sourceTable); + + if (collections.length == 1) { + patternResult.table = { + schema, + name: tablePattern.name, + replication_id: ['_id'], + data_queries: syncData, + parameter_queries: syncParameters, + errors: [] + }; + } else { + patternResult.table = { + schema, + name: tablePattern.name, + replication_id: ['_id'], + data_queries: syncData, + parameter_queries: syncParameters, + errors: [{ level: 'warning', message: `Collection ${schema}.${tablePattern.name} not found` }] + }; + } + } + } + return result; + } - return 0; + async getReplicationLag(syncRulesId: string): Promise { + // There is no fast way to get replication lag in bytes in MongoDB. + // We can get replication lag in seconds, but need a different API for that. + return undefined; } async getReplicationHead(): Promise { diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 92392d863..3d36e1814 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -10,6 +10,7 @@ import { getMongoRelation, mongoLsnToTimestamp } from './MongoRelation.js'; +import { escapeRegExp } from '../utils.js'; export const ZERO_LSN = '0000000000000000'; @@ -496,8 +497,3 @@ async function touch() { // or reduce PING_INTERVAL here. return container.probes.touch(); } - -function escapeRegExp(string: string) { - // https://stackoverflow.com/a/3561711/214837 - return string.replace(/[/\-\\^$*+?.()|[\]{}]/g, '\\$&'); -} diff --git a/modules/module-mongodb/src/utils.ts b/modules/module-mongodb/src/utils.ts new file mode 100644 index 000000000..badee3083 --- /dev/null +++ b/modules/module-mongodb/src/utils.ts @@ -0,0 +1,4 @@ +export function escapeRegExp(string: string) { + // https://stackoverflow.com/a/3561711/214837 + return string.replace(/[/\-\\^$*+?.()|[\]{}]/g, '\\$&'); +} diff --git a/packages/service-core/src/api/RouteAPI.ts b/packages/service-core/src/api/RouteAPI.ts index 393d56342..f77f5c26b 100644 --- a/packages/service-core/src/api/RouteAPI.ts +++ b/packages/service-core/src/api/RouteAPI.ts @@ -44,7 +44,7 @@ export interface RouteAPI { * replicated yet, in bytes. * @param {string} syncRulesId An identifier representing which set of sync rules the lag is required for. */ - getReplicationLag(syncRulesId: string): Promise; + getReplicationLag(syncRulesId: string): Promise; /** * Get the current LSN or equivalent replication HEAD position identifier From b863b67032c36e2a169d4943e711c1ab4eec641a Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 2 Oct 2024 12:44:06 +0200 Subject: [PATCH 32/35] Add test for getConnectionSchema. --- .../src/api/MongoRouteAPIAdapter.ts | 116 +++++++++++++++++- .../test/src/mongo_test.test.ts | 49 +++++++- 2 files changed, 158 insertions(+), 7 deletions(-) diff --git a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts index bcf53b195..508f30e38 100644 --- a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts +++ b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts @@ -5,12 +5,12 @@ import * as sync_rules from '@powersync/service-sync-rules'; import * as service_types from '@powersync/service-types'; import * as types from '../types/types.js'; import { MongoManager } from '../replication/MongoManager.js'; -import { createCheckpoint, getMongoLsn } from '../replication/MongoRelation.js'; +import { constructAfterRecord, createCheckpoint, getMongoLsn } from '../replication/MongoRelation.js'; import { escapeRegExp } from '../utils.js'; export class MongoRouteAPIAdapter implements api.RouteAPI { protected client: mongo.MongoClient; - private db: mongo.Db; + public db: mongo.Db; connectionTag: string; defaultSchema: string; @@ -176,8 +176,116 @@ export class MongoRouteAPIAdapter implements api.RouteAPI { } async getConnectionSchema(): Promise { - // TODO: Implement + const sampleSize = 50; - return []; + const databases = await this.db.admin().listDatabases({ authorizedDatabases: true, nameOnly: true }); + return ( + await Promise.all( + databases.databases.map(async (db) => { + if (db.name == 'local' || db.name == 'admin') { + return null; + } + const collections = await this.client.db(db.name).listCollections().toArray(); + + const tables = await Promise.all( + collections.map(async (collection) => { + const sampleDocuments = await this.db + .collection(collection.name) + .aggregate([{ $sample: { size: sampleSize } }]) + .toArray(); + + if (sampleDocuments.length > 0) { + const columns = this.getColumnsFromDocuments(sampleDocuments); + + return { + name: collection.name, + // Since documents are sampled in a random order, we need to sort + // to get a consistent order + columns: columns.sort((a, b) => a.name.localeCompare(b.name)) + } satisfies service_types.TableSchema; + } else { + return { + name: collection.name, + columns: [] + } satisfies service_types.TableSchema; + } + }) + ); + return { + name: db.name, + tables: tables + } satisfies service_types.DatabaseSchema; + }) + ) + ).filter((r) => r != null); + } + + private getColumnsFromDocuments(documents: mongo.BSON.Document[]) { + let columns = new Map }>(); + for (const document of documents) { + const parsed = constructAfterRecord(document); + for (const key in parsed) { + const value = parsed[key]; + const type = sync_rules.sqliteTypeOf(value); + const sqliteType = sync_rules.ExpressionType.fromTypeText(type); + let entry = columns.get(key); + if (entry == null) { + entry = { sqliteType, bsonTypes: new Set() }; + columns.set(key, entry); + } else { + entry.sqliteType = entry.sqliteType.or(sqliteType); + } + const bsonType = this.getBsonType(document[key]); + if (bsonType != null) { + entry.bsonTypes.add(bsonType); + } + } + } + return [...columns.entries()].map(([key, value]) => { + return { + name: key, + sqlite_type: value.sqliteType.typeFlags, + internal_type: value.bsonTypes.size == 0 ? '' : [...value.bsonTypes].join(' | ') + }; + }); + } + + private getBsonType(data: any): string | null { + if (data == null) { + // null or undefined + return 'Null'; + } else if (typeof data == 'string') { + return 'String'; + } else if (typeof data == 'number') { + if (Number.isInteger(data)) { + return 'Integer'; + } else { + return 'Double'; + } + } else if (typeof data == 'bigint') { + return 'Long'; + } else if (typeof data == 'boolean') { + return 'Boolean'; + } else if (data instanceof mongo.ObjectId) { + return 'ObjectId'; + } else if (data instanceof mongo.UUID) { + return 'UUID'; + } else if (data instanceof Date) { + return 'Date'; + } else if (data instanceof mongo.Timestamp) { + return 'Timestamp'; + } else if (data instanceof mongo.Binary) { + return 'Binary'; + } else if (data instanceof mongo.Long) { + return 'Long'; + } else if (Array.isArray(data)) { + return 'Array'; + } else if (data instanceof Uint8Array) { + return 'Binary'; + } else if (typeof data == 'object') { + return 'Object'; + } else { + return null; + } } } diff --git a/modules/module-mongodb/test/src/mongo_test.test.ts b/modules/module-mongodb/test/src/mongo_test.test.ts index 1b36dab90..ed03ebcd9 100644 --- a/modules/module-mongodb/test/src/mongo_test.test.ts +++ b/modules/module-mongodb/test/src/mongo_test.test.ts @@ -1,9 +1,10 @@ +import { MongoRouteAPIAdapter } from '@module/api/MongoRouteAPIAdapter.js'; import { ChangeStream } from '@module/replication/ChangeStream.js'; +import { constructAfterRecord } from '@module/replication/MongoRelation.js'; +import { SqliteRow } from '@powersync/service-sync-rules'; import * as mongo from 'mongodb'; import { describe, expect, test } from 'vitest'; -import { clearTestDb, connectMongoData } from './util.js'; -import { SqliteRow } from '@powersync/service-sync-rules'; -import { constructAfterRecord } from '@module/replication/MongoRelation.js'; +import { clearTestDb, connectMongoData, TEST_CONNECTION_OPTIONS } from './util.js'; describe('mongo data types', () => { async function setupTable(db: mongo.Db) { @@ -202,6 +203,48 @@ describe('mongo data types', () => { await client.close(); } }); + + test('connection schema', async () => { + const adapter = new MongoRouteAPIAdapter({ + type: 'mongodb', + ...TEST_CONNECTION_OPTIONS + }); + try { + const db = adapter.db; + await clearTestDb(db); + + const collection = db.collection('test_data'); + await setupTable(db); + await insert(collection); + + const schema = await adapter.getConnectionSchema(); + const dbSchema = schema.filter((s) => s.name == TEST_CONNECTION_OPTIONS.database)[0]; + expect(dbSchema).not.toBeNull(); + expect(dbSchema.tables).toEqual([ + { + name: 'test_data', + columns: [ + { name: '_id', sqlite_type: 4, internal_type: 'Integer' }, + { name: 'bool', sqlite_type: 4, internal_type: 'Boolean' }, + { name: 'bytea', sqlite_type: 1, internal_type: 'Binary' }, + { name: 'date', sqlite_type: 2, internal_type: 'Date' }, + { name: 'float', sqlite_type: 8, internal_type: 'Double' }, + { name: 'int2', sqlite_type: 4, internal_type: 'Integer' }, + { name: 'int4', sqlite_type: 4, internal_type: 'Integer' }, + { name: 'int8', sqlite_type: 4, internal_type: 'Long' }, + { name: 'nested', sqlite_type: 2, internal_type: 'Object' }, + { name: 'null', sqlite_type: 0, internal_type: 'Null' }, + { name: 'objectId', sqlite_type: 2, internal_type: 'ObjectId' }, + { name: 'text', sqlite_type: 2, internal_type: 'String' }, + { name: 'timestamp', sqlite_type: 4, internal_type: 'Timestamp' }, + { name: 'uuid', sqlite_type: 2, internal_type: 'UUID' } + ] + } + ]); + } finally { + await adapter.shutdown(); + } + }); }); /** From 5050204a5f7e3748533708e237d5134177a53144 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 2 Oct 2024 12:56:08 +0200 Subject: [PATCH 33/35] Improve schema filtering; return defaultSchema in API. --- .../src/api/MongoRouteAPIAdapter.ts | 73 ++++++++++--------- packages/service-core/src/api/RouteAPI.ts | 3 - packages/service-core/src/api/schema.ts | 8 +- packages/types/src/definitions.ts | 4 +- 4 files changed, 46 insertions(+), 42 deletions(-) diff --git a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts index 508f30e38..1297a67b1 100644 --- a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts +++ b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts @@ -179,45 +179,46 @@ export class MongoRouteAPIAdapter implements api.RouteAPI { const sampleSize = 50; const databases = await this.db.admin().listDatabases({ authorizedDatabases: true, nameOnly: true }); - return ( - await Promise.all( - databases.databases.map(async (db) => { - if (db.name == 'local' || db.name == 'admin') { - return null; - } - const collections = await this.client.db(db.name).listCollections().toArray(); + const filteredDatabases = databases.databases.filter((db) => { + return !['local', 'admin', 'config'].includes(db.name); + }); + return await Promise.all( + filteredDatabases.map(async (db) => { + const collections = await this.client.db(db.name).listCollections().toArray(); + const filtered = collections.filter((c) => { + return !['_powersync_checkpoints'].includes(c.name); + }); - const tables = await Promise.all( - collections.map(async (collection) => { - const sampleDocuments = await this.db - .collection(collection.name) - .aggregate([{ $sample: { size: sampleSize } }]) - .toArray(); + const tables = await Promise.all( + filtered.map(async (collection) => { + const sampleDocuments = await this.db + .collection(collection.name) + .aggregate([{ $sample: { size: sampleSize } }]) + .toArray(); - if (sampleDocuments.length > 0) { - const columns = this.getColumnsFromDocuments(sampleDocuments); + if (sampleDocuments.length > 0) { + const columns = this.getColumnsFromDocuments(sampleDocuments); - return { - name: collection.name, - // Since documents are sampled in a random order, we need to sort - // to get a consistent order - columns: columns.sort((a, b) => a.name.localeCompare(b.name)) - } satisfies service_types.TableSchema; - } else { - return { - name: collection.name, - columns: [] - } satisfies service_types.TableSchema; - } - }) - ); - return { - name: db.name, - tables: tables - } satisfies service_types.DatabaseSchema; - }) - ) - ).filter((r) => r != null); + return { + name: collection.name, + // Since documents are sampled in a random order, we need to sort + // to get a consistent order + columns: columns.sort((a, b) => a.name.localeCompare(b.name)) + } satisfies service_types.TableSchema; + } else { + return { + name: collection.name, + columns: [] + } satisfies service_types.TableSchema; + } + }) + ); + return { + name: db.name, + tables: tables + } satisfies service_types.DatabaseSchema; + }) + ); } private getColumnsFromDocuments(documents: mongo.BSON.Document[]) { diff --git a/packages/service-core/src/api/RouteAPI.ts b/packages/service-core/src/api/RouteAPI.ts index f77f5c26b..bbe92180c 100644 --- a/packages/service-core/src/api/RouteAPI.ts +++ b/packages/service-core/src/api/RouteAPI.ts @@ -54,9 +54,6 @@ export interface RouteAPI { /** * @returns The schema for tables inside the connected database. This is typically * used to validate sync rules. - * Side Note: https://github.com/powersync-ja/powersync-service/blob/33bbb8c0ab1c48555956593f427fc674a8f15768/packages/types/src/definitions.ts#L100 - * contains `pg_type` which we might need to deprecate and add another generic - * type field - or just use this field as the connection specific type. */ getConnectionSchema(): Promise; diff --git a/packages/service-core/src/api/schema.ts b/packages/service-core/src/api/schema.ts index aff6d770f..64b48e1a0 100644 --- a/packages/service-core/src/api/schema.ts +++ b/packages/service-core/src/api/schema.ts @@ -5,7 +5,9 @@ import * as api from '../api/api-index.js'; export async function getConnectionsSchema(api: api.RouteAPI): Promise { if (!api) { return { - connections: [] + connections: [], + defaultConnectionTag: 'default', + defaultSchema: '' }; } @@ -18,6 +20,8 @@ export async function getConnectionsSchema(api: api.RouteAPI): Promise; From 4a654f1f3f9608270e3747902f6a0e083ba0e673 Mon Sep 17 00:00:00 2001 From: stevensJourney <51082125+stevensJourney@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:20:11 +0200 Subject: [PATCH 34/35] list all databases (#96) --- .../src/api/MongoRouteAPIAdapter.ts | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts index 1297a67b1..589870f66 100644 --- a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts +++ b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts @@ -3,9 +3,9 @@ import * as mongo from 'mongodb'; import * as sync_rules from '@powersync/service-sync-rules'; import * as service_types from '@powersync/service-types'; -import * as types from '../types/types.js'; import { MongoManager } from '../replication/MongoManager.js'; -import { constructAfterRecord, createCheckpoint, getMongoLsn } from '../replication/MongoRelation.js'; +import { constructAfterRecord, createCheckpoint } from '../replication/MongoRelation.js'; +import * as types from '../types/types.js'; import { escapeRegExp } from '../utils.js'; export class MongoRouteAPIAdapter implements api.RouteAPI { @@ -178,13 +178,24 @@ export class MongoRouteAPIAdapter implements api.RouteAPI { async getConnectionSchema(): Promise { const sampleSize = 50; - const databases = await this.db.admin().listDatabases({ authorizedDatabases: true, nameOnly: true }); + const databases = await this.db.admin().listDatabases({ nameOnly: true }); const filteredDatabases = databases.databases.filter((db) => { return !['local', 'admin', 'config'].includes(db.name); }); - return await Promise.all( + const databaseSchemas = await Promise.all( filteredDatabases.map(async (db) => { - const collections = await this.client.db(db.name).listCollections().toArray(); + /** + * Filtering the list of database with `authorizedDatabases: true` + * does not produce the full list of databases under some circumstances. + * This catches any potential auth errors. + */ + let collections: mongo.CollectionInfo[]; + try { + collections = await this.client.db(db.name).listCollections().toArray(); + } catch (ex) { + return null; + } + const filtered = collections.filter((c) => { return !['_powersync_checkpoints'].includes(c.name); }); @@ -219,6 +230,7 @@ export class MongoRouteAPIAdapter implements api.RouteAPI { } satisfies service_types.DatabaseSchema; }) ); + return databaseSchemas.filter((schema) => !!schema); } private getColumnsFromDocuments(documents: mongo.BSON.Document[]) { From 9683172496c41e2f67b380f396439b7cefa94d41 Mon Sep 17 00:00:00 2001 From: stevensJourney <51082125+stevensJourney@users.noreply.github.com> Date: Mon, 7 Oct 2024 16:50:14 +0200 Subject: [PATCH 35/35] [MongoDB] Schema Endpoint Fix (#97) * test: populate schema type field * pg_type restore --- modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts | 5 ++++- packages/types/src/definitions.ts | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts index 589870f66..e96cc8cb8 100644 --- a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts +++ b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts @@ -255,10 +255,13 @@ export class MongoRouteAPIAdapter implements api.RouteAPI { } } return [...columns.entries()].map(([key, value]) => { + const internal_type = value.bsonTypes.size == 0 ? '' : [...value.bsonTypes].join(' | '); return { name: key, + type: internal_type, sqlite_type: value.sqliteType.typeFlags, - internal_type: value.bsonTypes.size == 0 ? '' : [...value.bsonTypes].join(' | ') + internal_type, + pg_type: internal_type }; }); } diff --git a/packages/types/src/definitions.ts b/packages/types/src/definitions.ts index 9f176c62a..dc3bdf879 100644 --- a/packages/types/src/definitions.ts +++ b/packages/types/src/definitions.ts @@ -126,13 +126,13 @@ export const TableSchema = t.object({ * Full type name, e.g. "character varying(255)[]" * @deprecated - use internal_type */ - type: t.string.optional(), + type: t.string, /** * Internal postgres type, e.g. "varchar[]". * @deprecated - use internal_type instead */ - pg_type: t.string.optional() + pg_type: t.string }) ) });