diff --git a/.changeset/perfect-kiwis-prove.md b/.changeset/perfect-kiwis-prove.md new file mode 100644 index 00000000..2fdf4c5d --- /dev/null +++ b/.changeset/perfect-kiwis-prove.md @@ -0,0 +1,10 @@ +--- +'@powersync/service-module-mongodb-storage': patch +'@powersync/service-rsocket-router': patch +'@powersync/service-module-mongodb': patch +'@powersync/service-core': patch +'@powersync/lib-services-framework': patch +'@powersync/lib-service-mongodb': patch +--- + +Upgrade mongodb driver to improve stability. diff --git a/libs/lib-mongodb/package.json b/libs/lib-mongodb/package.json index f20f2cbb..93e05e8e 100644 --- a/libs/lib-mongodb/package.json +++ b/libs/lib-mongodb/package.json @@ -29,10 +29,10 @@ }, "dependencies": { "@powersync/lib-services-framework": "workspace:*", - "bson": "^6.10.3", - "mongodb": "^6.14.1", + "bson": "^6.10.4", + "mongodb": "^6.20.0", "mongodb-connection-string-url": "^3.0.2", "ts-codec": "^1.3.0" }, "devDependencies": {} -} +} \ No newline at end of file diff --git a/libs/lib-mongodb/src/db/mongo.ts b/libs/lib-mongodb/src/db/mongo.ts index 0f8f9dff..b57d833f 100644 --- a/libs/lib-mongodb/src/db/mongo.ts +++ b/libs/lib-mongodb/src/db/mongo.ts @@ -61,6 +61,7 @@ export function createMongoClient(config: BaseMongoConfigDecoded, options?: Mong // Identify the client appName: options?.powersyncVersion ? `powersync-storage ${options.powersyncVersion}` : 'powersync-storage', + // Deprecated in the driver - in a future release we may have to rely on appName only. driverInfo: { // This is merged with the node driver info. name: 'powersync-storage', diff --git a/libs/lib-services/package.json b/libs/lib-services/package.json index a9594775..540dc011 100644 --- a/libs/lib-services/package.json +++ b/libs/lib-services/package.json @@ -24,7 +24,7 @@ "@powersync/service-sync-rules": "workspace:*", "ajv": "^8.12.0", "better-ajv-errors": "^1.2.0", - "bson": "^6.10.3", + "bson": "^6.10.4", "dotenv": "^16.4.5", "ipaddr.js": "^2.1.0", "lodash": "^4.17.21", @@ -37,4 +37,4 @@ "@types/lodash": "^4.17.5", "vitest": "^3.2.4" } -} +} \ No newline at end of file diff --git a/modules/module-mongodb-storage/package.json b/modules/module-mongodb-storage/package.json index e3ef6c1c..3cc9047d 100644 --- a/modules/module-mongodb-storage/package.json +++ b/modules/module-mongodb-storage/package.json @@ -34,7 +34,7 @@ "@powersync/service-jsonbig": "workspace:*", "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", - "bson": "^6.10.3", + "bson": "^6.10.4", "ix": "^5.0.0", "lru-cache": "^10.2.2", "ts-codec": "^1.3.0", @@ -43,4 +43,4 @@ "devDependencies": { "@powersync/service-core-tests": "workspace:*" } -} +} \ No newline at end of file diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts index b3b9830d..e81d566c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts @@ -184,7 +184,11 @@ export class MongoCompactor { } } ], - { batchSize: this.moveBatchQueryLimit } + { + // batchSize is 1 more than limit to auto-close the cursor. + // See https://github.com/mongodb/node-mongodb-native/pull/4580 + batchSize: this.moveBatchQueryLimit + 1 + } ); // We don't limit to a single batch here, since that often causes MongoDB to scan through more than it returns. // Instead, we load up to the limit. diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index bc199bdf..939abfde 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -403,7 +403,9 @@ export class MongoSyncBucketStorage limit: batchLimit, // Increase batch size above the default 101, so that we can fill an entire batch in // one go. - batchSize: batchLimit, + // batchSize is 1 more than limit to auto-close the cursor. + // See https://github.com/mongodb/node-mongodb-native/pull/4580 + batchSize: batchLimit + 1, // Raw mode is returns an array of Buffer instead of parsed documents. // We use it so that: // 1. We can calculate the document size accurately without serializing again. @@ -905,7 +907,9 @@ export class MongoSyncBucketStorage '_id.b': 1 }, limit: limit + 1, - batchSize: limit + 1, + // batchSize is 1 more than limit to auto-close the cursor. + // See https://github.com/mongodb/node-mongodb-native/pull/4580 + batchSize: limit + 2, singleBatch: true } ) @@ -935,7 +939,9 @@ export class MongoSyncBucketStorage lookup: 1 }, limit: limit + 1, - batchSize: limit + 1, + // batchSize is 1 more than limit to auto-close the cursor. + // See https://github.com/mongodb/node-mongodb-native/pull/4580 + batchSize: limit + 2, singleBatch: true } ) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoWriteCheckpointAPI.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoWriteCheckpointAPI.ts index 4baf65e7..f77ab1a8 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoWriteCheckpointAPI.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoWriteCheckpointAPI.ts @@ -111,7 +111,9 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI { }, { limit: limit + 1, - batchSize: limit + 1, + // batchSize is 1 more than limit to auto-close the cursor. + // See https://github.com/mongodb/node-mongodb-native/pull/4580 + batchSize: limit + 2, singleBatch: true } ) @@ -140,7 +142,9 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI { }, { limit: limit + 1, - batchSize: limit + 1, + // batchSize is 1 more than limit to auto-close the cursor. + // See https://github.com/mongodb/node-mongodb-native/pull/4580 + batchSize: limit + 2, singleBatch: true } ) diff --git a/modules/module-mongodb-storage/src/storage/implementation/util.ts b/modules/module-mongodb-storage/src/storage/implementation/util.ts index cae1a5b9..6c2c2c7f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/util.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/util.ts @@ -41,7 +41,7 @@ export function generateSlotName(prefix: string, sync_rules_id: number) { * However, that makes `has_more` detection very difficult, since the cursor is always closed * after the first batch. Instead, we do a workaround to only fetch a single batch below. * - * For this to be effective, set batchSize = limit in the find command. + * For this to be effective, set batchSize = limit + 1 in the find command. */ export async function readSingleBatch(cursor: mongo.AbstractCursor): Promise<{ data: T[]; hasMore: boolean }> { try { diff --git a/modules/module-mongodb/package.json b/modules/module-mongodb/package.json index cf29d90e..e517fc50 100644 --- a/modules/module-mongodb/package.json +++ b/modules/module-mongodb/package.json @@ -34,7 +34,7 @@ "@powersync/service-jsonbig": "workspace:*", "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", - "bson": "^6.10.3", + "bson": "^6.10.4", "ts-codec": "^1.3.0", "uuid": "^11.1.0" }, @@ -43,4 +43,4 @@ "@powersync/service-module-mongodb-storage": "workspace:*", "@powersync/service-module-postgres-storage": "workspace:*" } -} +} \ No newline at end of file diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 9ccb1fc6..62d3cda1 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -984,8 +984,8 @@ export class ChangeStream { // Checkpoint out of order - should never happen with MongoDB. // If it does happen, we throw an error to stop the replication - restarting should recover. // Since we use batch.lastCheckpointLsn for the next resumeAfter, this should not result in an infinite loop. - // This is a workaround for the issue below, but we can keep this as a safety-check even if the issue is fixed. - // Driver issue report: https://jira.mongodb.org/browse/NODE-7042 + // Originally a workaround for https://jira.mongodb.org/browse/NODE-7042. + // This has been fixed in the driver in the meantime, but we still keep this as a safety-check. throw new ReplicationAssertionError( `Change resumeToken ${(changeDocument._id as any)._data} (${timestampToDate(changeDocument.clusterTime!).toISOString()}) is less than last checkpoint LSN ${batch.lastCheckpointLsn}. Restarting replication.` ); diff --git a/modules/module-mongodb/src/replication/MongoManager.ts b/modules/module-mongodb/src/replication/MongoManager.ts index b66878f4..3290c22c 100644 --- a/modules/module-mongodb/src/replication/MongoManager.ts +++ b/modules/module-mongodb/src/replication/MongoManager.ts @@ -31,6 +31,7 @@ export class MongoManager { // Identify the client appName: `powersync ${POWERSYNC_VERSION}`, + // Deprecated in the driver - in a future release we may have to rely on appName only. driverInfo: { // This is merged with the node driver info. name: 'powersync', diff --git a/modules/module-mongodb/src/replication/MongoSnapshotQuery.ts b/modules/module-mongodb/src/replication/MongoSnapshotQuery.ts index 501563dc..64b64f9b 100644 --- a/modules/module-mongodb/src/replication/MongoSnapshotQuery.ts +++ b/modules/module-mongodb/src/replication/MongoSnapshotQuery.ts @@ -38,9 +38,11 @@ export class ChunkedSnapshotQuery implements AsyncDisposable { const filter: mongo.Filter = this.lastKey == null ? {} : { $expr: { $gt: ['$_id', { $literal: this.lastKey }] } }; cursor = this.collection.find(filter, { - batchSize: this.batchSize, readConcern: 'majority', limit: this.batchSize, + // batchSize is 1 more than limit to auto-close the cursor. + // See https://github.com/mongodb/node-mongodb-native/pull/4580 + batchSize: this.batchSize + 1, sort: { _id: 1 } }); newCursor = true; diff --git a/package.json b/package.json index baa66734..0bcbfed1 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "@changesets/cli": "^2.27.8", "@types/node": "^22.16.2", "async": "^3.2.4", - "bson": "^6.10.3", + "bson": "^6.10.4", "concurrently": "^8.2.2", "inquirer": "^9.2.7", "npm-check-updates": "^17.1.2", diff --git a/packages/rsocket-router/package.json b/packages/rsocket-router/package.json index faf875ba..6b11aaae 100644 --- a/packages/rsocket-router/package.json +++ b/packages/rsocket-router/package.json @@ -26,7 +26,7 @@ }, "devDependencies": { "@types/ws": "~8.2.0", - "bson": "^6.10.3", + "bson": "^6.10.4", "rsocket-websocket-client": "1.0.0-alpha.3" } -} +} \ No newline at end of file diff --git a/packages/service-core/package.json b/packages/service-core/package.json index 27c146ad..5165b59f 100644 --- a/packages/service-core/package.json +++ b/packages/service-core/package.json @@ -29,7 +29,7 @@ "@powersync/service-types": "workspace:*", "async": "^3.2.4", "async-mutex": "^0.5.0", - "bson": "^6.10.3", + "bson": "^6.10.4", "commander": "^12.0.0", "cors": "^2.8.5", "ipaddr.js": "^2.1.0", @@ -52,4 +52,4 @@ "fastify": "^5.4.0", "fastify-plugin": "^5.0.1" } -} +} \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 08656dd5..327052e4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -18,8 +18,8 @@ importers: specifier: ^3.2.4 version: 3.2.5 bson: - specifier: ^6.10.3 - version: 6.10.3 + specifier: ^6.10.4 + version: 6.10.4 concurrently: specifier: ^8.2.2 version: 8.2.2 @@ -72,11 +72,11 @@ importers: specifier: workspace:* version: link:../lib-services bson: - specifier: ^6.10.3 - version: 6.10.3 + specifier: ^6.10.4 + version: 6.10.4 mongodb: - specifier: ^6.14.1 - version: 6.14.2(socks@2.8.3) + specifier: ^6.20.0 + version: 6.20.0(socks@2.8.3) mongodb-connection-string-url: specifier: ^3.0.2 version: 3.0.2 @@ -123,8 +123,8 @@ importers: specifier: ^1.2.0 version: 1.2.0(ajv@8.16.0) bson: - specifier: ^6.10.3 - version: 6.10.3 + specifier: ^6.10.4 + version: 6.10.4 dotenv: specifier: ^16.4.5 version: 16.4.5 @@ -196,8 +196,8 @@ importers: specifier: workspace:* version: link:../../packages/types bson: - specifier: ^6.10.3 - version: 6.10.3 + specifier: ^6.10.4 + version: 6.10.4 ts-codec: specifier: ^1.3.0 version: 1.3.0 @@ -236,8 +236,8 @@ importers: specifier: workspace:* version: link:../../packages/types bson: - specifier: ^6.10.3 - version: 6.10.3 + specifier: ^6.10.4 + version: 6.10.4 ix: specifier: ^5.0.0 version: 5.0.0 @@ -461,8 +461,8 @@ importers: specifier: ~8.2.0 version: 8.2.3 bson: - specifier: ^6.10.3 - version: 6.10.3 + specifier: ^6.10.4 + version: 6.10.4 rsocket-websocket-client: specifier: 1.0.0-alpha.3 version: 1.0.0-alpha.3 @@ -533,8 +533,8 @@ importers: specifier: ^0.5.0 version: 0.5.0 bson: - specifier: ^6.10.3 - version: 6.10.3 + specifier: ^6.10.4 + version: 6.10.4 commander: specifier: ^12.0.0 version: 12.1.0 @@ -1043,8 +1043,8 @@ packages: '@manypkg/get-packages@1.1.3': resolution: {integrity: sha512-fo+QhuU3qE/2TQMQmbVMqaQ6EWbMhi4ABWP+O4AM1NqPBuy0OrApV5LO6BrrgnhtAHS2NH6RrVk9OL181tTi8A==} - '@mongodb-js/saslprep@1.1.9': - resolution: {integrity: sha512-tVkljjeEaAhCqTzajSdgbQ6gE6f3oneVwa3iXR6csiEwXXOFsiC6Uh9iAjAhXPtqa/XMDHWjjeNH/77m/Yq2dw==} + '@mongodb-js/saslprep@1.3.1': + resolution: {integrity: sha512-6nZrq5kfAz0POWyhljnbWQQJQ5uT8oE2ddX303q1uY0tWsivWKgBDXBBvuFPwOqRRalXJuVO9EjOdVtuhLX0zg==} '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} @@ -1876,8 +1876,8 @@ packages: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} - bson@6.10.3: - resolution: {integrity: sha512-MTxGsqgYTwfshYWTRdmZRC+M7FnG1b4y7RO7p2k3X24Wq0yv1m77Wsj0BzlPzd/IowgESfsruQCUToa7vbOpPQ==} + bson@6.10.4: + resolution: {integrity: sha512-WIsKqkSC0ABoBJuT1LEX+2HEvNmNKKgnTAyd0fL8qzK4SH2i9NXg+t08YtdZp/V9IZ33cxe3iV4yM0qg8lMQng==} engines: {node: '>=16.20.1'} buffer-from@1.1.2: @@ -2897,8 +2897,8 @@ packages: mongodb-connection-string-url@3.0.2: resolution: {integrity: sha512-rMO7CGo/9BFwyZABcKAWL8UJwH/Kc2x0g72uhDWzG48URRax5TCIcJ7Rc3RZqffZzO/Gwff/jyKwCU9TN8gehA==} - mongodb@6.14.2: - resolution: {integrity: sha512-kMEHNo0F3P6QKDq17zcDuPeaywK/YaJVCEQRzPF3TOM/Bl9MFg64YE5Tu7ifj37qZJMhwU1tl2Ioivws5gRG5Q==} + mongodb@6.20.0: + resolution: {integrity: sha512-Tl6MEIU3K4Rq3TSHd+sZQqRBoGlFsOgNrH5ltAcFBV62Re3Fd+FcaVf8uSEQFOJ51SDowDVttBTONMfoYWrWlQ==} engines: {node: '>=16.20.1'} peerDependencies: '@aws-sdk/credential-providers': ^3.188.0 @@ -2906,7 +2906,7 @@ packages: gcp-metadata: ^5.2.0 kerberos: ^2.0.1 mongodb-client-encryption: '>=6.0.0 <7' - snappy: ^7.2.2 + snappy: ^7.3.2 socks: ^2.7.1 peerDependenciesMeta: '@aws-sdk/credential-providers': @@ -4501,7 +4501,7 @@ snapshots: globby: 11.1.0 read-yaml-file: 1.1.0 - '@mongodb-js/saslprep@1.1.9': + '@mongodb-js/saslprep@1.3.1': dependencies: sparse-bitfield: 3.0.3 @@ -5396,7 +5396,7 @@ snapshots: dependencies: fill-range: 7.1.1 - bson@6.10.3: {} + bson@6.10.4: {} buffer-from@1.1.2: {} @@ -6452,10 +6452,10 @@ snapshots: '@types/whatwg-url': 11.0.5 whatwg-url: 13.0.0 - mongodb@6.14.2(socks@2.8.3): + mongodb@6.20.0(socks@2.8.3): dependencies: - '@mongodb-js/saslprep': 1.1.9 - bson: 6.10.3 + '@mongodb-js/saslprep': 1.3.1 + bson: 6.10.4 mongodb-connection-string-url: 3.0.2 optionalDependencies: socks: 2.8.3