diff --git a/.changeset/clever-kids-pretend.md b/.changeset/clever-kids-pretend.md new file mode 100644 index 000000000..67558dcf3 --- /dev/null +++ b/.changeset/clever-kids-pretend.md @@ -0,0 +1,11 @@ +--- +'@powersync/service-module-mongodb-storage': patch +'@powersync/service-rsocket-router': patch +'@powersync/service-module-mongodb': patch +'@powersync/service-core': patch +'@powersync/lib-services-framework': patch +'@powersync/lib-service-mongodb': patch +'@powersync/service-image': patch +--- + +Upgrade mongodb and bson packages, removing the need for some workarounds. diff --git a/libs/lib-mongodb/package.json b/libs/lib-mongodb/package.json index 84712f13a..f1a486d3e 100644 --- a/libs/lib-mongodb/package.json +++ b/libs/lib-mongodb/package.json @@ -29,8 +29,8 @@ }, "dependencies": { "@powersync/lib-services-framework": "workspace:*", - "bson": "^6.8.0", - "mongodb": "^6.11.0", + "bson": "^6.10.3", + "mongodb": "^6.13.0", "ts-codec": "^1.3.0", "uri-js": "^4.4.1" }, diff --git a/libs/lib-services/package.json b/libs/lib-services/package.json index fec1401ce..bd6800677 100644 --- a/libs/lib-services/package.json +++ b/libs/lib-services/package.json @@ -23,7 +23,7 @@ "@powersync/service-errors": "workspace:*", "ajv": "^8.12.0", "better-ajv-errors": "^1.2.0", - "bson": "^6.8.0", + "bson": "^6.10.3", "dotenv": "^16.4.5", "ipaddr.js": "^2.1.0", "lodash": "^4.17.21", diff --git a/modules/module-mongodb-storage/package.json b/modules/module-mongodb-storage/package.json index c982462e9..73339b9c5 100644 --- a/modules/module-mongodb-storage/package.json +++ b/modules/module-mongodb-storage/package.json @@ -34,7 +34,7 @@ "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", "@powersync/lib-service-mongodb": "workspace:*", - "bson": "^6.8.0", + "bson": "^6.10.3", "ts-codec": "^1.3.0", "ix": "^5.0.0", "lru-cache": "^10.2.2", diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts index b9238c1e2..06a14df52 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts @@ -5,7 +5,6 @@ import { storage, utils } from '@powersync/service-core'; import { PowerSyncMongo } from './db.js'; import { BucketDataDocument, BucketDataKey } from './models.js'; import { cacheKey } from './OperationBatch.js'; -import { safeBulkWrite } from './util.js'; interface CurrentBucketState { /** Bucket name */ @@ -265,7 +264,7 @@ export class MongoCompactor { private async flush() { if (this.updates.length > 0) { logger.info(`Compacting ${this.updates.length} ops`); - await safeBulkWrite(this.db.bucket_data, this.updates, { + await this.db.bucket_data.bulkWrite(this.updates, { // Order is not important. // Since checksums are not affected, these operations can happen in any order, // and it's fine if the operations are partially applied. diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 16126b346..ce8ab8677 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -313,13 +313,7 @@ export class MongoSyncBucketStorage // 1. We can calculate the document size accurately without serializing again. // 2. We can delay parsing the results until it's needed. // We manually use bson.deserialize below - raw: true, - - // Since we're using raw: true and parsing ourselves later, we don't need bigint - // support here. - // Disabling due to https://jira.mongodb.org/browse/NODE-6165, and the fact that this - // is one of our most common queries. - useBigInt64: false + raw: true } ) as unknown as mongo.FindCursor; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoWriteCheckpointAPI.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoWriteCheckpointAPI.ts index b2d6fff6b..fe6848371 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoWriteCheckpointAPI.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoWriteCheckpointAPI.ts @@ -1,7 +1,6 @@ import * as framework from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; import { PowerSyncMongo } from './db.js'; -import { safeBulkWrite } from './util.js'; export type MongoCheckpointAPIOptions = { db: PowerSyncMongo; @@ -127,8 +126,7 @@ export async function batchCreateCustomWriteCheckpoints( return; } - await safeBulkWrite( - db.custom_write_checkpoints, + await db.custom_write_checkpoints.bulkWrite( checkpoints.map((checkpointOptions) => ({ updateOne: { filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id }, diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts index 9ec06afd3..4eaae0d60 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts @@ -15,7 +15,7 @@ import { CurrentDataDocument, SourceKey } from './models.js'; -import { replicaIdToSubkey, safeBulkWrite } from './util.js'; +import { replicaIdToSubkey } from './util.js'; /** * Maximum size of operations we write in a single transaction. @@ -246,22 +246,21 @@ export class PersistedBatch { async flush(db: PowerSyncMongo, session: mongo.ClientSession) { if (this.bucketData.length > 0) { - // calculate total size - await safeBulkWrite(db.bucket_data, this.bucketData, { + await db.bucket_data.bulkWrite(this.bucketData, { session, // inserts only - order doesn't matter ordered: false }); } if (this.bucketParameters.length > 0) { - await safeBulkWrite(db.bucket_parameters, this.bucketParameters, { + await db.bucket_parameters.bulkWrite(this.bucketParameters, { session, // inserts only - order doesn't matter ordered: false }); } if (this.currentData.length > 0) { - await safeBulkWrite(db.current_data, this.currentData, { + await db.current_data.bulkWrite(this.currentData, { session, // may update and delete data within the same batch - order matters ordered: true diff --git a/modules/module-mongodb-storage/src/storage/implementation/util.ts b/modules/module-mongodb-storage/src/storage/implementation/util.ts index 0d183a8a6..7722ad253 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/util.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/util.ts @@ -124,48 +124,3 @@ export const connectMongoForTests = (url: string, isCI: boolean) => { }); return new PowerSyncMongo(client); }; - -/** - * MongoDB bulkWrite internally splits the operations into batches - * so that no batch exceeds 16MB. However, there are cases where - * the batch size is very close to 16MB, where additional metadata - * on the server pushes it over the limit, resulting in this error - * from the server: - * - * > MongoBulkWriteError: BSONObj size: 16814023 (0x1008FC7) is invalid. Size must be between 0 and 16793600(16MB) First element: insert: "bucket_data" - * - * We work around the issue by doing our own batching, limiting the - * batch size to 15MB. This does add additional overhead with - * BSON.calculateObjectSize. - */ -export async function safeBulkWrite( - collection: mongo.Collection, - operations: mongo.AnyBulkWriteOperation[], - options: mongo.BulkWriteOptions -) { - // Must be below 16MB. - // We could probably go a little closer, but 15MB is a safe threshold. - const BULK_WRITE_LIMIT = 15 * 1024 * 1024; - - let batch: mongo.AnyBulkWriteOperation[] = []; - let currentSize = 0; - // Estimated overhead per operation, should be smaller in reality. - const keySize = 8; - for (let op of operations) { - const bsonSize = - mongo.BSON.calculateObjectSize(op, { - checkKeys: false, - ignoreUndefined: true - } as any) + keySize; - if (batch.length > 0 && currentSize + bsonSize > BULK_WRITE_LIMIT) { - await collection.bulkWrite(batch, options); - currentSize = 0; - batch = []; - } - batch.push(op); - currentSize += bsonSize; - } - if (batch.length > 0) { - await collection.bulkWrite(batch, options); - } -} diff --git a/modules/module-mongodb/package.json b/modules/module-mongodb/package.json index d06fa61c7..134788429 100644 --- a/modules/module-mongodb/package.json +++ b/modules/module-mongodb/package.json @@ -34,7 +34,7 @@ "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", "@powersync/lib-service-mongodb": "workspace:*", - "bson": "^6.8.0", + "bson": "^6.10.3", "ts-codec": "^1.3.0", "uuid": "^9.0.1" }, diff --git a/package.json b/package.json index b2df21c8a..c543248d3 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "@changesets/cli": "^2.27.8", "@types/node": "^22.13.1", "async": "^3.2.4", - "bson": "^6.8.0", + "bson": "^6.10.3", "concurrently": "^8.2.2", "inquirer": "^9.2.7", "npm-check-updates": "^17.1.2", diff --git a/packages/rsocket-router/package.json b/packages/rsocket-router/package.json index dc63069fe..a5fa20130 100644 --- a/packages/rsocket-router/package.json +++ b/packages/rsocket-router/package.json @@ -27,7 +27,7 @@ "devDependencies": { "@types/uuid": "^9.0.4", "@types/ws": "~8.2.0", - "bson": "^6.8.0", + "bson": "^6.10.3", "rsocket-websocket-client": "1.0.0-alpha.3" } } diff --git a/packages/service-core/package.json b/packages/service-core/package.json index 5f590116f..aa1f5769b 100644 --- a/packages/service-core/package.json +++ b/packages/service-core/package.json @@ -29,7 +29,7 @@ "@powersync/service-types": "workspace:*", "async": "^3.2.4", "async-mutex": "^0.5.0", - "bson": "^6.8.0", + "bson": "^6.10.3", "commander": "^12.0.0", "cors": "^2.8.5", "ipaddr.js": "^2.1.0", diff --git a/packages/service-core/src/storage/bson.ts b/packages/service-core/src/storage/bson.ts index d0df1fedf..9d69af220 100644 --- a/packages/service-core/src/storage/bson.ts +++ b/packages/service-core/src/storage/bson.ts @@ -14,11 +14,10 @@ export const BSON_DESERIALIZE_INTERNAL_OPTIONS: bson.DeserializeOptions = { }; /** - * Use for data from external sources. + * Use for data from external sources, which could contain arbitrary fields. */ export const BSON_DESERIALIZE_DATA_OPTIONS: bson.DeserializeOptions = { - // Temporarily disable due to https://jira.mongodb.org/browse/NODE-6764 - useBigInt64: false + useBigInt64: true }; /** @@ -67,16 +66,11 @@ export const deserializeReplicaId = (id: Buffer): ReplicaId => { return deserialized.id; }; +/** + * Deserialize BSON - can be used for BSON containing arbitrary user data. + */ export const deserializeBson = (buffer: Uint8Array): bson.Document => { - const doc = bson.deserialize(buffer, BSON_DESERIALIZE_DATA_OPTIONS); - // Temporary workaround due to https://jira.mongodb.org/browse/NODE-6764 - for (let key in doc) { - const value = doc[key]; - if (value instanceof bson.Long) { - doc[key] = value.toBigInt(); - } - } - return doc; + return bson.deserialize(buffer, BSON_DESERIALIZE_DATA_OPTIONS); }; export const serializeBson = (document: any): NodeBuffer => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 42d713dfd..a38171f1f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -18,8 +18,8 @@ importers: specifier: ^3.2.4 version: 3.2.5 bson: - specifier: ^6.8.0 - version: 6.10.1 + specifier: ^6.10.3 + version: 6.10.3 concurrently: specifier: ^8.2.2 version: 8.2.2 @@ -72,11 +72,11 @@ importers: specifier: workspace:* version: link:../lib-services bson: - specifier: ^6.8.0 - version: 6.10.1 + specifier: ^6.10.3 + version: 6.10.3 mongodb: - specifier: ^6.11.0 - version: 6.11.0(socks@2.8.3) + specifier: ^6.13.0 + version: 6.13.0(socks@2.8.3) ts-codec: specifier: ^1.3.0 version: 1.3.0 @@ -124,8 +124,8 @@ importers: specifier: ^1.2.0 version: 1.2.0(ajv@8.16.0) bson: - specifier: ^6.8.0 - version: 6.10.1 + specifier: ^6.10.3 + version: 6.10.3 dotenv: specifier: ^16.4.5 version: 16.4.5 @@ -179,8 +179,8 @@ importers: specifier: workspace:* version: link:../../packages/types bson: - specifier: ^6.8.0 - version: 6.10.1 + specifier: ^6.10.3 + version: 6.10.3 ts-codec: specifier: ^1.3.0 version: 1.3.0 @@ -222,8 +222,8 @@ importers: specifier: workspace:* version: link:../../packages/types bson: - specifier: ^6.8.0 - version: 6.10.1 + specifier: ^6.10.3 + version: 6.10.3 ix: specifier: ^5.0.0 version: 5.0.0 @@ -452,8 +452,8 @@ importers: specifier: ~8.2.0 version: 8.2.3 bson: - specifier: ^6.8.0 - version: 6.10.1 + specifier: ^6.10.3 + version: 6.10.3 rsocket-websocket-client: specifier: 1.0.0-alpha.3 version: 1.0.0-alpha.3 @@ -500,8 +500,8 @@ importers: specifier: ^0.5.0 version: 0.5.0 bson: - specifier: ^6.8.0 - version: 6.10.1 + specifier: ^6.10.3 + version: 6.10.3 commander: specifier: ^12.0.0 version: 12.1.0 @@ -679,8 +679,8 @@ importers: specifier: ^0.5.0 version: 0.5.0 bson: - specifier: ^6.8.0 - version: 6.10.1 + specifier: ^6.10.3 + version: 6.10.3 commander: specifier: ^12.0.0 version: 12.1.0 @@ -703,8 +703,8 @@ importers: specifier: ^10.0.1 version: 10.4.3 mongodb: - specifier: ^6.11.0 - version: 6.11.0(socks@2.8.3) + specifier: ^6.13.0 + version: 6.13.0(socks@2.8.3) node-fetch: specifier: ^3.3.2 version: 3.3.2 @@ -1839,8 +1839,8 @@ packages: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} - bson@6.10.1: - resolution: {integrity: sha512-P92xmHDQjSKPLHqFxefqMxASNq/aWJMEZugpCjf+AF/pgcUpMMQCg7t7+ewko0/u8AapvF3luf/FoehddEK+sA==} + bson@6.10.3: + resolution: {integrity: sha512-MTxGsqgYTwfshYWTRdmZRC+M7FnG1b4y7RO7p2k3X24Wq0yv1m77Wsj0BzlPzd/IowgESfsruQCUToa7vbOpPQ==} engines: {node: '>=16.20.1'} buffer-from@1.1.2: @@ -2861,12 +2861,12 @@ packages: mongodb-connection-string-url@3.0.1: resolution: {integrity: sha512-XqMGwRX0Lgn05TDB4PyG2h2kKO/FfWJyCzYQbIhXUxz7ETt0I/FqHjUeqj37irJ+Dl1ZtU82uYyj14u2XsZKfg==} - mongodb@6.11.0: - resolution: {integrity: sha512-yVbPw0qT268YKhG241vAMLaDQAPbRyTgo++odSgGc9kXnzOujQI60Iyj23B9sQQFPSvmNPvMZ3dsFz0aN55KgA==} + mongodb@6.13.0: + resolution: {integrity: sha512-KeESYR5TEaFxOuwRqkOm3XOsMqCSkdeDMjaW5u2nuKfX7rqaofp7JQGoi7sVqQcNJTKuveNbzZtWMstb8ABP6Q==} engines: {node: '>=16.20.1'} peerDependencies: '@aws-sdk/credential-providers': ^3.188.0 - '@mongodb-js/zstd': ^1.1.0 + '@mongodb-js/zstd': ^1.1.0 || ^2.0.0 gcp-metadata: ^5.2.0 kerberos: ^2.0.1 mongodb-client-encryption: '>=6.0.0 <7' @@ -5353,7 +5353,7 @@ snapshots: dependencies: fill-range: 7.1.1 - bson@6.10.1: {} + bson@6.10.3: {} buffer-from@1.1.2: {} @@ -6429,10 +6429,10 @@ snapshots: '@types/whatwg-url': 11.0.5 whatwg-url: 13.0.0 - mongodb@6.11.0(socks@2.8.3): + mongodb@6.13.0(socks@2.8.3): dependencies: '@mongodb-js/saslprep': 1.1.9 - bson: 6.10.1 + bson: 6.10.3 mongodb-connection-string-url: 3.0.1 optionalDependencies: socks: 2.8.3 diff --git a/service/package.json b/service/package.json index 02fc1ca11..a2e70a925 100644 --- a/service/package.json +++ b/service/package.json @@ -28,7 +28,7 @@ "@powersync/service-types": "workspace:*", "@sentry/node": "^8.9.2", "async-mutex": "^0.5.0", - "bson": "^6.8.0", + "bson": "^6.10.3", "commander": "^12.0.0", "cors": "^2.8.5", "fastify": "4.23.2", @@ -36,7 +36,7 @@ "ix": "^5.0.0", "jose": "^4.15.1", "lru-cache": "^10.0.1", - "mongodb": "^6.11.0", + "mongodb": "^6.13.0", "node-fetch": "^3.3.2", "pgwire": "github:kagis/pgwire#f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87", "ts-codec": "^1.3.0",