diff --git a/.changeset/fast-flowers-scream.md b/.changeset/fast-flowers-scream.md new file mode 100644 index 000000000..75182d34e --- /dev/null +++ b/.changeset/fast-flowers-scream.md @@ -0,0 +1,7 @@ +--- +'@powersync/service-module-mongodb-storage': patch +'@powersync/lib-service-mongodb': patch +'@powersync/service-image': patch +--- + +Skip large rows, rather than causing hard replication errors diff --git a/libs/lib-mongodb/package.json b/libs/lib-mongodb/package.json index 419b5fb7e..6a7bdb875 100644 --- a/libs/lib-mongodb/package.json +++ b/libs/lib-mongodb/package.json @@ -30,7 +30,7 @@ "dependencies": { "@powersync/lib-services-framework": "workspace:*", "bson": "^6.10.3", - "mongodb": "^6.13.0", + "mongodb": "^6.14.1", "ts-codec": "^1.3.0", "uri-js": "^4.4.1" }, diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 04ed68ca4..41df4e9b9 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -24,7 +24,7 @@ import { idPrefixFilter } from './util.js'; /** * 15MB */ -const MAX_ROW_SIZE = 15 * 1024 * 1024; +export const MAX_ROW_SIZE = 15 * 1024 * 1024; // Currently, we can only have a single flush() at a time, since it locks the op_id sequence. // While the MongoDB transaction retry mechanism handles this okay, using an in-process Mutex diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts index e1f128eec..5e3ddfa71 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts @@ -5,7 +5,7 @@ import * as bson from 'bson'; import { logger } from '@powersync/lib-services-framework'; import { InternalOpId, storage, utils } from '@powersync/service-core'; -import { currentBucketKey } from './MongoBucketBatch.js'; +import { currentBucketKey, MAX_ROW_SIZE } from './MongoBucketBatch.js'; import { MongoIdSequence } from './MongoIdSequence.js'; import { PowerSyncMongo } from './db.js'; import { @@ -83,11 +83,20 @@ export class PersistedBatch { for (const k of options.evaluated) { const key = currentBucketKey(k); - remaining_buckets.delete(key); // INSERT const recordData = JSONBig.stringify(k.data); const checksum = utils.hashData(k.table, k.id, recordData); + if (recordData.length > MAX_ROW_SIZE) { + // In many cases, the raw data size would have been too large already. But there are cases where + // the BSON size is small enough, but the JSON size is too large. + // In these cases, we can't store the data, so we skip it, or generate a REMOVE operation if the row + // was synced previously. + logger.error(`powersync_${this.group_id} Row ${key} too large: ${recordData.length} bytes. Removing.`); + continue; + } + + remaining_buckets.delete(key); this.currentSize += recordData.length + 200; const op_id = options.op_seq.next(); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a38171f1f..a9a6db483 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -75,8 +75,8 @@ importers: specifier: ^6.10.3 version: 6.10.3 mongodb: - specifier: ^6.13.0 - version: 6.13.0(socks@2.8.3) + specifier: ^6.14.1 + version: 6.14.2(socks@2.8.3) ts-codec: specifier: ^1.3.0 version: 1.3.0 @@ -703,8 +703,8 @@ importers: specifier: ^10.0.1 version: 10.4.3 mongodb: - specifier: ^6.13.0 - version: 6.13.0(socks@2.8.3) + specifier: ^6.14.1 + version: 6.14.2(socks@2.8.3) node-fetch: specifier: ^3.3.2 version: 3.3.2 @@ -2861,8 +2861,8 @@ packages: mongodb-connection-string-url@3.0.1: resolution: {integrity: sha512-XqMGwRX0Lgn05TDB4PyG2h2kKO/FfWJyCzYQbIhXUxz7ETt0I/FqHjUeqj37irJ+Dl1ZtU82uYyj14u2XsZKfg==} - mongodb@6.13.0: - resolution: {integrity: sha512-KeESYR5TEaFxOuwRqkOm3XOsMqCSkdeDMjaW5u2nuKfX7rqaofp7JQGoi7sVqQcNJTKuveNbzZtWMstb8ABP6Q==} + mongodb@6.14.2: + resolution: {integrity: sha512-kMEHNo0F3P6QKDq17zcDuPeaywK/YaJVCEQRzPF3TOM/Bl9MFg64YE5Tu7ifj37qZJMhwU1tl2Ioivws5gRG5Q==} engines: {node: '>=16.20.1'} peerDependencies: '@aws-sdk/credential-providers': ^3.188.0 @@ -6429,7 +6429,7 @@ snapshots: '@types/whatwg-url': 11.0.5 whatwg-url: 13.0.0 - mongodb@6.13.0(socks@2.8.3): + mongodb@6.14.2(socks@2.8.3): dependencies: '@mongodb-js/saslprep': 1.1.9 bson: 6.10.3 diff --git a/service/package.json b/service/package.json index 0c292da19..4a3c4f34c 100644 --- a/service/package.json +++ b/service/package.json @@ -36,7 +36,7 @@ "ix": "^5.0.0", "jose": "^4.15.1", "lru-cache": "^10.0.1", - "mongodb": "^6.13.0", + "mongodb": "^6.14.1", "node-fetch": "^3.3.2", "pgwire": "github:kagis/pgwire#f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87", "ts-codec": "^1.3.0",