From 595a2b23a2b6f4bc6a693699a7315ee12e4b05df Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 19 Jun 2025 14:36:30 +0200 Subject: [PATCH 1/3] Convert checksums to long when aggregating. --- .../src/storage/implementation/MongoSyncBucketStorage.ts | 5 ++++- .../src/storage/implementation/PersistedBatch.ts | 4 ++-- .../src/storage/implementation/models.ts | 2 +- .../module-mongodb-storage/test/src/storage_sync.test.ts | 7 +++++++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 404994980..676ef7130 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -473,7 +473,10 @@ export class MongoSyncBucketStorage { $group: { _id: '$_id.b', - checksum_total: { $sum: '$checksum' }, + // Historically, checksum may be stored as 'int' or 'double'. + // More recently, this should be a 'long'. + // $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations. + checksum_total: { $sum: { $toLong: '$checksum' } }, count: { $sum: 1 }, has_clear_op: { $max: { diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts index 8770fae83..b319053d5 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts @@ -97,7 +97,7 @@ export class PersistedBatch { remaining_buckets.set(key, b); } - const dchecksum = utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey)); + const dchecksum = BigInt(utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey))); for (const k of options.evaluated) { const key = currentBucketKey(k); @@ -133,7 +133,7 @@ export class PersistedBatch { source_key: options.sourceKey, table: k.table, row_id: k.id, - checksum: checksum, + checksum: BigInt(checksum), data: recordData } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index e3e03ca55..181d50fff 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -56,7 +56,7 @@ export interface BucketDataDocument { source_key?: ReplicaId; table?: string; row_id?: string; - checksum: number; + checksum: bigint; data: string | null; target_op?: bigint | null; } diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index d75c35e8f..f49d595bf 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -117,5 +117,12 @@ describe('sync - mongodb', () => { has_more: false, next_after: '4' }); + + // Test that the checksum type is correct. + // Specifically, test that it never persisted as double. + const checksumTypes = await factory.db.bucket_data + .aggregate([{ $group: { _id: { $type: '$checksum' }, count: { $sum: 1 } } }]) + .toArray(); + expect(checksumTypes).toEqual([{ _id: 'long', count: 4 }]); }); }); From a44bc461e9ad504b2a5e0e4bd876304d6dfa1242 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 19 Jun 2025 14:48:00 +0200 Subject: [PATCH 2/3] Fix the compactor. --- .../src/storage/implementation/MongoCompactor.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts index 612ad1aba..6f4d852f9 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts @@ -317,7 +317,7 @@ export class MongoCompactor { let numberOfOpsToClear = 0; for await (let op of query.stream()) { if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') { - checksum = utils.addChecksums(checksum, op.checksum); + checksum = utils.addChecksums(checksum, Number(op.checksum)); lastOpId = op._id; numberOfOpsToClear += 1; if (op.op != 'CLEAR') { @@ -358,7 +358,7 @@ export class MongoCompactor { { _id: lastOpId!, op: 'CLEAR', - checksum: checksum, + checksum: BigInt(checksum), data: null, target_op: targetOp }, From e8cfbedd9facdd52f65aa79cf3750d0938e45a89 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 19 Jun 2025 14:50:09 +0200 Subject: [PATCH 3/3] Changeset. --- .changeset/popular-keys-switch.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/popular-keys-switch.md diff --git a/.changeset/popular-keys-switch.md b/.changeset/popular-keys-switch.md new file mode 100644 index 000000000..91da6e918 --- /dev/null +++ b/.changeset/popular-keys-switch.md @@ -0,0 +1,7 @@ +--- +'@powersync/service-module-mongodb-storage': patch +'@powersync/service-core': patch +'@powersync/service-image': patch +--- + +[MongoDB Storage] Fix checksum calculations in buckets with more than 4 million operations