diff --git a/.changeset/funny-pianos-allow.md b/.changeset/funny-pianos-allow.md new file mode 100644 index 000000000..3df90e3bc --- /dev/null +++ b/.changeset/funny-pianos-allow.md @@ -0,0 +1,5 @@ +--- +'@powersync/service-core': patch +--- + +Fix "operation exceeded time limit" error diff --git a/packages/service-core/src/db/mongo.ts b/packages/service-core/src/db/mongo.ts index be915bdc3..ca2253028 100644 --- a/packages/service-core/src/db/mongo.ts +++ b/packages/service-core/src/db/mongo.ts @@ -22,6 +22,13 @@ export const MONGO_SOCKET_TIMEOUT_MS = 60_000; */ export const MONGO_OPERATION_TIMEOUT_MS = 30_000; +/** + * Same as above, but specifically for clear operations. + * + * These are retried when reaching the timeout. + */ +export const MONGO_CLEAR_OPERATION_TIMEOUT_MS = 5_000; + export function createMongoClient(config: configFile.PowerSyncConfig['storage']) { return new mongo.MongoClient(config.uri, { auth: { diff --git a/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts b/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts index d36c7dc4a..855fce8ee 100644 --- a/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts +++ b/packages/service-core/src/storage/mongo/MongoSyncBucketStorage.ts @@ -26,6 +26,8 @@ import { BucketDataDocument, BucketDataKey, SourceKey, SyncRuleState } from './m import { MongoBucketBatch } from './MongoBucketBatch.js'; import { MongoCompactor } from './MongoCompactor.js'; import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, mapOpEntry, readSingleBatch, serializeLookup } from './util.js'; +import { logger } from '@powersync/lib-services-framework'; +import * as timers from 'timers/promises'; export class MongoSyncBucketStorage implements SyncRulesBucketStorage { private readonly db: PowerSyncMongo; @@ -438,10 +440,28 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage { } async clear(): Promise { + while (true) { + try { + await this.clearIteration(); + return; + } catch (e: unknown) { + if (e instanceof mongo.MongoServerError && e.codeName == 'MaxTimeMSExpired') { + logger.info( + `Clearing took longer than ${db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, waiting and triggering another iteration.` + ); + await timers.setTimeout(db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5); + continue; + } else { + throw e; + } + } + } + } + + private async clearIteration(): Promise { // Individual operations here may time out with the maxTimeMS option. // It is expected to still make progress, and continue on the next try. - // TODO: Transactional? await this.db.sync_rules.updateOne( { _id: this.group_id @@ -455,33 +475,33 @@ export class MongoSyncBucketStorage implements SyncRulesBucketStorage { no_checkpoint_before: null } }, - { maxTimeMS: db.mongo.MONGO_OPERATION_TIMEOUT_MS } + { maxTimeMS: db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS } ); await this.db.bucket_data.deleteMany( { _id: idPrefixFilter({ g: this.group_id }, ['b', 'o']) }, - { maxTimeMS: db.mongo.MONGO_OPERATION_TIMEOUT_MS } + { maxTimeMS: db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS } ); await this.db.bucket_parameters.deleteMany( { key: idPrefixFilter({ g: this.group_id }, ['t', 'k']) }, - { maxTimeMS: db.mongo.MONGO_OPERATION_TIMEOUT_MS } + { maxTimeMS: db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS } ); await this.db.current_data.deleteMany( { _id: idPrefixFilter({ g: this.group_id }, ['t', 'k']) }, - { maxTimeMS: db.mongo.MONGO_OPERATION_TIMEOUT_MS } + { maxTimeMS: db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS } ); await this.db.source_tables.deleteMany( { group_id: this.group_id }, - { maxTimeMS: db.mongo.MONGO_OPERATION_TIMEOUT_MS } + { maxTimeMS: db.mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS } ); }