From cadb128f7edd89b48e089ba5cf555ae2d4c91632 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Thu, 9 Jan 2025 09:01:57 +0200 Subject: [PATCH 1/3] MySQL: Removed duplicate snapshot query stream invocation Added abortcontroller check during MySQL initial snapshot --- modules/module-mysql/src/replication/BinLogStream.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index 483f292f9..59fd8309c 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -278,9 +278,7 @@ AND table_type = 'BASE TABLE';`, } ); logger.info(`Initial replication done`); - await promiseConnection.query('COMMIT'); } catch (e) { - await promiseConnection.query('ROLLBACK'); throw e; } finally { connection.release(); @@ -305,12 +303,16 @@ AND table_type = 'BASE TABLE';`, columns = toColumnDescriptors(fields); }); - for await (let row of query.stream()) { + for await (let row of stream) { + if (this.stopped) { + throw new Error('Abort signal received - initial replication interrupted.'); + } + if (columns == null) { throw new Error(`No 'fields' event emitted`); } - const record = common.toSQLiteRow(row, columns!); + const record = common.toSQLiteRow(row, columns!); await batch.save({ tag: storage.SaveOperationTag.INSERT, sourceTable: table, From c5705f467dcd4770c94e2e475b443fda2b1a80e5 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Thu, 9 Jan 2025 09:17:46 +0200 Subject: [PATCH 2/3] Added changeset --- .changeset/perfect-plums-cry.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/perfect-plums-cry.md diff --git a/.changeset/perfect-plums-cry.md b/.changeset/perfect-plums-cry.md new file mode 100644 index 000000000..b35b3c950 --- /dev/null +++ b/.changeset/perfect-plums-cry.md @@ -0,0 +1,5 @@ +--- +'@powersync/service-module-mysql': patch +--- + +Resolved excessive memory consumption during MySQL initial replication. From 1bf04de9a23d3a205a280320e244a3626a0b9182 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Thu, 9 Jan 2025 09:29:01 +0200 Subject: [PATCH 3/3] Revert transaction commit and rollback --- modules/module-mysql/src/replication/BinLogStream.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index 59fd8309c..bf1b24a75 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -278,7 +278,9 @@ AND table_type = 'BASE TABLE';`, } ); logger.info(`Initial replication done`); + await promiseConnection.query('COMMIT'); } catch (e) { + await promiseConnection.query('ROLLBACK'); throw e; } finally { connection.release();