Skip to content

Commit 6530cba

Browse files
alexolealexole
authored
add MySQL 8.4 support (#52)
1 parent 425b1b3 commit 6530cba

File tree

4 files changed

+86
-32
lines changed

4 files changed

+86
-32
lines changed

‎Dockerfile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,14 @@ FROM python:3.12-bookworm
22

33
RUN gpg --keyserver keyserver.ubuntu.com --recv-keys A8D3785C \
44
&& gpg --export A8D3785C > /etc/apt/keyrings/mysql.gpg \
5-
&& echo "deb [signed-by=/etc/apt/keyrings/mysql.gpg] http://repo.mysql.com/apt/debian/ bookworm mysql-8.0" | tee /etc/apt/sources.list.d/mysql.list \
5+
&& echo "deb [signed-by=/etc/apt/keyrings/mysql.gpg] http://repo.mysql.com/apt/debian/ bookworm mysql-8.4-lts" | tee /etc/apt/sources.list.d/mysql.list \
66
&& apt-get update -y \
77
&& apt-get install -y mysql-client
88

99
# Install mydumper/myloader
1010
RUN apt-get install -y wget zstd
11-
RUN wget https://github.com/mydumper/mydumper/releases/download/v0.21.2-2/mydumper_0.21.2-2.bookworm_amd64.deb
12-
RUN apt-get install -y ./mydumper_0.21.2-2.bookworm_amd64.deb
11+
RUN wget https://github.com/mydumper/mydumper/releases/download/v0.21.3-1/mydumper_0.21.3-1.bookworm_amd64.deb
12+
RUN apt-get install -y ./mydumper_0.21.3-1.bookworm_amd64.deb
1313

1414
COPY . /app
1515
WORKDIR /app

‎aiven_mysql_migrate/migration.py‎

Lines changed: 34 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -99,8 +99,8 @@ def _check_versions_replication_support(self):
9999
LOGGER.info("Checking MySQL versions for replication support")
100100

101101
if (
102-
LooseVersion("5.7.0") <= LooseVersion(self.source.version) < LooseVersion("8.1")
103-
and LooseVersion("8.0.0") <= LooseVersion(self.target.version) < LooseVersion("8.1")
102+
LooseVersion("5.7.0") <= LooseVersion(self.source.version) < LooseVersion("8.5")
103+
and LooseVersion("8.0.0") <= LooseVersion(self.target.version) < LooseVersion("8.5")
104104
):
105105
LOGGER.info("\tSource - %s, target - %s -- OK", self.source.version, self.target.version)
106106
else:
@@ -142,11 +142,17 @@ def _check_gtid_mode_enabled(self):
142142
gtid_mode = select_global_var(cur, "gtid_mode")
143143
if gtid_mode.upper() != "ON":
144144
raise GTIDModeDisabledException(f"GTID mode should be enabled on the {conn_info.name}")
145-
cur.execute("SHOW MASTER STATUS")
145+
146+
if conn_info.version >= LooseVersion("8.2.0"):
147+
show_status_query = "SHOW BINARY LOG STATUS"
148+
else:
149+
show_status_query = "SHOW MASTER STATUS"
150+
cur.execute(show_status_query)
151+
146152
master_status = cur.fetchone()
147153
if master_status is None:
148154
raise GTIDModeDisabledException(
149-
f"GTID mode should be enabled on the {conn_info.name}: SHOW MASTER STATUS is empty"
155+
f"GTID mode should be enabled on the {conn_info.name}: {show_status_query} is empty"
150156
)
151157
executed_gtid_set = master_status.get("Executed_Gtid_Set", None)
152158
if not executed_gtid_set:
@@ -157,7 +163,9 @@ def _check_gtid_mode_enabled(self):
157163
def _check_user_can_replicate(self):
158164
LOGGER.info("Checking if user has replication grants on the source")
159165

160-
user_can_replicate = any(grant in self.source.global_grants for grant in ("REPLICATION SLAVE", "ALL PRIVILEGES"))
166+
user_can_replicate = any(
167+
grant in self.source.global_grants for grant in ("REPLICATION SLAVE", "ALL PRIVILEGES", "REPLICATION CLIENT")
168+
)
161169
if not user_can_replicate:
162170
raise MissingReplicationGrants("User does not have replication permissions")
163171

@@ -277,17 +285,17 @@ def run_checks(
277285

278286
return migration_method
279287

280-
def _stop_and_reset_slave(self):
288+
def _stop_and_reset_replica(self):
281289
LOGGER.info("Stopping replication on target database")
282290

283291
with self.target_master.cur() as cur:
284-
cur.execute("STOP SLAVE")
285-
cur.execute("RESET SLAVE ALL")
292+
cur.execute("STOP REPLICA")
293+
cur.execute("RESET REPLICA ALL")
286294

287295
def _stop_replication(self):
288296
LOGGER.info("Stopping replication")
289297

290-
self._stop_and_reset_slave()
298+
self._stop_and_reset_replica()
291299

292300
def _migrate_data(self, migration_method: MySQLMigrateMethod) -> Optional[str]:
293301
"""Migrate data using the configured dump tool, return GTID from the dump"""
@@ -323,9 +331,9 @@ def _start_replication(self):
323331

324332
with self.target_master.cur() as cur:
325333
query = (
326-
"CHANGE MASTER TO MASTER_HOST = %s, MASTER_PORT = %s, MASTER_USER = %s, MASTER_PASSWORD = %s, "
327-
f"MASTER_AUTO_POSITION = 1, MASTER_SSL = {1 if self.source.ssl else 0}, "
328-
"MASTER_SSL_VERIFY_SERVER_CERT = 0, MASTER_SSL_CA = '', MASTER_SSL_CAPATH = ''"
334+
"CHANGE REPLICATION SOURCE TO SOURCE_HOST = %s, SOURCE_PORT = %s, SOURCE_USER = %s, SOURCE_PASSWORD = %s, "
335+
f"SOURCE_AUTO_POSITION = 1, SOURCE_SSL = {1 if self.source.ssl else 0}, "
336+
"SOURCE_SSL_VERIFY_SERVER_CERT = 0, SOURCE_SSL_CA = '', SOURCE_SSL_CAPATH = ''"
329337
)
330338
if LooseVersion(self.target.version) >= LooseVersion("8.0.19"):
331339
query += ", REQUIRE_ROW_FORMAT = 1"
@@ -351,27 +359,27 @@ def _start_replication(self):
351359
f"CHANGE REPLICATION FILTER REPLICATE_WILD_IGNORE_TABLE = ({', '.join('%s' for _ in self.ignore_dbs)})",
352360
[f"{db}.%" for db in self.ignore_dbs]
353361
)
354-
cur.execute("START SLAVE")
362+
cur.execute("START REPLICA")
355363

356364
def _ensure_target_replica_running(self, check_interval: float = 2.0, retries: int = 30):
357365
LOGGER.info("Ensure replica is running")
358366

359367
with self.target.cur() as cur:
360368
for _ in range(retries):
361-
cur.execute("SHOW SLAVE STATUS")
369+
cur.execute("SHOW REPLICA STATUS")
362370
rows = cur.fetchall()
363371
if not rows:
364-
raise ReplicaSetupException("SHOW SLAVE STATUS didn't return any rows")
372+
raise ReplicaSetupException("SHOW REPLICA STATUS didn't return any rows")
365373

366374
try:
367-
slave_status = next(
375+
replica_status = next(
368376
row for row in rows
369-
if row["Master_Host"] == self.source.hostname and row["Master_Port"] == self.source.port
377+
if row["Source_Host"] == self.source.hostname and row["Source_Port"] == self.source.port
370378
)
371379
except StopIteration as e:
372-
raise ReplicaSetupException("Replication didn't start, Master info not available") from e
380+
raise ReplicaSetupException("Replication didn't start, Source info not available") from e
373381

374-
if slave_status["Slave_IO_Running"] == "Yes" and slave_status["Slave_SQL_Running"] == "Yes":
382+
if replica_status["Replica_IO_Running"] == "Yes" and replica_status["Replica_SQL_Running"] == "Yes":
375383
return
376384

377385
time.sleep(check_interval)
@@ -384,22 +392,22 @@ def _wait_for_replication(self, *, seconds_behind_master: int = 0, check_interva
384392

385393
while True:
386394
with self.target.cur() as cur:
387-
cur.execute("SHOW SLAVE STATUS")
395+
cur.execute("SHOW REPLICA STATUS")
388396
rows = cur.fetchall()
389397
if not rows:
390-
raise ReplicaSetupException("SHOW SLAVE STATUS didn't return any rows")
398+
raise ReplicaSetupException("SHOW REPLICA STATUS didn't return any rows")
391399

392400
try:
393-
slave_status = next(
401+
replica_status = next(
394402
row for row in rows
395-
if row["Master_Host"] == self.source.hostname and row["Master_Port"] == self.source.port
403+
if row["Source_Host"] == self.source.hostname and row["Source_Port"] == self.source.port
396404
)
397405
except StopIteration as e:
398-
raise ReplicaSetupException("Replication didn't catch up, Master info not available") from e
406+
raise ReplicaSetupException("Replication didn't catch up, source info not available") from e
399407

400-
lag = slave_status["Seconds_Behind_Master"]
408+
lag = replica_status["Seconds_Behind_Source"]
401409
if lag is None:
402-
raise ReplicaSetupException("Replication didn't catch up, Seconds_Behind_Master is null")
410+
raise ReplicaSetupException("Replication didn't catch up, Seconds_Behind_Source is null")
403411

404412
LOGGER.info("Current replication lag: %s seconds", lag)
405413
if lag <= seconds_behind_master:

‎docker-compose.test.yaml‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
version: '3.4'
2-
31
services:
42

53
test:
@@ -11,9 +9,11 @@ services:
119
- mysql80-src-2
1210
- mysql80-src-3
1311
- mysql80-src-4
12+
- mysql84-src-5
1413
- mysql80-dst-1
1514
- mysql80-dst-2
1615
- mysql80-dst-3
16+
- mysql84-dst-4
1717
volumes:
1818
- .:/app
1919

@@ -98,6 +98,19 @@ services:
9898
- --binlog-format=ROW
9999
- --ssl=OFF
100100

101+
mysql84-src-5:
102+
image: mysql:8.4
103+
restart: always
104+
environment:
105+
MYSQL_ROOT_PASSWORD: test
106+
command:
107+
- --server-id=1
108+
- --gtid-mode=ON
109+
- --enforce-gtid-consistency=ON
110+
- --binlog-checksum=NONE
111+
- --log-replica-updates=ON
112+
- --log-bin=binlog
113+
101114
mysql80-dst-1:
102115
image: mysql:8.0
103116
restart: always
@@ -145,3 +158,16 @@ services:
145158
- --log-slave-updates=ON
146159
- --log-bin=binlog
147160
- --binlog-format=ROW
161+
162+
mysql84-dst-4:
163+
image: mysql:8.4
164+
restart: always
165+
environment:
166+
MYSQL_ROOT_PASSWORD: test
167+
command:
168+
- --server-id=2
169+
- --gtid-mode=ON
170+
- --enforce-gtid-consistency=ON
171+
- --binlog-checksum=NONE
172+
- --log-replica-updates=ON
173+
- --log-bin=binlog

‎test/sys/test_migration.py‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,10 @@ def my_wait(host, ssl=True, retries=MYSQL_WAIT_RETRIES) -> MySQLConnectionInfo:
5252
(my_wait("mysql80-src-2"), my_wait("mysql80-dst-2"), "mysqldump"),
5353
(my_wait("mysql80-src-2"), my_wait("mysql80-dst-2"), "mydumper"),
5454
55+
(my_wait("mysql80-src-2"), my_wait("mysql84-dst-4"), "mysqldump"),
56+
(my_wait("mysql84-src-5"), my_wait("mysql84-dst-4"), "mysqldump"),
57+
(my_wait("mysql80-src-2"), my_wait("mysql84-dst-4"), "mydumper"),
58+
(my_wait("mysql84-src-5"), my_wait("mysql84-dst-4"), "mydumper"),
5559
]
5660
)
5761
def test_migration_replication(
@@ -111,7 +115,10 @@ def test_migration_replication(
111115
(my_wait("mysql57-src-1"), my_wait("mysql80-dst-1"), "mysqldump"),
112116
(my_wait("mysql80-src-2"), my_wait("mysql80-dst-2"), "mysqldump"),
113117
(my_wait("mysql80-src-2"), my_wait("mysql80-dst-2"), "mydumper"),
114-
118+
(my_wait("mysql80-src-2"), my_wait("mysql84-dst-4"), "mysqldump"),
119+
(my_wait("mysql84-src-5"), my_wait("mysql84-dst-4"), "mysqldump"),
120+
(my_wait("mysql80-src-2"), my_wait("mysql84-dst-4"), "mydumper"),
121+
(my_wait("mysql84-src-5"), my_wait("mysql84-dst-4"), "mydumper"),
115122
]
116123
)
117124
def test_migration_replication_with_reestablish_replication(
@@ -231,6 +238,15 @@ def test_migration_fallback(src: MySQLConnectionInfo, dst: MySQLConnectionInfo,
231238
),
232239
(my_wait("mysql80-src-3"), my_wait("mysql80-dst-3"), MySQLMigrateMethod.dump, does_not_raise(), "mysqldump"),
233240
(my_wait("mysql80-src-3"), my_wait("mysql80-dst-3"), MySQLMigrateMethod.dump, does_not_raise(), "mydumper"),
241+
242+
(my_wait("mysql80-src-2"), my_wait("mysql84-dst-4"), MySQLMigrateMethod.replication, does_not_raise(), "mysqldump"),
243+
(my_wait("mysql84-src-5"), my_wait("mysql84-dst-4"), MySQLMigrateMethod.replication, does_not_raise(), "mysqldump"),
244+
(my_wait("mysql80-src-2"), my_wait("mysql84-dst-4"), MySQLMigrateMethod.replication, does_not_raise(), "mydumper"),
245+
(my_wait("mysql84-src-5"), my_wait("mysql84-dst-4"), MySQLMigrateMethod.replication, does_not_raise(), "mydumper"),
246+
(my_wait("mysql80-src-2"), my_wait("mysql84-dst-4"), MySQLMigrateMethod.dump, does_not_raise(), "mysqldump"),
247+
(my_wait("mysql84-src-5"), my_wait("mysql84-dst-4"), MySQLMigrateMethod.dump, does_not_raise(), "mysqldump"),
248+
(my_wait("mysql80-src-2"), my_wait("mysql84-dst-4"), MySQLMigrateMethod.dump, does_not_raise(), "mydumper"),
249+
(my_wait("mysql84-src-5"), my_wait("mysql84-dst-4"), MySQLMigrateMethod.dump, does_not_raise(), "mydumper"),
234250
]
235251
)
236252
def test_force_migration_method( # pylint: disable=too-many-positional-arguments
@@ -260,6 +276,10 @@ def test_force_migration_method( # pylint: disable=too-many-positional-argument
260276
"src,dst,dump_tool", [
261277
(my_wait("mysql80-src-3"), my_wait("mysql80-dst-3"), "mysqldump"),
262278
(my_wait("mysql80-src-3"), my_wait("mysql80-dst-3"), "mydumper"),
279+
(my_wait("mysql80-src-2"), my_wait("mysql84-dst-4"), "mysqldump"),
280+
(my_wait("mysql84-src-5"), my_wait("mysql84-dst-4"), "mysqldump"),
281+
(my_wait("mysql80-src-2"), my_wait("mysql84-dst-4"), "mydumper"),
282+
(my_wait("mysql84-src-5"), my_wait("mysql84-dst-4"), "mydumper"),
263283
]
264284
)
265285
def test_database_size_check(src, dst, dump_tool, db_name):

0 commit comments

Comments
 (0)