cdc(ddl): ensure strict ordering for multi-table DDLs after split (#12450) (#12457)

close #12449

Author: ti-chi-bot

cdc/entry/schema_storage.go

@@ -519,6 +519,7 @@ func (s *schemaStorage) buildRenameEvents(
 		tableInfo := model.WrapTableInfo(info.NewSchemaID, newSchemaName,
 			job.BinlogInfo.FinishedTS, tableInfo)
 		event.FromJobWithArgs(job, preTableInfo, tableInfo, oldSchemaName, newSchemaName)
+		event.Seq = uint64(i)
 		ddlEvents = append(ddlEvents, event)
 	}
 	return ddlEvents, nil

cdc/model/sink.go

@@ -1056,6 +1056,10 @@ type DDLEvent struct {
 	// the DDL is executed by the primary cluster.
 	BDRRole string        `msg:"-"`
 	SQLMode mysql.SQLMode `msg:"-"`
+	// Seq is used to order the DDLs with the same commit ts
+	// Only used in the splited DDLEvent generated by a multi-table DDL,
+	// we need to keep the order of the original multi-table DDL
+	Seq uint64 `msg:"seq"`
 }
 
 // FromJob fills the values with DDLEvent from DDL job

cdc/model/sink_gen.go

@@ -484,7 +484,7 @@ func (m *ddlManager) getNextDDL() *model.DDLEvent {
 			delete(m.pendingDDLs, tb)
 			continue
 		}
-		if res == nil || res.CommitTs > ddls[0].CommitTs {
+		if res == nil || res.CommitTs > ddls[0].CommitTs || (res.CommitTs == ddls[0].CommitTs && res.Seq > ddls[0].Seq) {
 			res = ddls[0]
 		}
 	}

cdc/owner/ddl_manager.go

@@ -175,3 +175,33 @@ VALUES (1),
 UPDATE `column_is_null`
 SET t = NULL
 WHERE id = 1;
+
+-- rename tables DDL should keep the order of events
+
+CREATE TABLE `rename_t3` (
+    `id`  INT PRIMARY KEY,
+    `val` INT
+);
+
+CREATE TABLE `rename_t1` (
+    `id`  INT PRIMARY KEY,
+    `val` INT
+);
+
+CREATE TABLE `rename_t2` (
+    `id`  INT PRIMARY KEY,
+    `val` INT
+);
+
+INSERT INTO `rename_t1` VALUES (1, 1);
+INSERT INTO `rename_t2` VALUES (1, 2);
+INSERT INTO `rename_t3` VALUES (1, 3);
+
+RENAME TABLE
+    `rename_t1` TO `rename_t4`,
+    `rename_t2` TO `rename_t1`,
+    `rename_t3` TO `rename_t2`;
+
+INSERT INTO `rename_t4` VALUES (2, 4);
+INSERT INTO `rename_t1` VALUES (2, 1);
+INSERT INTO `rename_t2` VALUES (2, 2);

tests/integration_tests/common_1/data/test.sql

@@ -86,6 +86,9 @@ EOF
 	# sync_diff can't check non-exist table, so we check expected tables are created in downstream first
 	check_table_exists common_1.v1 ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
 	check_table_exists common_1.recover_and_insert ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
+	check_table_exists common_1.rename_t4 ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
+	check_table_exists common_1.rename_t1 ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
+	check_table_exists common_1.rename_t2 ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
 	check_table_exists common_1.finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
 	check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml