sharedSchema: add more colum info checks for using shared schema (#3875) (#3877)

ti-chi-bot · hongyunyan · web-flow · commit 83c7c3dbe0d9 · 2026-01-01T22:50:36.000+08:00
* This is an automated cherry-pick of #3875 Signed-off-by: ti-chi-bot <ti-community-prow-bot@tidb.io> * update --------- Signed-off-by: ti-chi-bot <ti-community-prow-bot@tidb.io> Co-authored-by: hongyunyan <649330952@qq.com>
diff --git a/pkg/common/table_info_helper.go b/pkg/common/table_info_helper.go
@@ -17,6 +17,7 @@ import (
 	"crypto/sha256"
 	"encoding/binary"
 	"encoding/json"
+	"reflect"
 	"strings"
 	"sync"
 
@@ -90,6 +91,15 @@ func hashTableInfo(tableInfo *model.TableInfo) Digest {
 		}
 		binary.BigEndian.PutUint64(buf, uint64(boolToInt(columnType.IsArray())))
 		sha256Hasher.Write(buf)
+
+		binary.BigEndian.PutUint64(buf, uint64(boolToInt(col.DefaultIsExpr)))
+		sha256Hasher.Write(buf)
+		binary.BigEndian.PutUint64(buf, uint64(boolToInt(col.GeneratedStored)))
+		sha256Hasher.Write(buf)
+		binary.BigEndian.PutUint64(buf, uint64(boolToInt(col.Hidden)))
+		sha256Hasher.Write(buf)
+		binary.BigEndian.PutUint64(buf, col.Version)
+		sha256Hasher.Write(buf)
 	}
 	// idx info
 	sha256Hasher.Write([]byte("idxInfo"))
@@ -173,7 +183,27 @@ func (s *columnSchema) sameColumnsAndIndices(columns []*model.ColumnInfo, indice
 		if col.ID != columns[i].ID {
 			return false
 		}
-		if col.GetDefaultValue() != columns[i].GetDefaultValue() {
+		if !reflect.DeepEqual(col.GetDefaultValue(), columns[i].GetDefaultValue()) {
+			return false
+		}
+
+		if !reflect.DeepEqual(col.GetOriginDefaultValue(), columns[i].GetOriginDefaultValue()) {
+			return false
+		}
+
+		if col.DefaultIsExpr != columns[i].DefaultIsExpr {
+			return false
+		}
+		if col.GeneratedStored != columns[i].GeneratedStored {
+			return false
+		}
+		if col.Hidden != columns[i].Hidden {
+			return false
+		}
+		if col.GeneratedExprString != columns[i].GeneratedExprString {
+			return false
+		}
+		if col.Version != columns[i].Version {
 			return false
 		}
 	}
diff --git a/pkg/common/table_info_helper_test.go b/pkg/common/table_info_helper_test.go
@@ -504,15 +504,16 @@ func TestGetOrSetColumnSchema_SharedSchema(t *testing.T) {
 		},
 	}
 
-	// Get shared column schema storage
-	storage := GetSharedColumnSchemaStorage()
+	storage := &SharedColumnSchemaStorage{
+		m: make(map[Digest][]ColumnSchemaWithCount),
+	}
 
 	// Get column schema for both tables
 	columnSchema1 := storage.GetOrSetColumnSchema(tableInfo1)
 	columnSchema2 := storage.GetOrSetColumnSchema(tableInfo2)
 
 	// Verify that both tables share the same columnSchema object
-	require.Equal(t, columnSchema1, columnSchema2, "Tables with same schema should share the same columnSchema object")
+	require.Same(t, columnSchema1, columnSchema2, "Tables with same schema should share the same columnSchema object")
 
 	// Verify that the digest is the same
 	require.Equal(t, columnSchema1.Digest, columnSchema2.Digest, "Digest should be the same for tables with same schema")
@@ -572,10 +573,143 @@ func TestGetOrSetColumnSchema_SharedSchema(t *testing.T) {
 	columnSchema3 := storage.GetOrSetColumnSchema(tableInfo3)
 
 	// Verify that different schema creates a different columnSchema object
-	require.NotEqual(t, columnSchema1, columnSchema3, "Tables with different schema should have different columnSchema objects")
+	require.NotSame(t, columnSchema1, columnSchema3, "Tables with different schema should have different columnSchema objects")
 	require.NotEqual(t, columnSchema1.Digest, columnSchema3.Digest, "Digest should be different for tables with different schema")
 }
 
+func TestGetOrSetColumnSchema_SameColumnsAndIndices_ChecksAdditionalColumnAttrs(t *testing.T) {
+	tests := []struct {
+		name             string
+		mutate           func(col *model.ColumnInfo) error
+		expectDigestSame bool
+	}{
+		{
+			name: "origin default value",
+			mutate: func(col *model.ColumnInfo) error {
+				return col.SetOriginDefaultValue(int64(2))
+			},
+			expectDigestSame: true,
+		},
+		{
+			name: "default is expr",
+			mutate: func(col *model.ColumnInfo) error {
+				col.DefaultIsExpr = true
+				return nil
+			},
+			expectDigestSame: false,
+		},
+		{
+			name: "generated stored",
+			mutate: func(col *model.ColumnInfo) error {
+				col.GeneratedStored = true
+				return nil
+			},
+			expectDigestSame: false,
+		},
+		{
+			name: "hidden column",
+			mutate: func(col *model.ColumnInfo) error {
+				col.Hidden = true
+				return nil
+			},
+			expectDigestSame: false,
+		},
+		{
+			name: "generated expr string",
+			mutate: func(col *model.ColumnInfo) error {
+				col.GeneratedExprString = "id + 2"
+				return nil
+			},
+			expectDigestSame: true,
+		},
+		{
+			name: "column info version",
+			mutate: func(col *model.ColumnInfo) error {
+				col.Version = model.ColumnInfoVersion0
+				return nil
+			},
+			expectDigestSame: false,
+		},
+	}
+
+	newStorage := func() *SharedColumnSchemaStorage {
+		return &SharedColumnSchemaStorage{m: make(map[Digest][]ColumnSchemaWithCount)}
+	}
+
+	buildTable := func(idCol, otherCol *model.ColumnInfo) *model.TableInfo {
+		return &model.TableInfo{
+			ID:         1,
+			Name:       ast.NewCIStr("t"),
+			PKIsHandle: true,
+			Columns: []*model.ColumnInfo{
+				idCol,
+				otherCol,
+			},
+			Indices: []*model.IndexInfo{
+				{
+					ID:      1,
+					Name:    ast.NewCIStr("PRIMARY"),
+					Primary: true,
+					Unique:  true,
+					State:   model.StatePublic,
+					Columns: []*model.IndexColumn{
+						{
+							Name:   ast.NewCIStr("id"),
+							Offset: 0,
+						},
+					},
+				},
+			},
+		}
+	}
+
+	buildBaseTable := func() (*model.TableInfo, error) {
+		idCol := newColumnInfo(1, "id", mysql.TypeLong, mysql.PriKeyFlag|mysql.NotNullFlag)
+		idCol.Offset = 0
+		otherCol := newColumnInfo(2, "c", mysql.TypeLong, mysql.NotNullFlag|mysql.GeneratedColumnFlag)
+		otherCol.Offset = 1
+		if err := otherCol.SetDefaultValue(int64(1)); err != nil {
+			return nil, err
+		}
+		if err := otherCol.SetOriginDefaultValue(int64(1)); err != nil {
+			return nil, err
+		}
+		otherCol.DefaultIsExpr = false
+		otherCol.GeneratedExprString = "id + 1"
+		otherCol.GeneratedStored = false
+		otherCol.Hidden = false
+		otherCol.Version = model.CurrLatestColumnInfoVersion
+		return buildTable(idCol, otherCol), nil
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			storage := newStorage()
+
+			baseTable, err := buildBaseTable()
+			require.NoError(t, err)
+			baseSchema := storage.GetOrSetColumnSchema(baseTable)
+
+			variantTable, err := buildBaseTable()
+			require.NoError(t, err)
+			err = tt.mutate(variantTable.Columns[1])
+			require.NoError(t, err)
+			variantSchema := storage.GetOrSetColumnSchema(variantTable)
+
+			if tt.expectDigestSame {
+				require.Equal(t, baseSchema.Digest, variantSchema.Digest)
+				require.NotSame(t, baseSchema, variantSchema)
+				require.Len(t, storage.m[baseSchema.Digest], 2)
+			} else {
+				require.NotEqual(t, baseSchema.Digest, variantSchema.Digest)
+				require.NotSame(t, baseSchema, variantSchema)
+				require.Len(t, storage.m[baseSchema.Digest], 1)
+				require.Len(t, storage.m[variantSchema.Digest], 1)
+			}
+		})
+	}
+}
+
 func TestIsEligible(t *testing.T) {
 	t.Parallel()
 
diff --git a/tests/integration_tests/correctness_for_shared_column_schema/conf/diff_config.toml b/tests/integration_tests/correctness_for_shared_column_schema/conf/diff_config.toml
@@ -0,0 +1,29 @@
+# diff Configuration.
+
+check-thread-count = 4
+
+export-fix-sql = true
+
+check-struct-only = false
+
+[task]
+    output-dir = "/tmp/tidb_cdc_test/correctness_for_shared_column_schema/sync_diff/output"
+
+    source-instances = ["mysql1"]
+
+    target-instance = "tidb0"
+
+    target-check-tables = ["test_?00.t1"]
+
+[data-sources]
+[data-sources.mysql1]
+    host = "127.0.0.1"
+    port = 4000
+    user = "root"
+    password = ""
+
+[data-sources.tidb0]
+    host = "127.0.0.1"
+    port = 3306
+    user = "root"
+    password = ""
diff --git a/tests/integration_tests/correctness_for_shared_column_schema/data/downstream_prepare.sql b/tests/integration_tests/correctness_for_shared_column_schema/data/downstream_prepare.sql
@@ -0,0 +1,29 @@
+DROP DATABASE IF EXISTS test_100;
+CREATE DATABASE test_100;
+USE test_100;
+CREATE TABLE t1 (a INT, b INT, site_code VARCHAR(64) NOT NULL DEFAULT '', PRIMARY KEY (a, site_code));
+INSERT INTO t1 VALUES (1, 2, '100'), (2, 3, '100'), (12, 13, '100');
+
+DROP DATABASE IF EXISTS test_200;
+CREATE DATABASE test_200;
+USE test_200;
+CREATE TABLE t1 (a INT, b INT, site_code VARCHAR(64) NOT NULL DEFAULT '', PRIMARY KEY (a, site_code));
+INSERT INTO t1 VALUES (1, 2, '200'), (2, 3, '200'), (12, 13, '200');
+
+DROP DATABASE IF EXISTS test_300;
+CREATE DATABASE test_300;
+USE test_300;
+CREATE TABLE t1 (a INT, b INT, site_code VARCHAR(64) NOT NULL DEFAULT '', PRIMARY KEY (a, site_code));
+INSERT INTO t1 VALUES (1, 2, '300'), (2, 3, '300'), (12, 13, '300');
+
+DROP DATABASE IF EXISTS test_400;
+CREATE DATABASE test_400;
+USE test_400;
+CREATE TABLE t1 (a INT, b INT, site_code VARCHAR(64) NOT NULL DEFAULT '', PRIMARY KEY (a, site_code));
+INSERT INTO t1 VALUES (1, 2, '400'), (2, 3, '400'), (12, 13, '400');
+
+DROP DATABASE IF EXISTS test_500;
+CREATE DATABASE test_500;
+USE test_500;
+CREATE TABLE t1 (a INT, b INT, site_code VARCHAR(64) NOT NULL DEFAULT '', PRIMARY KEY (a, site_code));
+INSERT INTO t1 VALUES (1, 2, '500'), (2, 3, '500'), (12, 13, '500');
diff --git a/tests/integration_tests/correctness_for_shared_column_schema/data/upstream_dml.sql b/tests/integration_tests/correctness_for_shared_column_schema/data/upstream_dml.sql
@@ -0,0 +1,29 @@
+USE test_100;
+INSERT INTO t1 (a, b) VALUES (20, 21);
+UPDATE t1 SET b = b + 10 WHERE a = 1 AND site_code = '100';
+DELETE FROM t1 WHERE a = 2 AND site_code = '100';
+UPDATE t1 SET b = b + 1 WHERE a = 12 AND site_code = '100';
+
+USE test_200;
+INSERT INTO t1 (a, b) VALUES (20, 21);
+UPDATE t1 SET b = b + 10 WHERE a = 1 AND site_code = '200';
+DELETE FROM t1 WHERE a = 2 AND site_code = '200';
+UPDATE t1 SET b = b + 1 WHERE a = 12 AND site_code = '200';
+
+USE test_300;
+INSERT INTO t1 (a, b) VALUES (20, 21);
+UPDATE t1 SET b = b + 10 WHERE a = 1 AND site_code = '300';
+DELETE FROM t1 WHERE a = 2 AND site_code = '300';
+UPDATE t1 SET b = b + 1 WHERE a = 12 AND site_code = '300';
+
+USE test_400;
+INSERT INTO t1 (a, b) VALUES (20, 21);
+UPDATE t1 SET b = b + 10 WHERE a = 1 AND site_code = '400';
+DELETE FROM t1 WHERE a = 2 AND site_code = '400';
+UPDATE t1 SET b = b + 1 WHERE a = 12 AND site_code = '400';
+
+USE test_500;
+INSERT INTO t1 (a, b) VALUES (20, 21);
+UPDATE t1 SET b = b + 10 WHERE a = 1 AND site_code = '500';
+DELETE FROM t1 WHERE a = 2 AND site_code = '500';
+UPDATE t1 SET b = b + 1 WHERE a = 12 AND site_code = '500';
diff --git a/tests/integration_tests/correctness_for_shared_column_schema/data/upstream_prepare.sql b/tests/integration_tests/correctness_for_shared_column_schema/data/upstream_prepare.sql
@@ -0,0 +1,49 @@
+DROP DATABASE IF EXISTS test_100;
+CREATE DATABASE test_100;
+USE test_100;
+CREATE TABLE t1 (a INT, b INT);
+INSERT INTO t1 VALUES (1, 2), (2, 3);
+ALTER TABLE t1 ADD COLUMN site_code VARCHAR(64) NOT NULL DEFAULT '100';
+INSERT INTO t1 VALUES (12, 13, '100');
+ALTER TABLE t1 MODIFY COLUMN site_code VARCHAR(64) NOT NULL DEFAULT '';
+ALTER TABLE t1 ADD PRIMARY KEY (a, site_code);
+
+DROP DATABASE IF EXISTS test_200;
+CREATE DATABASE test_200;
+USE test_200;
+CREATE TABLE t1 (a INT, b INT);
+INSERT INTO t1 VALUES (1, 2), (2, 3);
+ALTER TABLE t1 ADD COLUMN site_code VARCHAR(64) NOT NULL DEFAULT '200';
+INSERT INTO t1 VALUES (12, 13, '200');
+ALTER TABLE t1 MODIFY COLUMN site_code VARCHAR(64) NOT NULL DEFAULT '';
+ALTER TABLE t1 ADD PRIMARY KEY (a, site_code);
+
+DROP DATABASE IF EXISTS test_300;
+CREATE DATABASE test_300;
+USE test_300;
+CREATE TABLE t1 (a INT, b INT);
+INSERT INTO t1 VALUES (1, 2), (2, 3);
+ALTER TABLE t1 ADD COLUMN site_code VARCHAR(64) NOT NULL DEFAULT '300';
+INSERT INTO t1 VALUES (12, 13, '300');
+ALTER TABLE t1 MODIFY COLUMN site_code VARCHAR(64) NOT NULL DEFAULT '';
+ALTER TABLE t1 ADD PRIMARY KEY (a, site_code);
+
+DROP DATABASE IF EXISTS test_400;
+CREATE DATABASE test_400;
+USE test_400;
+CREATE TABLE t1 (a INT, b INT);
+INSERT INTO t1 VALUES (1, 2), (2, 3);
+ALTER TABLE t1 ADD COLUMN site_code VARCHAR(64) NOT NULL DEFAULT '400';
+INSERT INTO t1 VALUES (12, 13, '400');
+ALTER TABLE t1 MODIFY COLUMN site_code VARCHAR(64) NOT NULL DEFAULT '';
+ALTER TABLE t1 ADD PRIMARY KEY (a, site_code);
+
+DROP DATABASE IF EXISTS test_500;
+CREATE DATABASE test_500;
+USE test_500;
+CREATE TABLE t1 (a INT, b INT);
+INSERT INTO t1 VALUES (1, 2), (2, 3);
+ALTER TABLE t1 ADD COLUMN site_code VARCHAR(64) NOT NULL DEFAULT '500';
+INSERT INTO t1 VALUES (12, 13, '500');
+ALTER TABLE t1 MODIFY COLUMN site_code VARCHAR(64) NOT NULL DEFAULT '';
+ALTER TABLE t1 ADD PRIMARY KEY (a, site_code);
diff --git a/tests/integration_tests/correctness_for_shared_column_schema/run.sh b/tests/integration_tests/correctness_for_shared_column_schema/run.sh
diff --git a/tests/integration_tests/run_heavy_it_in_ci.sh b/tests/integration_tests/run_heavy_it_in_ci.sh
diff --git a/tests/integration_tests/run_light_it_in_ci.sh b/tests/integration_tests/run_light_it_in_ci.sh
