Skip to content

feat: Support MySQL data synchronization collation #11067

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wanghe-fit2cloud merged 1 commit into from
Nov 25, 2025
Merged

feat: Support MySQL data synchronization collation #11067

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions agent/app/dto/database.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type MysqlDBInfo struct {
From string `json:"from"`
MysqlName string `json:"mysqlName"`
Format string `json:"format"`
Collation string `json:"collation"`
Username string `json:"username"`
Password string `json:"password"`
Permission string `json:"permission"`
Expand Down
29 changes: 26 additions & 3 deletions agent/init/migration/migrations/init.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -610,7 +610,7 @@
if !strings.Contains(item.Spec, ",") {
continue
}
if err := tx.Model(&model.Cronjob{}).Where("id = ?", item.ID).Updates(

Check failure on line 613 in agent/init/migration/migrations/init.go

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Define a constant instead of duplicating this literal "id = ?" 3 times. 


See more on https://sonarcloud.io/project/issues?id=1Panel-dev_1Panel&issues=AZq5wxw4vAx7qZsi7YKr&open=AZq5wxw4vAx7qZsi7YKr&pullRequest=11067
map[string]interface{}{"spec": strings.ReplaceAll(item.Spec, ",", "&&")}).Error; err != nil {
return err
}
Expand Down Expand Up @@ -728,8 +728,31 @@
}

var UpdateDatabaseMysql = &gormigrate.Migration{
ID: "20251125-update-database-mysql",
Migrate: func(tx *gorm.DB) error {
return tx.AutoMigrate(&model.DatabaseMysql{})
ID: "20251126-update-database-mysql",
Migrate: func(tx *gorm.DB) error {
if err := tx.AutoMigrate(&model.DatabaseMysql{}); err != nil {
return err
}
var data []model.DatabaseMysql
_ = tx.Where("1 = 1").Find(&data).Error
for _, item := range data {
if len(item.Collation) == 0 {
collation := ""
switch item.Format {
case "utf8":
collation = "utf8_general_ci"
case "utf8mb4":
collation = "utf8mb4_general_ci"
case "gbk":
collation = "gbk_chinese_ci"
case "big5":
collation = "big5_chinese_ci"
default:
collation = "utf8mb4_general_ci"
}
_ = tx.Model(&model.DatabaseMysql{}).Where("id = ?", item.ID).Updates(map[string]interface{}{"collation": collation}).Error
}
}
return nil
},
}
1 change: 1 addition & 0 deletions agent/utils/mysql/client/info.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ type SyncDBInfo struct {
From string `json:"from"`
MysqlName string `json:"mysqlName"`
Format string `json:"format"`
Collation string `json:"collation"`
Username string `json:"username"`
Password string `json:"password"`
Permission string `json:"permission"`
Expand Down
8 changes: 6 additions & 2 deletions agent/utils/mysql/client/local.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@

func (r *Local) Create(info CreateInfo) error {
createSql := fmt.Sprintf("create database `%s` default character set %s collate %s", info.Name, info.Format, info.Collation)
if len(info.Collation) == 0 {
createSql = fmt.Sprintf("create database `%s` default character set %s", info.Name, info.Format)
}
if err := r.ExecSQL(createSql, info.Timeout); err != nil {
if strings.Contains(strings.ToLower(err.Error()), "error 1007") {
return buserr.New("ErrDatabaseIsExist")
Expand Down Expand Up @@ -285,15 +288,15 @@
return nil
}

func (r *Local) SyncDB(version string) ([]SyncDBInfo, error) {

Check failure on line 291 in agent/utils/mysql/client/local.go

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Refactor this method to reduce its Cognitive Complexity from 35 to the 15 allowed. 


See more on https://sonarcloud.io/project/issues?id=1Panel-dev_1Panel&issues=AZq5wxwkvAx7qZsi7YKq&open=AZq5wxwkvAx7qZsi7YKq&pullRequest=11067
var datas []SyncDBInfo
lines, err := r.ExecSQLForRows("SELECT SCHEMA_NAME, DEFAULT_CHARACTER_SET_NAME FROM information_schema.SCHEMATA", 300)
lines, err := r.ExecSQLForRows("SELECT SCHEMA_NAME, DEFAULT_CHARACTER_SET_NAME, DEFAULT_COLLATION_NAME FROM information_schema.SCHEMATA", 300)
if err != nil {
return datas, err
}
for _, line := range lines {
parts := strings.Fields(line)
if len(parts) != 2 {
if len(parts) != 3 {
continue
}
if parts[0] == "SCHEMA_NAME" || parts[0] == "information_schema" || parts[0] == "mysql" || parts[0] == "performance_schema" || parts[0] == "sys" || parts[0] == "__recycle_bin__" || parts[0] == "recycle_bin" {
Expand All @@ -304,6 +307,7 @@
From: "local",
MysqlName: r.Database,
Format: parts[1],
Collation: parts[2],
}
userLines, err := r.ExecSQLForRows(fmt.Sprintf("select user,host from mysql.db where db = '%s'", parts[0]), 300)
if err != nil {
Expand Down
10 changes: 7 additions & 3 deletions agent/utils/mysql/client/remote.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@

func (r *Remote) Create(info CreateInfo) error {
createSql := fmt.Sprintf("create database `%s` default character set %s collate %s", info.Name, info.Format, info.Collation)
if len(info.Collation) == 0 {
createSql = fmt.Sprintf("create database `%s` default character set %s", info.Name, info.Format)
}
if err := r.ExecSQL(createSql, info.Timeout); err != nil {
if strings.Contains(strings.ToLower(err.Error()), "error 1007") {
return buserr.New("ErrDatabaseIsExist")
Expand Down Expand Up @@ -309,17 +312,17 @@
return nil
}

func (r *Remote) SyncDB(version string) ([]SyncDBInfo, error) {

Check failure on line 315 in agent/utils/mysql/client/remote.go

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Refactor this method to reduce its Cognitive Complexity from 36 to the 15 allowed. 


See more on https://sonarcloud.io/project/issues?id=1Panel-dev_1Panel&issues=AZq5wxr8vAx7qZsi7YKp&open=AZq5wxr8vAx7qZsi7YKp&pullRequest=11067
var datas []SyncDBInfo
rows, err := r.Client.Query("select schema_name, default_character_set_name from information_schema.SCHEMATA")
rows, err := r.Client.Query("select schema_name, default_character_set_name, default_collation_name from information_schema.SCHEMATA")
if err != nil {
return datas, err
}
defer rows.Close()

for rows.Next() {
var dbName, charsetName string
if err = rows.Scan(&dbName, &charsetName); err != nil {
var dbName, charsetName, collation string
if err = rows.Scan(&dbName, &charsetName, &collation); err != nil {
return datas, err
}
if dbName == "information_schema" || dbName == "mysql" || dbName == "performance_schema" || dbName == "sys" || dbName == "__recycle_bin__" || dbName == "recycle_bin" {
Expand All @@ -330,6 +333,7 @@
From: "remote",
MysqlName: r.Database,
Format: charsetName,
Collation: collation,
}
userRows, err := r.Client.Query("select user,host from mysql.db where db = ?", dbName)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion frontend/src/views/database/mysql/create/index.vue
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
</el-select>
</el-form-item>
<el-form-item :label="$t('database.collation')" prop="collation">
<el-select filterable v-model="form.collation">
<el-select filterable v-model="form.collation" clearable>
<el-option v-for="item of collationOptions" :key="item" :label="item" :value="item" />
</el-select>
<span class="input-help">{{ $t('database.collationHelper', [form.format]) }}</span>
Expand Down
Loading