feat: Support more MySQL character sets and collations

ssongliu · ssongliu · commit 7bde7ed116b6 · 2025-11-24T15:58:31.000+08:00
diff --git a/agent/app/api/v2/database_mysql.go b/agent/app/api/v2/database_mysql.go
@@ -212,6 +212,22 @@ func (b *BaseApi) ListDBName(c *gin.Context) {
 	helper.SuccessWithData(c, list)
 }
 
+// @Tags Database Mysql
+// @Summary List mysql database format collation options
+// @Accept json
+// @Param request body dto.OperationWithName true "request"
+// @Success 200 {array} dto.MysqlFormatCollationOption
+// @Security ApiKeyAuth
+// @Security Timestamp
+// @Router /databases/format/options [post]
+func (b *BaseApi) ListDBFormatCollationOptions(c *gin.Context) {
+	var req dto.OperationWithName
+	if err := helper.CheckBindAndValidate(&req, c); err != nil {
+		return
+	}
+	helper.SuccessWithData(c, mysqlService.LoadFormatOption(req))
+}
+
 // @Tags Database Mysql
 // @Summary Load mysql database from remote
 // @Accept json
diff --git a/agent/app/dto/database.go b/agent/app/dto/database.go
@@ -58,12 +58,18 @@ type MysqlDBCreate struct {
 	From        string `json:"from" validate:"required,oneof=local remote"`
 	Database    string `json:"database" validate:"required"`
 	Format      string `json:"format" validate:"required,oneof=utf8mb4 utf8 gbk big5"`
+	Collation   string `json:"collation" validate:"required"`
 	Username    string `json:"username" validate:"required"`
 	Password    string `json:"password" validate:"required"`
 	Permission  string `json:"permission" validate:"required"`
 	Description string `json:"description"`
 }
 
+type MysqlFormatCollationOption struct {
+	Format     string   `json:"format"`
+	Collations []string `json:"collations"`
+}
+
 type BindUser struct {
 	Database   string `json:"database" validate:"required"`
 	DB         string `json:"db" validate:"required"`
diff --git a/agent/app/model/database_mysql.go b/agent/app/model/database_mysql.go
@@ -6,6 +6,7 @@ type DatabaseMysql struct {
 	From        string `json:"from" gorm:"not null;default:local"`
 	MysqlName   string `json:"mysqlName" gorm:"not null"`
 	Format      string `json:"format" gorm:"not null"`
+	Collation   string `json:"collation" gorm:"not null"`
 	Username    string `json:"username" gorm:"not null"`
 	Password    string `json:"password" gorm:"not null"`
 	Permission  string `json:"permission" gorm:"not null"`
diff --git a/agent/app/service/database_mysql.go b/agent/app/service/database_mysql.go
@@ -23,8 +23,8 @@ import (
 	"github.com/1Panel-dev/1Panel/agent/utils/compose"
 	"github.com/1Panel-dev/1Panel/agent/utils/encrypt"
 	"github.com/1Panel-dev/1Panel/agent/utils/mysql"
-	"github.com/1Panel-dev/1Panel/agent/utils/re"
 	"github.com/1Panel-dev/1Panel/agent/utils/mysql/client"
+	"github.com/1Panel-dev/1Panel/agent/utils/re"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/jinzhu/copier"
 	"github.com/pkg/errors"
@@ -45,6 +45,7 @@ type IMysqlService interface {
 	DeleteCheck(req dto.MysqlDBDeleteCheck) ([]dto.DBResource, error)
 	Delete(ctx context.Context, req dto.MysqlDBDelete) error
 
+	LoadFormatOption(req dto.OperationWithName) []dto.MysqlFormatCollationOption
 	LoadStatus(req dto.OperationWithNameAndType) (*dto.MysqlStatus, error)
 	LoadVariables(req dto.OperationWithNameAndType) (*dto.MysqlVariables, error)
 	LoadRemoteAccess(req dto.OperationWithNameAndType) (bool, error)
@@ -126,6 +127,7 @@ func (u *MysqlService) Create(ctx context.Context, req dto.MysqlDBCreate) (*mode
 	if err := cli.Create(client.CreateInfo{
 		Name:       req.Name,
 		Format:     req.Format,
+		Collation:  req.Collation,
 		Username:   req.Username,
 		Password:   req.Password,
 		Permission: req.Permission,
@@ -584,6 +586,19 @@ func (u *MysqlService) LoadStatus(req dto.OperationWithNameAndType) (*dto.MysqlS
 	return &info, nil
 }
 
+func (u *MysqlService) LoadFormatOption(req dto.OperationWithName) []dto.MysqlFormatCollationOption {
+	defaultList := []dto.MysqlFormatCollationOption{{Format: "utf8mb4"}, {Format: "utf8mb3"}, {Format: "gbk"}, {Format: "big5"}}
+	client, _, err := LoadMysqlClientByFrom(req.Name)
+	if err != nil {
+		return defaultList
+	}
+	options, err := client.LoadFormatCollation(3)
+	if err != nil {
+		return defaultList
+	}
+	return options
+}
+
 func executeSqlForMaps(containerName, dbType, password, command string) (map[string]string, error) {
 	if dbType == "mysql-cluster" {
 		dbType = "mysql"
diff --git a/agent/init/migration/migrate.go b/agent/init/migration/migrate.go
@@ -53,6 +53,7 @@ func InitAgentDB() {
 		migrations.UpdateTensorrtLLM,
 		migrations.AddIptablesFilterRuleTable,
 		migrations.UpdateDatabase,
+		migrations.UpdateDatabaseMysql,
 	})
 	if err := m.Migrate(); err != nil {
 		global.LOG.Error(err)
diff --git a/agent/init/migration/migrations/init.go b/agent/init/migration/migrations/init.go
@@ -712,3 +712,9 @@ var UpdateDatabase = &gormigrate.Migration{
 		return tx.AutoMigrate(&model.Database{})
 	},
 }
+var UpdateDatabaseMysql = &gormigrate.Migration{
+	ID: "20251124-update-database-mysql",
+	Migrate: func(tx *gorm.DB) error {
+		return tx.AutoMigrate(&model.Database{})
+	},
+}
diff --git a/agent/router/ro_database.go b/agent/router/ro_database.go
@@ -28,7 +28,7 @@ func (s *DatabaseRouter) InitRouter(Router *gin.RouterGroup) {
 		cmdRouter.POST("/variables", baseApi.LoadVariables)
 		cmdRouter.POST("/status", baseApi.LoadStatus)
 		cmdRouter.POST("/remote", baseApi.LoadRemoteAccess)
-		cmdRouter.GET("/options", baseApi.ListDBName)
+		cmdRouter.POST("/format/options", baseApi.ListDBFormatCollationOptions)
 
 		cmdRouter.POST("/redis/persistence/conf", baseApi.LoadPersistenceConf)
 		cmdRouter.POST("/redis/status", baseApi.LoadRedisStatus)
diff --git a/agent/utils/mysql/client.go b/agent/utils/mysql/client.go
@@ -8,6 +8,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/1Panel-dev/1Panel/agent/app/dto"
 	"github.com/1Panel-dev/1Panel/agent/buserr"
 	"github.com/1Panel-dev/1Panel/agent/global"
 	"github.com/1Panel-dev/1Panel/agent/utils/mysql/client"
@@ -24,6 +25,7 @@ type MysqlClient interface {
 	Backup(info client.BackupInfo) error
 	Recover(info client.RecoverInfo) error
 
+	LoadFormatCollation(timeout uint) ([]dto.MysqlFormatCollationOption, error)
 	SyncDB(version string) ([]client.SyncDBInfo, error)
 	Close()
 }
diff --git a/agent/utils/mysql/client/info.go b/agent/utils/mysql/client/info.go
@@ -30,6 +30,7 @@ type DBInfo struct {
 type CreateInfo struct {
 	Name       string `json:"name"`
 	Format     string `json:"format"`
+	Collation  string `json:"collation"`
 	Version    string `json:"version"`
 	Username   string `json:"userName"`
 	Password   string `json:"password"`
@@ -80,6 +81,11 @@ type BackupInfo struct {
 	Timeout uint `json:"timeout"` // second
 }
 
+type FormatCollation struct {
+	Format    string `json:"format" gorm:"column:CHARACTER_SET_NAME"`
+	Collation string `json:"collation" gorm:"column:COLLATION_NAME"`
+}
+
 type RecoverInfo struct {
 	Name       string `json:"name"`
 	Type       string `json:"type"`
@@ -100,13 +106,6 @@ type SyncDBInfo struct {
 	Permission string `json:"permission"`
 }
 
-var formatMap = map[string]string{
-	"utf8":    "utf8_general_ci",
-	"utf8mb4": "utf8mb4_general_ci",
-	"gbk":     "gbk_chinese_ci",
-	"big5":    "big5_chinese_ci",
-}
-
 func ConnWithSSL(ssl, skipVerify bool, clientKey, clientCert, rootCert string) (string, error) {
 	if !ssl {
 		return "", nil
diff --git a/agent/utils/mysql/client/local.go b/agent/utils/mysql/client/local.go
@@ -12,6 +12,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/1Panel-dev/1Panel/agent/app/dto"
 	"github.com/1Panel-dev/1Panel/agent/buserr"
 	"github.com/1Panel-dev/1Panel/agent/constant"
 	"github.com/1Panel-dev/1Panel/agent/global"
@@ -31,7 +32,7 @@ func NewLocal(command []string, dbType, containerName, password, database string
 }
 
 func (r *Local) Create(info CreateInfo) error {
-	createSql := fmt.Sprintf("create database `%s` default character set %s collate %s", info.Name, info.Format, formatMap[info.Format])
+	createSql := fmt.Sprintf("create database `%s` default character set %s collate %s", info.Name, info.Format, info.Collation)
 	if err := r.ExecSQL(createSql, info.Timeout); err != nil {
 		if strings.Contains(strings.ToLower(err.Error()), "error 1007") {
 			return buserr.New("ErrDatabaseIsExist")
@@ -385,3 +386,33 @@ func (r *Local) ExecSQLForRows(command string, timeout uint) ([]string, error) {
 	}
 	return strings.Split(stdStr, "\n"), nil
 }
+
+func (r *Local) LoadFormatCollation(timeout uint) ([]dto.MysqlFormatCollationOption, error) {
+	std, err := r.ExecSQLForRows("SELECT CHARACTER_SET_NAME, COLLATION_NAME FROM INFORMATION_SCHEMA.COLLATIONS ORDER BY CHARACTER_SET_NAME, COLLATION_NAME;", timeout)
+	if err != nil {
+		return nil, err
+	}
+	formatMap := make(map[string][]string)
+	for _, item := range std {
+		if strings.ToLower(item) == "character_set_name\tcollation_name" {
+			continue
+		}
+		parts := strings.Split(item, "\t")
+		if len(parts) != 2 {
+			continue
+		}
+		if _, ok := formatMap[parts[0]]; !ok {
+			formatMap[parts[0]] = []string{parts[1]}
+		} else {
+			formatMap[parts[0]] = append(formatMap[parts[0]], parts[1])
+		}
+	}
+	options := []dto.MysqlFormatCollationOption{}
+	for key, val := range formatMap {
+		options = append(options, dto.MysqlFormatCollationOption{
+			Format:     key,
+			Collations: val,
+		})
+	}
+	return options, nil
+}
diff --git a/agent/utils/mysql/client/remote.go b/agent/utils/mysql/client/remote.go
@@ -13,6 +13,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/1Panel-dev/1Panel/agent/app/dto"
 	"github.com/1Panel-dev/1Panel/agent/buserr"
 	"github.com/1Panel-dev/1Panel/agent/constant"
 	"github.com/1Panel-dev/1Panel/agent/global"
@@ -42,7 +43,7 @@ func NewRemote(db Remote) *Remote {
 }
 
 func (r *Remote) Create(info CreateInfo) error {
-	createSql := fmt.Sprintf("create database `%s` default character set %s collate %s", info.Name, info.Format, formatMap[info.Format])
+	createSql := fmt.Sprintf("create database `%s` default character set %s collate %s", info.Name, info.Format, info.Collation)
 	if err := r.ExecSQL(createSql, info.Timeout); err != nil {
 		if strings.Contains(strings.ToLower(err.Error()), "error 1007") {
 			return buserr.New("ErrDatabaseIsExist")
@@ -397,6 +398,42 @@ func (r *Remote) ExecSQL(command string, timeout uint) error {
 	return nil
 }
 
+func (r *Remote) LoadFormatCollation(timeout uint) ([]dto.MysqlFormatCollationOption, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
+	defer cancel()
+
+	rows, err := r.Client.QueryContext(ctx, "SELECT CHARACTER_SET_NAME, COLLATION_NAME FROM INFORMATION_SCHEMA.COLLATIONS ORDER BY CHARACTER_SET_NAME, COLLATION_NAME;")
+	if errors.Is(ctx.Err(), context.DeadlineExceeded) {
+		return nil, buserr.New("ErrExecTimeOut")
+	}
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	formatMap := make(map[string][]string)
+	for rows.Next() {
+		var item FormatCollation
+		if err := rows.Scan(&item.Format, &item.Collation); err != nil {
+			return nil, err
+		}
+		if _, ok := formatMap[item.Format]; !ok {
+			formatMap[item.Format] = []string{item.Collation}
+		} else {
+			formatMap[item.Format] = append(formatMap[item.Format], item.Collation)
+		}
+	}
+	options := []dto.MysqlFormatCollationOption{}
+	for key, val := range formatMap {
+		options = append(options, dto.MysqlFormatCollationOption{
+			Format:     key,
+			Collations: val,
+		})
+	}
+
+	return options, nil
+}
+
 func (r *Remote) ExecSQLForHosts(timeout uint) ([]string, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
 	defer cancel()
diff --git a/frontend/src/api/interface/database.ts b/frontend/src/api/interface/database.ts
@@ -142,6 +142,10 @@ export namespace Database {
         File: string;
         Position: number;
     }
+    export interface FormatCollationOption {
+        format: string;
+        collations: Array<string>;
+    }
     export interface PgLoadDB {
         from: string;
         type: string;
diff --git a/frontend/src/api/modules/database.ts b/frontend/src/api/modules/database.ts
@@ -106,6 +106,9 @@ export const loadMysqlStatus = (type: string, database: string) => {
 export const loadRemoteAccess = (type: string, database: string) => {
     return http.post<boolean>(`/databases/remote`, { type: type, name: database });
 };
+export const loadFormatCollations = (database: string) => {
+    return http.post<Array<Database.FormatCollationOption>>(`/databases/format/options`, { name: database });
+};
 
 // redis
 export const loadRedisStatus = (type: string, database: string) => {
diff --git a/frontend/src/lang/modules/en.ts b/frontend/src/lang/modules/en.ts
@@ -481,6 +481,9 @@ const message = {
         goInstall: 'Go to install',
         isDelete: 'Deleted',
         permission: 'Change permissions',
+        format: 'Character Set',
+        collation: 'Collation',
+        collationHelper: 'If empty, use the default collation of the {0} character set',
         permissionForIP: 'IP',
         permissionAll: 'All of them(%)',
         localhostHelper:
diff --git a/frontend/src/lang/modules/es-es.ts b/frontend/src/lang/modules/es-es.ts
@@ -485,6 +485,9 @@ const message = {
         goInstall: 'Ir a instalar',
         isDelete: 'Eliminada',
         permission: 'Cambiar permisos',
+        format: 'Juego de Caracteres',
+        collation: 'Intercalación',
+        collationHelper: 'Si está vacío, use la intercalación predeterminada del juego de caracteres {0}',
         permissionForIP: 'IP',
         permissionAll: 'Todos (%)',
         localhostHelper:
diff --git a/frontend/src/lang/modules/ja.ts b/frontend/src/lang/modules/ja.ts
@@ -469,6 +469,9 @@ const message = {
         goInstall: 'インストールに移動します',
         isDelete: '削除されました',
         permission: '権限',
+        format: '文字セット',
+        collation: '照合順序',
+        collationHelper: '空の場合は {0} 文字セットのデフォルトの照合順序を使用します',
         permissionForIP: 'ip',
         permissionAll: 'それらすべて（％）',
         localhostHelper:
diff --git a/frontend/src/lang/modules/ko.ts b/frontend/src/lang/modules/ko.ts
@@ -472,6 +472,9 @@ const message = {
         goInstall: '설치로 이동',
         isDelete: '삭제됨',
         permission: '권한',
+        format: '문자 집합',
+        collation: '콜레이션',
+        collationHelper: '비어 있으면 {0} 문자 집합의 기본 콜레이션을 사용합니다',
         permissionForIP: 'IP',
         permissionAll: '모두(%)',
         localhostHelper:
diff --git a/frontend/src/lang/modules/ms.ts b/frontend/src/lang/modules/ms.ts
@@ -478,6 +478,9 @@ const message = {
         goInstall: 'Pergi pasang',
         isDelete: 'Dihapuskan',
         permission: 'Kebenaran',
+        format: 'Set Aksara',
+        collation: 'Kolasi',
+        collationHelper: 'Jika kosong, gunakan kolasi lalai set aksara {0}',
         permissionForIP: 'IP',
         permissionAll: 'Kesemuanya(%)',
         localhostHelper:
diff --git a/frontend/src/lang/modules/pt-br.ts b/frontend/src/lang/modules/pt-br.ts
@@ -476,6 +476,9 @@ const message = {
         goInstall: 'Ir para instalação',
         isDelete: 'Excluído',
         permission: 'Permissões',
+        format: 'Conjunto de Caracteres',
+        collation: 'Collation',
+        collationHelper: 'Se vazio, use a collation padrão do conjunto de caracteres {0}',
         permissionForIP: 'IP',
         permissionAll: 'Todos (% de)',
         localhostHelper:
diff --git a/frontend/src/lang/modules/ru.ts b/frontend/src/lang/modules/ru.ts
@@ -473,6 +473,9 @@ const message = {
         goInstall: 'Установить',
         isDelete: 'Удалено',
         permission: 'Разрешения',
+        format: 'Набор Символов',
+        collation: 'Сопоставление',
+        collationHelper: 'Если пусто, используйте сопоставление по умолчанию для набора символов {0}',
         permissionForIP: 'IP',
         permissionAll: 'Все (%)',
         databaseConnInfo: 'Информация о подключении',
diff --git a/frontend/src/lang/modules/tr.ts b/frontend/src/lang/modules/tr.ts
@@ -484,6 +484,9 @@ const message = {
         goInstall: 'Yüklemeye git',
         isDelete: 'Silindi',
         permission: 'İzinleri değiştir',
+        format: 'Karakter Seti',
+        collation: 'Karşılaştırma',
+        collationHelper: 'Boşsa, {0} karakter setinin varsayılan karşılaştırmasını kullanın',
         permissionForIP: 'IP',
         permissionAll: 'Tümü(%)',
         localhostHelper:
diff --git a/frontend/src/lang/modules/zh-Hant.ts b/frontend/src/lang/modules/zh-Hant.ts
@@ -467,6 +467,9 @@ const message = {
         goInstall: '去應用商店安裝',
         isDelete: '已刪除',
         permission: '權限',
+        format: '字元集',
+        collation: '排序規則',
+        collationHelper: '為空則使用 {0} 字元集的預設排序規則',
         permissionForIP: '指定 IP',
         permissionAll: '所有人(%)',
         localhostHelper: '將容器部署的資料庫權限配置為"localhost"會導致容器外部無法存取，請謹慎選擇！',
diff --git a/frontend/src/lang/modules/zh.ts b/frontend/src/lang/modules/zh.ts
diff --git a/frontend/src/views/database/mysql/create/index.vue b/frontend/src/views/database/mysql/create/index.vue
