Skip to content

Commit a3feff4

Browse files
SlachSlach
committed
fix --restore-database-mapping with special characters in source table, fix #820
Signed-off-by: Slach <bloodjazman@gmail.com>
1 parent 67f7e1b commit a3feff4

File tree

3 files changed

+35
-30
lines changed

3 files changed

+35
-30
lines changed

‎ChangeLog.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ IMPROVEMENTS
1414

1515
BUG FIXES
1616

17+
- fix `--restore-database-mapping` with special characters in source table, fix
18+
reopened [820](https://github.com/Altinity/clickhouse-backup/issues/820#issuecomment-2675628282),
19+
thanks @IvaskevychYuriy
1720
- add workaround for `metdata_path` field change behavior in 25.1+, look details
1821
in https://github.com/ClickHouse/ClickHouse/issues/76546,
1922
fix [1093](https://github.com/Altinity/clickhouse-backup/issues/1093)

‎pkg/backup/table_pattern.go‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -322,21 +322,23 @@ func changeTableQueryToAdjustDatabaseMapping(originTables *ListOfTables, dbMapRu
322322
if matches[0][4] != originTable.Database {
323323
return fmt.Errorf("invalid SQL: %s for restore-database-mapping[%s]=%s", originTable.Query, originTable.Database, targetDB)
324324
}
325-
setMatchedDb := func(clauseTargetDb string) string {
325+
setMatchedDb := func(clauseTargetDb, beforeQuote string) string {
326326
if clauseMappedDb, isClauseMapped := dbMapRule[clauseTargetDb]; isClauseMapped {
327327
clauseTargetDb = clauseMappedDb
328-
if !usualIdentifier.MatchString(clauseTargetDb) {
328+
// https://github.com/Altinity/clickhouse-backup/issues/820
329+
if !usualIdentifier.MatchString(clauseTargetDb) && !strings.Contains(beforeQuote, "`") {
329330
clauseTargetDb = "`" + clauseTargetDb + "`"
330331
}
331332
}
332333
return clauseTargetDb
333334
}
334335
createTargetDb := targetDB
335-
if !usualIdentifier.MatchString(createTargetDb) {
336+
// https://github.com/Altinity/clickhouse-backup/issues/820
337+
if !usualIdentifier.MatchString(createTargetDb) && !strings.Contains(matches[0][3], "`") {
336338
createTargetDb = "`" + createTargetDb + "`"
337339
}
338-
toClauseTargetDb := setMatchedDb(matches[0][10])
339-
fromClauseTargetDb := setMatchedDb(matches[0][18])
340+
toClauseTargetDb := setMatchedDb(matches[0][10], matches[0][9])
341+
fromClauseTargetDb := setMatchedDb(matches[0][18], matches[0][17])
340342
// matching CREATE|ATTACH ... TO .. SELECT ... FROM ... command
341343
substitution = fmt.Sprintf("${1} ${2} ${3}%v${5}.${6}${7}${8}${9}%v${11}${12}${13}${14}${15}${16}${17}%v${19}${20}${21}${22}${23}", createTargetDb, toClauseTargetDb, fromClauseTargetDb)
342344
} else {
@@ -387,8 +389,8 @@ func changeTableQueryToAdjustTableMapping(originTables *ListOfTables, tableMapRu
387389

388390
if createOrAttachRE.MatchString(originTable.Query) {
389391
matches := queryRE.FindAllStringSubmatch(originTable.Query, -1)
390-
if matches[0][6] != originTable.Table {
391-
return fmt.Errorf("invalid SQL: %s for restore-table-mapping[%s]=%s", originTable.Query, originTable.Table, targetTable)
392+
if len(matches) == 0 || len(matches[0]) < 7 || matches[0][6] != originTable.Table {
393+
return fmt.Errorf("invalid SQL: %s\nRE: `%s`\nmatches=%#v for restore-table-mapping[%s]=%s", originTable.Query, queryRE.String(), matches, originTable.Table, targetTable)
392394
}
393395
setMatchedDb := func(clauseTargetTable string) string {
394396
if clauseMappedTable, isClauseMapped := tableMapRule[clauseTargetTable]; isClauseMapped {

‎test/integration/integration_test.go‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2469,40 +2469,40 @@ func TestRestoreMapping(t *testing.T) {
24692469
}
24702470

24712471
testBackupName := "test_restore_database_mapping"
2472-
databaseList := []string{"database1", "database-2"}
2472+
databaseList := []string{"database-1", "database-2"}
24732473
fullCleanup(t, r, env, []string{testBackupName}, []string{"local"}, databaseList, false, false, "config-database-mapping.yml")
24742474

2475-
env.queryWithNoError(r, "CREATE DATABASE database1")
2476-
env.queryWithNoError(r, "CREATE TABLE database1.t1 (dt DateTime, v UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/database1/t1','{replica}') PARTITION BY v % 10 ORDER BY dt")
2477-
env.queryWithNoError(r, "CREATE TABLE database1.d1 AS database1.t1 ENGINE=Distributed('{cluster}', 'database1', 't1')")
2475+
env.queryWithNoError(r, "CREATE DATABASE `database-1`")
2476+
env.queryWithNoError(r, "CREATE TABLE `database-1`.t1 (dt DateTime, v UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/database-1/t1','{replica}') PARTITION BY v % 10 ORDER BY dt")
2477+
env.queryWithNoError(r, "CREATE TABLE `database-1`.d1 AS `database-1`.t1 ENGINE=Distributed('{cluster}', 'database-1', 't1')")
24782478
if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "22.3") < 0 {
2479-
env.queryWithNoError(r, "CREATE TABLE database1.t2 AS database1.t1 ENGINE=ReplicatedMergeTree('/clickhouse/tables/database1/t2','{replica}') PARTITION BY toYYYYMM(dt) ORDER BY dt")
2479+
env.queryWithNoError(r, "CREATE TABLE `database-1`.t2 AS `database-1`.t1 ENGINE=ReplicatedMergeTree('/clickhouse/tables/database-1/t2','{replica}') PARTITION BY toYYYYMM(dt) ORDER BY dt")
24802480
} else {
2481-
env.queryWithNoError(r, "CREATE TABLE database1.t2 AS database1.t1 ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/{table}','{replica}') PARTITION BY toYYYYMM(dt) ORDER BY dt")
2481+
env.queryWithNoError(r, "CREATE TABLE `database-1`.t2 AS `database-1`.t1 ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/{table}','{replica}') PARTITION BY toYYYYMM(dt) ORDER BY dt")
24822482
}
2483-
env.queryWithNoError(r, "CREATE MATERIALIZED VIEW database1.mv1 TO database1.t2 AS SELECT * FROM database1.t1")
2484-
env.queryWithNoError(r, "CREATE VIEW database1.v1 AS SELECT * FROM database1.t1")
2485-
env.queryWithNoError(r, "INSERT INTO database1.t1 SELECT '2022-01-01 00:00:00', number FROM numbers(10)")
2483+
env.queryWithNoError(r, "CREATE MATERIALIZED VIEW `database-1`.mv1 TO `database-1`.t2 AS SELECT * FROM `database-1`.t1")
2484+
env.queryWithNoError(r, "CREATE VIEW `database-1`.v1 AS SELECT * FROM `database-1`.t1")
2485+
env.queryWithNoError(r, "INSERT INTO `database-1`.t1 SELECT '2022-01-01 00:00:00', number FROM numbers(10)")
24862486

24872487
log.Debug().Msg("Create backup")
24882488
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "create", testBackupName)
24892489

24902490
log.Debug().Msg("Restore schema")
2491-
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "restore", "--schema", "--rm", "--restore-database-mapping", "database1:database-2", "--restore-table-mapping", "t1:t3,t2:t4,d1:d2,mv1:mv2,v1:v2", "--tables", "database1.*", testBackupName)
2491+
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "restore", "--schema", "--rm", "--restore-database-mapping", "database-1:database-2", "--restore-table-mapping", "t1:t3,t2:t4,d1:d2,mv1:mv2,v1:v2", "--tables", "database-1.*", testBackupName)
24922492

2493-
log.Debug().Msg("Check result database1")
2494-
env.queryWithNoError(r, "INSERT INTO database1.t1 SELECT '2023-01-01 00:00:00', number FROM numbers(10)")
2495-
checkRecordset(1, 20, "SELECT count() FROM database1.t1")
2496-
checkRecordset(1, 20, "SELECT count() FROM database1.t2")
2497-
checkRecordset(1, 20, "SELECT count() FROM database1.d1")
2498-
checkRecordset(1, 20, "SELECT count() FROM database1.mv1")
2499-
checkRecordset(1, 20, "SELECT count() FROM database1.v1")
2493+
log.Debug().Msg("Check result database-1")
2494+
env.queryWithNoError(r, "INSERT INTO `database-1`.t1 SELECT '2023-01-01 00:00:00', number FROM numbers(10)")
2495+
checkRecordset(1, 20, "SELECT count() FROM `database-1`.t1")
2496+
checkRecordset(1, 20, "SELECT count() FROM `database-1`.t2")
2497+
checkRecordset(1, 20, "SELECT count() FROM `database-1`.d1")
2498+
checkRecordset(1, 20, "SELECT count() FROM `database-1`.mv1")
2499+
checkRecordset(1, 20, "SELECT count() FROM `database-1`.v1")
25002500

2501-
log.Debug().Msg("Drop database1")
2502-
r.NoError(env.dropDatabase("database1"))
2501+
log.Debug().Msg("Drop database-1")
2502+
r.NoError(env.dropDatabase("database-1"))
25032503

25042504
log.Debug().Msg("Restore data")
2505-
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "restore", "--data", "--restore-database-mapping", "database1:database-2", "--restore-table-mapping", "t1:t3,t2:t4,d1:d2,mv1:mv2,v1:v2", "--tables", "database1.*", testBackupName)
2505+
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "restore", "--data", "--restore-database-mapping", "database-1:database-2", "--restore-table-mapping", "t1:t3,t2:t4,d1:d2,mv1:mv2,v1:v2", "--tables", "database-1.*", testBackupName)
25062506

25072507
log.Debug().Msg("Check result database-2")
25082508
checkRecordset(1, 10, "SELECT count() FROM `database-2`.t3")
@@ -2511,14 +2511,14 @@ func TestRestoreMapping(t *testing.T) {
25112511
checkRecordset(1, 10, "SELECT count() FROM `database-2`.mv2")
25122512
checkRecordset(1, 10, "SELECT count() FROM `database-2`.v2")
25132513

2514-
log.Debug().Msg("Check database1 not exists")
2515-
checkRecordset(1, 0, "SELECT count() FROM system.databases WHERE name='database1' SETTINGS empty_result_for_aggregation_by_empty_set=0")
2514+
log.Debug().Msg("Check database-1 not exists")
2515+
checkRecordset(1, 0, "SELECT count() FROM system.databases WHERE name='database-1' SETTINGS empty_result_for_aggregation_by_empty_set=0")
25162516

25172517
log.Debug().Msg("Drop database2")
25182518
r.NoError(env.dropDatabase("database2"))
25192519

25202520
log.Debug().Msg("Restore data with partitions")
2521-
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "restore", "--restore-database-mapping", "database1:database-2", "--restore-table-mapping", "t1:t3,t2:t4,d1:d2,mv1:mv2,v1:v2", "--partitions", "3", "--partitions", "database1.t2:202201", "--tables", "database1.*", testBackupName)
2521+
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "restore", "--restore-database-mapping", "database-1:database-2", "--restore-table-mapping", "t1:t3,t2:t4,d1:d2,mv1:mv2,v1:v2", "--partitions", "3", "--partitions", "database-1.t2:202201", "--tables", "database-1.*", testBackupName)
25222522

25232523
log.Debug().Msg("Check result database-2 after restore with partitions")
25242524
// t1->t3 restored only 1 partition with name 3 partition with 1 rows

0 commit comments

Comments
 (0)