Skip to content

Commit 8774f64

Browse files
SlachSlach
committed
fix corner case for --restore-database-mapping, #820 (comment) 
1 parent b8b25e7 commit 8774f64

File tree

4 files changed

+38
-26
lines changed

4 files changed

+38
-26
lines changed

‎ChangeLog.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
# v2.6.13
2+
BUG FIXES
3+
- fix corner case for `--restore-database-mapping`, fix [820](https://github.com/Altinity/clickhouse-backup/issues/820#issuecomment-2773501803)
4+
15
# v2.6.12
26
BUG FIXES
37
- fix corner case for wrong path for state file during `ResumeOperationsAfterRestart` fix [1126](https://github.com/Altinity/clickhouse-backup/issues/1126)

‎pkg/backup/restore.go‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -68,11 +68,11 @@ func (b *Backuper) Restore(backupName, tablePattern string, databaseMapping, tab
6868
}
6969
defer b.ch.Close()
7070

71-
clickHouseVersion, versionErr := b.ch.GetVersion(ctx)
71+
version, versionErr := b.ch.GetVersion(ctx)
7272
if versionErr != nil {
7373
return versionErr
7474
}
75-
if clickHouseVersion < 24003000 && skipProjections != nil && len(skipProjections) > 0 {
75+
if version < 24003000 && skipProjections != nil && len(skipProjections) > 0 {
7676
return fmt.Errorf("backup with skip-projections can restore only in 24.3+")
7777
}
7878
// https://github.com/Altinity/clickhouse-backup/issues/868
@@ -88,10 +88,6 @@ func (b *Backuper) Restore(backupName, tablePattern string, databaseMapping, tab
8888
if err != nil {
8989
return err
9090
}
91-
version, err := b.ch.GetVersion(ctx)
92-
if err != nil {
93-
return err
94-
}
9591
b.DefaultDataPath, err = b.ch.GetDefaultPath(disks)
9692
if err != nil {
9793
log.Warn().Msgf("%v", err)

‎pkg/backup/table_pattern.go‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,7 @@ func (b *Backuper) enrichTablePatternsByInnerDependencies(metadataPath string, t
326326
return tablePatterns, nil
327327
}
328328

329-
var queryRE = regexp.MustCompile(`(?m)^(CREATE|ATTACH) (TABLE|VIEW|LIVE VIEW|MATERIALIZED VIEW|DICTIONARY|FUNCTION) (\x60?)([^\s\x60.]*)(\x60?)\.\x60?([^\s\x60.]*)\x60?( UUID '[^']+')?(?:( TO )(\x60?)([^\s\x60.]*)(\x60?)(\.)(\x60?)([^\s\x60.]*)(\x60?))?(?:(.+FROM )(\x60?)([^\s\x60.]*)(\x60?)(\.)(\x60?)([^\s\x60.]*)(\x60?))?`)
329+
var queryRE = regexp.MustCompile(`(?m)^(CREATE|ATTACH) (TABLE|VIEW|LIVE VIEW|MATERIALIZED VIEW|DICTIONARY|FUNCTION) (\x60?)([^\s\x60.]*)(\x60?)\.(\x60?)([^\s\x60.]*)(\x60?)( UUID '[^']+')?(?:( TO )(\x60?)([^\s\x60.]*)(\x60?)(\.)(\x60?)([^\s\x60.]*)(\x60?))?(?:(.+FROM )(\x60?)([^\s\x60.]*)(\x60?)(\.)(\x60?)([^\s\x60.]*)(\x60?))?`)
330330
var createOrAttachRE = regexp.MustCompile(`(?m)^(CREATE|ATTACH)`)
331331
var uuidRE = regexp.MustCompile(`UUID '([a-f\d\-]+)'`)
332332

@@ -361,10 +361,10 @@ func changeTableQueryToAdjustDatabaseMapping(originTables *ListOfTables, dbMapRu
361361
if !usualIdentifier.MatchString(createTargetDb) && !strings.Contains(matches[0][3], "`") {
362362
createTargetDb = "`" + createTargetDb + "`"
363363
}
364-
toClauseTargetDb := setMatchedDb(matches[0][10], matches[0][9])
365-
fromClauseTargetDb := setMatchedDb(matches[0][18], matches[0][17])
364+
toClauseTargetDb := setMatchedDb(matches[0][12], matches[0][11])
365+
fromClauseTargetDb := setMatchedDb(matches[0][20], matches[0][19])
366366
// matching CREATE|ATTACH ... TO .. SELECT ... FROM ... command
367-
substitution = fmt.Sprintf("${1} ${2} ${3}%v${5}.${6}${7}${8}${9}%v${11}${12}${13}${14}${15}${16}${17}%v${19}${20}${21}${22}${23}", createTargetDb, toClauseTargetDb, fromClauseTargetDb)
367+
substitution = fmt.Sprintf("${1} ${2} ${3}%v${5}.${6}${7}${8}${9}${10}${11}%v${13}${14}${15}${16}${17}${18}${19}%v${21}${22}${23}${24}${25}", createTargetDb, toClauseTargetDb, fromClauseTargetDb)
368368
} else {
369369
if originTable.Query == "" {
370370
continue
@@ -413,26 +413,28 @@ func changeTableQueryToAdjustTableMapping(originTables *ListOfTables, tableMapRu
413413

414414
if createOrAttachRE.MatchString(originTable.Query) {
415415
matches := queryRE.FindAllStringSubmatch(originTable.Query, -1)
416-
if len(matches) == 0 || len(matches[0]) < 7 || matches[0][6] != originTable.Table {
416+
if len(matches) == 0 || len(matches[0]) < 8 || matches[0][7] != originTable.Table {
417417
return fmt.Errorf("invalid SQL: %s\nRE: `%s`\nmatches=%#v for restore-table-mapping[%s]=%s", originTable.Query, queryRE.String(), matches, originTable.Table, targetTable)
418418
}
419-
setMatchedDb := func(clauseTargetTable string) string {
419+
setMatchedTable := func(clauseTargetTable, beforeQuote string) string {
420420
if clauseMappedTable, isClauseMapped := tableMapRule[clauseTargetTable]; isClauseMapped {
421421
clauseTargetTable = clauseMappedTable
422-
if !usualIdentifier.MatchString(clauseTargetTable) {
422+
// https://github.com/Altinity/clickhouse-backup/issues/820#issuecomment-2773501803
423+
if !usualIdentifier.MatchString(clauseTargetTable) && !strings.Contains(beforeQuote, "`") {
423424
clauseTargetTable = "`" + clauseTargetTable + "`"
424425
}
425426
}
426427
return clauseTargetTable
427428
}
428429
createTargetTable := targetTable
429-
if !usualIdentifier.MatchString(createTargetTable) {
430+
// https://github.com/Altinity/clickhouse-backup/issues/820#issuecomment-2773501803
431+
if !usualIdentifier.MatchString(createTargetTable) && !strings.Contains(matches[0][6], "`") {
430432
createTargetTable = "`" + createTargetTable + "`"
431433
}
432-
toClauseTargetTable := setMatchedDb(matches[0][14])
433-
fromClauseTargetTable := setMatchedDb(matches[0][22])
434+
toClauseTargetTable := setMatchedTable(matches[0][16], matches[0][15])
435+
fromClauseTargetTable := setMatchedTable(matches[0][24], matches[0][23])
434436
// matching CREATE|ATTACH ... TO .. SELECT ... FROM ... command
435-
substitution = fmt.Sprintf("${1} ${2} ${3}${4}${5}.%v${7}${8}${9}${10}${11}${12}${13}%v${15}${16}${17}${18}${19}${20}${21}%v${23}", createTargetTable, toClauseTargetTable, fromClauseTargetTable)
437+
substitution = fmt.Sprintf("${1} ${2} ${3}${4}${5}.${6}%v${8}${9}${10}${11}${12}${13}${14}${15}%v${17}${18}${19}${20}${21}${22}${23}%v${25}", createTargetTable, toClauseTargetTable, fromClauseTargetTable)
436438
} else {
437439
if originTable.Query == "" {
438440
continue

‎test/integration/integration_test.go‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2895,7 +2895,7 @@ func TestRestoreMapping(t *testing.T) {
28952895

28962896
env.queryWithNoError(r, "CREATE DATABASE `database-1`")
28972897
env.queryWithNoError(r, "CREATE TABLE `database-1`.t1 (dt DateTime, v UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/database-1/t1','{replica}') PARTITION BY v % 10 ORDER BY dt")
2898-
env.queryWithNoError(r, "CREATE TABLE `database-1`.d1 AS `database-1`.t1 ENGINE=Distributed('{cluster}', 'database-1', 't1')")
2898+
env.queryWithNoError(r, "CREATE TABLE `database-1`.`t-d1` AS `database-1`.t1 ENGINE=Distributed('{cluster}', 'database-1', 't1')")
28992899
if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "22.3") < 0 {
29002900
env.queryWithNoError(r, "CREATE TABLE `database-1`.t2 AS `database-1`.t1 ENGINE=ReplicatedMergeTree('/clickhouse/tables/database-1/t2','{replica}') PARTITION BY toYYYYMM(dt) ORDER BY dt")
29012901
} else {
@@ -2909,26 +2909,36 @@ func TestRestoreMapping(t *testing.T) {
29092909
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "create", testBackupName)
29102910

29112911
log.Debug().Msg("Restore schema")
2912-
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "restore", "--schema", "--rm", "--restore-database-mapping", "database-1:database-2", "--restore-table-mapping", "t1:t3,t2:t4,d1:d2,mv1:mv2,v1:v2", "--tables", "database-1.*", testBackupName)
2912+
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "restore", "--schema", "--rm", "--restore-database-mapping", "database-1:database-2", "--restore-table-mapping", "t1:t3,t2:t4,t-d1:t-d2,mv1:mv2,v1:v2", "--tables", "database-1.*", testBackupName)
29132913

29142914
log.Debug().Msg("Check result database-1")
29152915
env.queryWithNoError(r, "INSERT INTO `database-1`.t1 SELECT '2023-01-01 00:00:00', number FROM numbers(10)")
29162916
env.checkCount(r, 1, 20, "SELECT count() FROM `database-1`.t1")
29172917
env.checkCount(r, 1, 20, "SELECT count() FROM `database-1`.t2")
2918-
env.checkCount(r, 1, 20, "SELECT count() FROM `database-1`.d1")
2918+
env.checkCount(r, 1, 20, "SELECT count() FROM `database-1`.`t-d1`")
29192919
env.checkCount(r, 1, 20, "SELECT count() FROM `database-1`.mv1")
29202920
env.checkCount(r, 1, 20, "SELECT count() FROM `database-1`.v1")
29212921

29222922
log.Debug().Msg("Drop database-1")
29232923
r.NoError(env.dropDatabase("database-1", false))
29242924

2925-
log.Debug().Msg("Restore data")
2926-
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "restore", "--data", "--restore-database-mapping", "database-1:database-2", "--restore-table-mapping", "t1:t3,t2:t4,d1:d2,mv1:mv2,v1:v2", "--tables", "database-1.*", testBackupName)
2925+
log.Debug().Msg("Restore data only --restore-database-mappings")
2926+
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "restore", "--rm", "--restore-database-mapping", "database-1:database-2", testBackupName)
2927+
2928+
log.Debug().Msg("Check result database-2 without table mapping")
2929+
env.checkCount(r, 1, 10, "SELECT count() FROM `database-2`.t1")
2930+
env.checkCount(r, 1, 10, "SELECT count() FROM `database-2`.t2")
2931+
env.checkCount(r, 1, 10, "SELECT count() FROM `database-2`.`t-d1`")
2932+
env.checkCount(r, 1, 10, "SELECT count() FROM `database-2`.mv1")
2933+
env.checkCount(r, 1, 10, "SELECT count() FROM `database-2`.v1")
2934+
2935+
log.Debug().Msg("Restore data only --restore-table-mappings+--restore-database-mappings")
2936+
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "restore", "--data", "--restore-database-mapping", "database-1:database-2", "--restore-table-mapping", "t1:t3,t2:t4,t-d1:t-d2,mv1:mv2,v1:v2", "--tables", "database-1.*", testBackupName)
29272937

29282938
log.Debug().Msg("Check result database-2")
29292939
env.checkCount(r, 1, 10, "SELECT count() FROM `database-2`.t3")
29302940
env.checkCount(r, 1, 10, "SELECT count() FROM `database-2`.t4")
2931-
env.checkCount(r, 1, 10, "SELECT count() FROM `database-2`.d2")
2941+
env.checkCount(r, 1, 10, "SELECT count() FROM `database-2`.`t-d2`")
29322942
env.checkCount(r, 1, 10, "SELECT count() FROM `database-2`.mv2")
29332943
env.checkCount(r, 1, 10, "SELECT count() FROM `database-2`.v2")
29342944

@@ -2939,14 +2949,14 @@ func TestRestoreMapping(t *testing.T) {
29392949
r.NoError(env.dropDatabase("database-2", false))
29402950

29412951
log.Debug().Msg("Restore data with partitions")
2942-
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "restore", "--restore-database-mapping", "database-1:database-2", "--restore-table-mapping", "t1:t3,t2:t4,d1:d2,mv1:mv2,v1:v2", "--partitions", "3", "--partitions", "database-1.t2:202201", "--tables", "database-1.*", testBackupName)
2952+
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "restore", "--restore-database-mapping", "database-1:database-2", "--restore-table-mapping", "t1:t3,t2:t4,t-d1:t-d2,mv1:mv2,v1:v2", "--partitions", "3", "--partitions", "database-1.t2:202201", "--tables", "database-1.*", testBackupName)
29432953

29442954
log.Debug().Msg("Check result database-2 after restore with partitions")
29452955
// t1->t3 restored only 1 partition with name 3 partition with 1 rows
2946-
// t1->t3 restored only 1 partition with name 3 partition with 10 rows
29472956
env.checkCount(r, 1, 1, "SELECT count() FROM `database-2`.t3")
2957+
// t2->t4 restored only 1 partition with name 3 partition with 10 rows
29482958
env.checkCount(r, 1, 10, "SELECT count() FROM `database-2`.t4")
2949-
env.checkCount(r, 1, 1, "SELECT count() FROM `database-2`.d2")
2959+
env.checkCount(r, 1, 1, "SELECT count() FROM `database-2`.`t-d2`")
29502960
env.checkCount(r, 1, 10, "SELECT count() FROM `database-2`.mv2")
29512961
env.checkCount(r, 1, 1, "SELECT count() FROM `database-2`.v2")
29522962

0 commit comments

Comments
 (0)