Skip to content

Commit 5a36566

Browse files
SlachSlach
committed
can't reproduce behavior, add testcase to fix #871, change list format
Signed-off-by: Slach <bloodjazman@gmail.com>
1 parent 4df077c commit 5a36566

File tree

3 files changed

+68
-28
lines changed

3 files changed

+68
-28
lines changed

‎pkg/backup/list.go‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,8 @@ func printBackupsRemote(w io.Writer, backupList []storage.Backup, format string)
4848
}
4949
fmt.Println(backupList[len(backupList)-2].BackupName)
5050
case "all", "":
51-
// if len(backupList) == 0 {
52-
// fmt.Println("no backups found")
53-
// }
5451
for _, backup := range backupList {
55-
size := utils.FormatBytes(backup.GetFullSize())
52+
size := fmt.Sprintf("all:%s,data:%s,arch:%s,obj:%s,meta:%s,rbac:%s,conf:%s", utils.FormatBytes(backup.GetFullSize()), utils.FormatBytes(backup.DataSize), utils.FormatBytes(backup.CompressedSize), utils.FormatBytes(backup.MetadataSize), utils.FormatBytes(backup.ObjectDiskSize), utils.FormatBytes(backup.RBACSize), utils.FormatBytes(backup.ConfigSize))
5653
description := backup.DataFormat
5754
uploadDate := backup.UploadDate.Format("02/01/2006 15:04:05")
5855
if backup.Tags != "" {
@@ -66,7 +63,7 @@ func printBackupsRemote(w io.Writer, backupList []storage.Backup, format string)
6663
description = backup.Broken
6764
size = "???"
6865
}
69-
if bytes, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", backup.BackupName, size, uploadDate, "remote", required, description); err != nil {
66+
if bytes, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", backup.BackupName, uploadDate, "remote", required, size, description); err != nil {
7067
log.Error().Msgf("fmt.Fprintf write %d bytes return error: %v", bytes, err)
7168
}
7269
}
@@ -94,7 +91,7 @@ func printBackupsLocal(ctx context.Context, w io.Writer, backupList []LocalBacku
9491
case <-ctx.Done():
9592
return ctx.Err()
9693
default:
97-
size := utils.FormatBytes(backup.GetFullSize())
94+
size := fmt.Sprintf("all:%s,data:%s,arch:%s,obj:%s,meta:%s,rbac:%s,conf:%s", utils.FormatBytes(backup.GetFullSize()), utils.FormatBytes(backup.DataSize), utils.FormatBytes(backup.CompressedSize), utils.FormatBytes(backup.MetadataSize), utils.FormatBytes(backup.ObjectDiskSize), utils.FormatBytes(backup.RBACSize), utils.FormatBytes(backup.ConfigSize))
9895
description := backup.DataFormat
9996
if backup.Tags != "" {
10097
if description != "" {
@@ -111,7 +108,7 @@ func printBackupsLocal(ctx context.Context, w io.Writer, backupList []LocalBacku
111108
description = backup.Broken
112109
size = "???"
113110
}
114-
if bytes, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", backup.BackupName, size, creationDate, "local", required, description); err != nil {
111+
if bytes, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", backup.BackupName, creationDate, "local", required, size, description); err != nil {
115112
log.Error().Msgf("fmt.Fprintf write %d bytes return error: %v", bytes, err)
116113
}
117114
}

‎pkg/server/server.go‎

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -766,6 +766,12 @@ func (api *APIServer) httpListHandler(w http.ResponseWriter, r *http.Request) {
766766
Name string `json:"name"`
767767
Created string `json:"created"`
768768
Size uint64 `json:"size,omitempty"`
769+
DataSize uint64 `json:"data_size,omitempty"`
770+
ObjectDiskSize uint64 `json:"object_disk_size,omitempty"`
771+
MetadataSize uint64 `json:"metadata_size"`
772+
RBACSize uint64 `json:"rbac_size,omitempty"`
773+
ConfigSize uint64 `json:"config_size,omitempty"`
774+
CompressedSize uint64 `json:"compressed_size,omitempty"`
769775
Location string `json:"location"`
770776
RequiredBackup string `json:"required"`
771777
Desc string `json:"desc"`
@@ -807,6 +813,12 @@ func (api *APIServer) httpListHandler(w http.ResponseWriter, r *http.Request) {
807813
Name: item.BackupName,
808814
Created: item.CreationDate.Format(common.TimeFormat),
809815
Size: item.GetFullSize(),
816+
DataSize: item.DataSize,
817+
ObjectDiskSize: item.ObjectDiskSize,
818+
MetadataSize: item.MetadataSize,
819+
RBACSize: item.RBACSize,
820+
ConfigSize: item.ConfigSize,
821+
CompressedSize: item.CompressedSize,
810822
Location: "local",
811823
RequiredBackup: item.RequiredBackup,
812824
Desc: description,
@@ -821,25 +833,31 @@ func (api *APIServer) httpListHandler(w http.ResponseWriter, r *http.Request) {
821833
api.writeError(w, http.StatusInternalServerError, "list", err)
822834
return
823835
}
824-
for i, b := range remoteBackups {
825-
description := b.DataFormat
826-
if b.Broken != "" {
827-
description = b.Broken
836+
for i, item := range remoteBackups {
837+
description := item.DataFormat
838+
if item.Broken != "" {
839+
description = item.Broken
828840
brokenBackups++
829841
}
830-
if b.Tags != "" {
842+
if item.Tags != "" {
831843
if description != "" {
832844
description += ", "
833845
}
834-
description += b.Tags
846+
description += item.Tags
835847
}
836-
fullSize := b.GetFullSize()
848+
fullSize := item.GetFullSize()
837849
backupsJSON = append(backupsJSON, backupJSON{
838-
Name: b.BackupName,
839-
Created: b.CreationDate.Format(common.TimeFormat),
850+
Name: item.BackupName,
851+
Created: item.CreationDate.Format(common.TimeFormat),
840852
Size: fullSize,
853+
DataSize: item.DataSize,
854+
ObjectDiskSize: item.ObjectDiskSize,
855+
MetadataSize: item.MetadataSize,
856+
RBACSize: item.RBACSize,
857+
ConfigSize: item.ConfigSize,
858+
CompressedSize: item.CompressedSize,
841859
Location: "remote",
842-
RequiredBackup: b.RequiredBackup,
860+
RequiredBackup: item.RequiredBackup,
843861
Desc: description,
844862
})
845863
if i == len(remoteBackups)-1 {
@@ -1740,7 +1758,7 @@ func (api *APIServer) CreateIntegrationTables() error {
17401758
if err := ch.CreateTable(clickhouse.Table{Database: "system", Name: "backup_actions"}, query, true, false, "", 0, defaultDataPath); err != nil {
17411759
return err
17421760
}
1743-
query = fmt.Sprintf("CREATE TABLE system.backup_list (name String, created DateTime, size Int64, location String, required String, desc String) ENGINE=URL('%s://%s:%s/backup/list%s', JSONEachRow) %s", schema, host, port, auth, settings)
1761+
query = fmt.Sprintf("CREATE TABLE system.backup_list (name String, created DateTime, size UInt64, data_size UInt64, object_disk_size UInt64,metadata_size UInt64,rbac_size UInt64,config_size UInt64, compressed_size UInt64, location String, required String, desc String) ENGINE=URL('%s://%s:%s/backup/list%s', JSONEachRow) %s", schema, host, port, auth, settings)
17441762
if err := ch.CreateTable(clickhouse.Table{Database: "system", Name: "backup_list"}, query, true, false, "", 0, defaultDataPath); err != nil {
17451763
return err
17461764
}

‎test/integration/integration_test.go‎

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2619,12 +2619,11 @@ func (env *TestEnvironment) runMainIntegrationScenario(t *testing.T, remoteStora
26192619
var err error
26202620
r := require.New(t)
26212621
env.connectWithWait(r, 500*time.Millisecond, 1500*time.Millisecond, 3*time.Minute)
2622-
// test for specified partitions backup
2623-
testBackupSpecifiedPartitions(t, r, env, remoteStorageType, backupConfig)
26242622

26252623
// main test scenario
26262624
fullBackupName := fmt.Sprintf("%s_full_%d", t.Name(), rand.Int())
26272625
incrementBackupName := fmt.Sprintf("%s_increment_%d", t.Name(), rand.Int())
2626+
incrementBackupNameEmpty := fmt.Sprintf("%s_incrementEmpty_%d", t.Name(), rand.Int())
26282627
incrementBackupName2 := fmt.Sprintf("%s_increment2_%d", t.Name(), rand.Int())
26292628
databaseList := []string{dbNameOrdinary, dbNameAtomic, dbNameMySQL, dbNamePostgreSQL, Issue331Issue1091Atomic, Issue331Issue1091Ordinary}
26302629
tablesPattern := fmt.Sprintf("*_%s.*", t.Name())
@@ -2633,17 +2632,39 @@ func (env *TestEnvironment) runMainIntegrationScenario(t *testing.T, remoteStora
26332632
createAllTypesOfObjectTables := !strings.Contains(remoteStorageType, "CUSTOM")
26342633
testData := generateTestData(t, r, env, remoteStorageType, createAllTypesOfObjectTables, defaultTestData)
26352634

2636-
log.Debug().Msg("Create backup")
2635+
log.Debug().Msg("Create full backup")
26372636
createCmd := "clickhouse-backup -c /etc/clickhouse-backup/" + backupConfig + " create --resume --tables=" + tablesPattern + " " + fullBackupName
26382637
env.checkResumeAlreadyProcessed(createCmd, fullBackupName, "create", r, remoteStorageType)
26392638

2640-
incrementData := generateIncrementTestData(t, r, env, remoteStorageType, createAllTypesOfObjectTables, defaultIncrementData, 1)
2641-
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "create", "--tables", tablesPattern, incrementBackupName)
2642-
2643-
log.Debug().Msg("Upload full")
2639+
log.Debug().Msg("Upload full backup")
26442640
uploadCmd := fmt.Sprintf("%s_COMPRESSION_FORMAT=zstd CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/%s clickhouse-backup upload --resume %s", remoteStorageType, backupConfig, fullBackupName)
26452641
env.checkResumeAlreadyProcessed(uploadCmd, fullBackupName, "upload", r, remoteStorageType)
26462642

2643+
// https://github.com/Altinity/clickhouse-backup/issues/871
2644+
log.Debug().Msg("Create+upload incrementEmpty without data")
2645+
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "create", "--tables", tablesPattern, "--diff-from-remote", fullBackupName, incrementBackupNameEmpty)
2646+
out, err = env.DockerExecOut("clickhouse-backup", "bash", "-ec", "clickhouse-backup -c /etc/clickhouse-backup/"+backupConfig+" list local | grep "+incrementBackupNameEmpty)
2647+
r.NoError(err)
2648+
r.Contains(out, "+"+fullBackupName)
2649+
r.Contains(out, incrementBackupNameEmpty)
2650+
r.Contains(out, "data:0B")
2651+
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "upload", "--env", "BACKUPS_TO_KEEP_REMOTE=1", "--env", "BACKUPS_TO_KEEP_REMOTE=2", "--env", "ALLOW_EMPTY_BACKUPS=1", "--diff-from-remote", fullBackupName, incrementBackupNameEmpty)
2652+
out, err = env.DockerExecOut("clickhouse-backup", "bash", "-ec", "clickhouse-backup -c /etc/clickhouse-backup/"+backupConfig+" list remote | grep '^"+fullBackupName+"'")
2653+
r.NoError(err)
2654+
r.Contains(out, fullBackupName)
2655+
r.NotContains(out, "data:0B")
2656+
out, err = env.DockerExecOut("clickhouse-backup", "bash", "-ec", "clickhouse-backup -c /etc/clickhouse-backup/"+backupConfig+" list remote | grep "+incrementBackupNameEmpty)
2657+
r.NoError(err)
2658+
r.Contains(out, "+"+fullBackupName)
2659+
r.Contains(out, incrementBackupNameEmpty)
2660+
r.Contains(out, "data:0B")
2661+
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "remote", incrementBackupNameEmpty)
2662+
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", incrementBackupNameEmpty)
2663+
2664+
log.Debug().Msg("Create increment1 with data")
2665+
incrementData := generateIncrementTestData(t, r, env, remoteStorageType, createAllTypesOfObjectTables, defaultIncrementData, 1)
2666+
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "create", "--tables", tablesPattern, incrementBackupName)
2667+
26472668
// https://github.com/Altinity/clickhouse-backup/pull/900
26482669
if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.8") >= 0 {
26492670
log.Debug().Msg("create --diff-from-remote backup")
@@ -2667,9 +2688,9 @@ func (env *TestEnvironment) runMainIntegrationScenario(t *testing.T, remoteStora
26672688
log.Debug().Msg("Delete backup")
26682689
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", fullBackupName)
26692690
env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", incrementBackupName)
2670-
out, err = env.DockerExecOut("clickhouse-backup", "bash", "-ce", "ls -lha "+backupDir+" | grep "+t.Name())
2671-
r.NotNil(err)
2672-
r.Equal("", strings.Trim(out, " \t\r\n"), "expect '0' backup exists in backup directory")
2691+
out, err = env.DockerExecOut("clickhouse-backup", "bash", "-ce", "ls -lha "+backupDir)
2692+
r.NoError(err)
2693+
r.NotContains(strings.Trim(out, " \t\r\n"), t.Name(), "expect no backup exists in backup directory")
26732694

26742695
dropDatabasesFromTestDataDataSet(t, r, env, databaseList)
26752696

@@ -2740,6 +2761,10 @@ func (env *TestEnvironment) runMainIntegrationScenario(t *testing.T, remoteStora
27402761
fullCleanup(t, r, env, []string{incrementBackupName}, []string{"local"}, nil, true, false, backupConfig)
27412762
fullCleanup(t, r, env, []string{fullBackupName, incrementBackupName}, []string{"remote"}, databaseList, true, true, backupConfig)
27422763
replaceStorageDiskNameForReBalance(r, env, remoteStorageType, true)
2764+
2765+
// test for specified partitions backup
2766+
testBackupSpecifiedPartitions(t, r, env, remoteStorageType, backupConfig)
2767+
27432768
env.checkObjectStorageIsEmpty(t, r, remoteStorageType)
27442769
}
27452770

0 commit comments

Comments
 (0)