fix slowdown and corner cases, for incremental backup creation, affected 2.6.30+, fix #1249

Signed-off-by: Slach <[email protected]>

Author: Slach

ChangeLog.md

@@ -1,3 +1,7 @@
+# v2.6.37
+BUG FIXES
+- fix slowdown and corner cases, for incremental backup creation, affected 2.6.30+, fix https://github.com/Altinity/clickhouse-backup/issues/1249
+
 # v2.6.36
 IMPROVEMENTS
 - add `GCS_SA_EMAIL` option to support service account authorization, fix [1246](https://github.com/Altinity/clickhouse-backup/pull/1246) thanks @kamushadenes

pkg/filesystemhelper/filesystemhelper.go

@@ -392,24 +392,27 @@ func IsSkipProjections(skipProjections []string, relativePath string) bool {
 func addRequiredPartIfNotExists(parts []metadata.Part, relativePath string, tableDiffFromRemote metadata.TableMetadata, disk clickhouse.Disk) ([]metadata.Part, bool, bool) {
 	isRequiredPartFound := false
 	exists := false
-	for _, p := range parts {
-		if p.Name == relativePath || strings.HasPrefix(relativePath, p.Name+"/") {
-			exists = true
+	for _, diffPart := range tableDiffFromRemote.Parts[disk.Name] {
+		if diffPart.Name == relativePath || strings.HasPrefix(relativePath, diffPart.Name+"/") {
+			isRequiredPartFound = true
 			break
 		}
-		for _, diffPart := range tableDiffFromRemote.Parts[disk.Name] {
-			if diffPart.Name == relativePath || strings.HasPrefix(relativePath, diffPart.Name+"/") {
-				isRequiredPartFound = true
+	}
+	if isRequiredPartFound {
+		for _, p := range parts {
+			if p.Name == relativePath || strings.HasPrefix(relativePath, p.Name+"/") {
+				exists = true
+				break
 			}
 		}
+		if !exists {
+			parts = append(parts, metadata.Part{
+				Name:     relativePath,
+				Required: true,
+			})
+		}
 	}
-	// we use relativePath here cause during Walk, directory will walked first
-	if !exists && isRequiredPartFound {
-		parts = append(parts, metadata.Part{
-			Name:     relativePath,
-			Required: true,
-		})
-	}
+
 	return parts, isRequiredPartFound, exists
 }