polishing after merge #1248, thanks @KimDoKy

Signed-off-by: Slach <[email protected]>

Author: Slach

ChangeLog.md

@@ -2,6 +2,7 @@
 IMPROVEMENTS
 - add `GCS_SA_EMAIL` option to support service account authorization, fix [1246](https://github.com/Altinity/clickhouse-backup/pull/1246) thanks @kamushadenes
 - improve `--hardlink-exists-files` behavior, will look to exists local backups for the same part to avoid download unnecessary, fix [1244](https://github.com/Altinity/clickhouse-backup/issues/1244)
+- add `S3_CHUNK_SIZE` option to allow more flexible workload for S3-compatible remote storage, fix [1248](https://github.com/Altinity/clickhouse-backup/pull/1248) thanks @KimDoKy
 
 # v2.6.35
 BUG FIXES

pkg/storage/s3.go

@@ -262,13 +262,12 @@ func (s *S3) GetFileReaderWithLocalPath(ctx context.Context, key, localPath stri
 			// Use configured chunk size
 			partSize = s.Config.ChunkSize
 		} else {
-			partSize := remoteSize / s.Config.MaxPartsCount
+			partSize = remoteSize / s.Config.MaxPartsCount
 			if remoteSize%s.Config.MaxPartsCount > 0 {
 				partSize += max(1, (remoteSize%s.Config.MaxPartsCount)/s.Config.MaxPartsCount)
 			}
 		}
 		downloader.PartSize = AdjustValueByRange(partSize, 5*1024*1024, 5*1024*1024*1024)
-		log.Debug().Msgf("S3 Download PartSize: %d bytes (%.2f MB)", downloader.PartSize, float64(downloader.PartSize)/(1024*1024))
 
 		_, err = downloader.Download(ctx, writer, &s3.GetObjectInput{
 			Bucket: aws.String(s.Config.Bucket),
@@ -338,13 +337,12 @@ func (s *S3) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser, l
 	if s.Config.ChunkSize > 0 && (localSize+s.Config.ChunkSize-1)/s.Config.ChunkSize < 10000 {
 		partSize = s.Config.ChunkSize
 	} else {
-		partSize := localSize / s.Config.MaxPartsCount
+		partSize = localSize / s.Config.MaxPartsCount
 		if localSize%s.Config.MaxPartsCount > 0 {
 			partSize += max(1, (localSize%s.Config.MaxPartsCount)/s.Config.MaxPartsCount)
 		}
 	}
 	uploader.PartSize = AdjustValueByRange(partSize, 5*1024*1024, 5*1024*1024*1024)
-	log.Debug().Msgf("S3 Upload PartSize: %d bytes (%.2f MB)", uploader.PartSize, float64(uploader.PartSize)/(1024*1024))
 
 	_, err := uploader.Upload(ctx, &params)
 	return err
@@ -551,13 +549,12 @@ func (s *S3) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, d
 	if s.Config.ChunkSize > 0 && (srcSize+s.Config.ChunkSize-1)/s.Config.ChunkSize < 10000 {
 		partSize = s.Config.ChunkSize
 	} else {
-		partSize := srcSize / s.Config.MaxPartsCount
+		partSize = srcSize / s.Config.MaxPartsCount
 		if srcSize%s.Config.MaxPartsCount > 0 {
 			partSize += max(1, (srcSize%s.Config.MaxPartsCount)/s.Config.MaxPartsCount)
 		}
 	}
 	partSize = AdjustValueByRange(partSize, 128*1024*1024, 5*1024*1024*1024)
-	log.Debug().Msgf("S3 CopyObject PartSize: %d bytes (%.2f MB)", partSize, float64(partSize)/(1024*1024))
 
 	// Calculate the number of parts
 	numParts := (srcSize + partSize - 1) / partSize

test/integration/run.sh

@@ -105,7 +105,7 @@ if [[ "0" == "${TEST_FAILED}" ]]; then
   go tool covdata textfmt -i "${CUR_DIR}/_coverage_/" -o "${CUR_DIR}/_coverage_/coverage.out"
 fi
 
-if [[ "1" == "${CLEAN_AFTER:-0}" || "0" == "${TEST_FAILED}" ]]; then
+if [[ "0" == "${TEST_FAILED}" && "1" == "${CLEAN_AFTER:-1}" ]]; then
   pids=()
   for project in $(docker compose -f "${CUR_DIR}/${COMPOSE_FILE}" ls --all -q); do
     docker compose -f "${CUR_DIR}/${COMPOSE_FILE}" --project-name "${project}" --progress plain down --remove-orphans --volumes --timeout=1 &