Merge #148794

148794: cli(debug.zip): update retry configuration for GCS upload of logs r=aa-joshi a=aa-joshi

Previously, we are observing intermittent "EOF" errors during uploading logs to GCS. This patch updates the retry configuration for debug.zip log upload for rehydration. We are using "RetryAlways" policy in GCS client so that it will retry on all types errors.

Epic: None
Fixes: CRDB-51115
Release note: None

Co-authored-by: Akshay Joshi <[email protected]>

Author: craig[bot]

pkg/cli/zip_upload.go

@@ -97,7 +97,7 @@ const (
 	ddArchiveDefaultClient   = "datadog-archive" // TODO(arjunmahishi): make this a flag also
 
 	gcsPathTimeFormat = "dt=20060102/hour=15"
-	zipUploadRetries  = 5
+	zipUploadRetries  = 100
 
 	// datadog allows us to use logs API logs only for the last 72 hours. So, we
 	// are setting the oldest allowed log duration to 71 hours. The -1 hour is to
@@ -950,9 +950,14 @@ var gcsLogUpload = func(ctx context.Context, sig logUploadSig) (int, error) {
 
 	retryOpts := base.DefaultRetryOptions()
 	retryOpts.MaxRetries = zipUploadRetries
+	retryOpts.InitialBackoff = 1 * time.Second
+	retryOpts.MaxBackoff = 10 * time.Second
 
 	for retry := retry.Start(retryOpts); retry.Next(); {
-		objectWriter := gcsClient.Bucket(ddArchiveBucketName).Object(filename).NewWriter(ctx)
+
+		objectWriter := gcsClient.Bucket(ddArchiveBucketName).Object(filename).Retryer(
+			storage.WithPolicy(storage.RetryAlways),
+		).NewWriter(ctx)
 		w := gzip.NewWriter(objectWriter)
 		_, err = w.Write(data)
 		if err != nil {