hotfix(operator): retry logic for batches download (#2030)

Co-authored-by: MauroFab <[email protected]>
Co-authored-by: Mauro Toscano <[email protected]>

Author: JuArce

operator/pkg/operator.go

@@ -6,8 +6,6 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
-	rapidsnark_types "github.com/iden3/go-rapidsnark/types"
-	"github.com/iden3/go-rapidsnark/verifier"
 	"log"
 	"math/big"
 	"net/http"
@@ -16,6 +14,9 @@ import (
 	"sync"
 	"time"
 
+	rapidsnark_types "github.com/iden3/go-rapidsnark/types"
+	"github.com/iden3/go-rapidsnark/verifier"
+
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/urfave/cli/v2"
 	"github.com/yetanotherco/aligned_layer/operator/risc_zero"
@@ -64,8 +65,10 @@ type Operator struct {
 }
 
 const (
-	BatchDownloadTimeout    = 1 * time.Minute
-	BatchDownloadMaxRetries = 3
+	// This time out will even kill the retries, so it should be enough
+	// for them to be completed in most cases
+	BatchDownloadTimeout    = 3 * time.Minute
+	BatchDownloadMaxRetries = 4
 	BatchDownloadRetryDelay = 5 * time.Second
 	UnverifiedBatchOffset   = 100
 )

operator/pkg/s3.go

@@ -18,6 +18,13 @@ func (o *Operator) getBatchFromDataService(ctx context.Context, batchURL string,
 	var err error
 	var req *http.Request
 
+	// Create HTTP client with response header timeout to prevent hanging on silent servers
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	transport.ResponseHeaderTimeout = 15 * time.Second
+	client := &http.Client{
+		Transport: transport,
+	}
+
 	for attempt := 0; attempt < maxRetries; attempt++ {
 		if attempt > 0 {
 			o.Logger.Infof("Waiting for %s before retrying data fetch (attempt %d of %d)", retryDelay, attempt+1, maxRetries)
@@ -35,7 +42,7 @@ func (o *Operator) getBatchFromDataService(ctx context.Context, batchURL string,
 			return nil, err
 		}
 
-		resp, err = http.DefaultClient.Do(req)
+		resp, err = client.Do(req)
 		if err == nil && resp.StatusCode == http.StatusOK {
 			break // Successful request, exit retry loop
 		}