fix: Retry connection to valkey (#1700)

Fixes #1698

NewClient tries to dial the valkey instance before returning.
If there's a problem with dialing valkey, it will fail and not recover.
As a result, the cloud run instance will fail to deploy.

This change adds retry logic in an attempt to give the server more attempts
before failing.

Author: jcscottiii

lib/go.mod

@@ -9,6 +9,7 @@ require (
 	firebase.google.com/go/v4 v4.18.0
 	github.com/GoogleChrome/webstatus.dev/lib/gen v0.0.0-20250804165824-fd606c7d439d
 	github.com/antlr4-go/antlr/v4 v4.13.1
+	github.com/cenkalti/backoff/v5 v5.0.3
 	github.com/deckarep/golang-set v1.8.0
 	github.com/google/go-github/v73 v73.0.0
 	github.com/oapi-codegen/runtime v1.1.2
@@ -32,7 +33,6 @@ require (
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect
 	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
-	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/containerd/errdefs v1.0.0 // indirect
 	github.com/containerd/errdefs/pkg v0.3.0 // indirect
 	github.com/containerd/platforms v0.2.1 // indirect

lib/valkeycache/cache.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"github.com/GoogleChrome/webstatus.dev/lib/cachetypes"
+	"github.com/cenkalti/backoff/v5"
 	"github.com/valkey-io/valkey-go"
 )
 
@@ -42,10 +43,19 @@ func NewValkeyDataCache[K comparable, V []byte](
 	ttl time.Duration) (*ValkeyDataCache[K, V], error) {
 
 	addr := fmt.Sprintf("%s:%s", host, port)
-	// nolint: exhaustruct // No need to use every option of 3rd party struct.
-	c, err := valkey.NewClient(valkey.ClientOption{
-		InitAddress: []string{addr},
-	})
+	operation := func() (valkey.Client, error) {
+		// nolint: exhaustruct // No need to use every option of 3rd party struct.
+		return valkey.NewClient(valkey.ClientOption{
+			InitAddress: []string{addr},
+		})
+	}
+
+	c, err := backoff.Retry(context.TODO(), operation,
+		backoff.WithBackOff(backoff.NewExponentialBackOff()),
+		// Should be less than the total time in the startup probe for the backend container in
+		// infra/backend/service.tf
+		backoff.WithMaxElapsedTime(25*time.Second),
+	)
 	if err != nil {
 		return nil, err
 	}