INCIDENT-37491: ocitool push: add retry mechanism

Author: georgesfarah

MODULE.bazel

@@ -42,6 +42,7 @@ use_repo(
     "com_github_sirupsen_logrus",
     "com_github_stretchr_testify",
     "com_github_urfave_cli_v2",
+    "com_github_sethvargo_go_retry",
     "land_oras_oras_go",
     "org_golang_google_grpc",
     "org_golang_x_sync",

go.mod

@@ -65,6 +65,7 @@ require (
 	github.com/prometheus/common v0.53.0 // indirect
 	github.com/prometheus/procfs v0.14.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sethvargo/go-retry v0.3.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect

go.sum

@@ -209,6 +209,8 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE=
+github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=

go/cmd/ocitool/BUILD.bazel

@@ -34,6 +34,7 @@ go_library(
         "@com_github_opencontainers_go_digest//:go_default_library",
         "@com_github_opencontainers_image_spec//specs-go:go_default_library",
         "@com_github_opencontainers_image_spec//specs-go/v1:go_default_library",
+        "@com_github_sethvargo_go_retry//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_urfave_cli_v2//:go_default_library",
         "@gazelle//rule:go_default_library",

go/cmd/ocitool/main.go

@@ -153,6 +153,11 @@ var app = &cli.App{
 					Name:  "x_meta_headers",
 					Value: &flagutil.KeyValueFlag{},
 				},
+				&cli.UintFlag{
+					Name:  "retry",
+					Usage: "Maximum number of retries for pushing content to the registry",
+					Value: 3,
+				},
 			},
 		},
 		{

go/cmd/ocitool/push_cmd.go

@@ -1,13 +1,16 @@
 package main
 
 import (
+	"context"
 	"fmt"
+	"time"
 
 	"github.com/DataDog/rules_oci/go/internal/flagutil"
 	"github.com/DataDog/rules_oci/go/pkg/ociutil"
 
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/images"
+	"github.com/sethvargo/go-retry"
 	"github.com/urfave/cli/v2"
 )
 
@@ -70,7 +73,17 @@ func PushCmd(c *cli.Context) error {
 	}
 
 	// push the parent last (in case of image index)
-	err = ociutil.CopyContent(c.Context, allProviders, regIng, baseDesc)
+	attempt := 0
+	maxRetries := uint64(c.Uint("retry"))
+	err = retry.Do(c.Context, retry.WithMaxRetries(maxRetries, retry.NewFibonacci(1*time.Second)), func(_ context.Context) error {
+		attempt++
+		fmt.Printf("Retry attempt %d/%d\n", attempt, maxRetries+1) // MaxAttempt = maxRetries + 1
+		err := ociutil.CopyContent(c.Context, allProviders, regIng, baseDesc)
+		if err != nil {
+			fmt.Printf("Attempt %d failed: %v\n", attempt, err)
+		}
+		return err
+	})
 	if err != nil {
 		return fmt.Errorf("failed to push parent content to registry: %w", err)
 	}

oci/BUILD.bazel

@@ -99,16 +99,16 @@ bzl_library(
 )
 
 bzl_library(
-    name = "debug_flag",
-    srcs = ["debug_flag.bzl"],
+    name = "layer",
+    srcs = ["layer.bzl"],
     visibility = ["//visibility:public"],
+    deps = ["@com_github_datadog_rules_oci//oci:providers"],
 )
 
 bzl_library(
-    name = "layer",
-    srcs = ["layer.bzl"],
+    name = "debug_flag",
+    srcs = ["debug_flag.bzl"],
     visibility = ["//visibility:public"],
-    deps = ["@com_github_datadog_rules_oci//oci:providers"],
 )
 
 bzl_library(