[image-builder] Detect "429 - Too Many Request" and bubble up as "Unavailable" (#20112)

Author: geropl

components/image-builder-mk3/pkg/orchestrator/orchestrator.go

@@ -624,6 +624,9 @@ func (o *Orchestrator) getAbsoluteImageRef(ctx context.Context, ref string, allo
 		}
 		return "", status.Error(codes.Unauthenticated, "cannot resolve image")
 	}
+	if resolve.TooManyRequestsMatcher(err) {
+		return "", status.Errorf(codes.Unavailable, "upstream registry responds with 'too many request': %v", err)
+	}
 	if err != nil {
 		return "", status.Errorf(codes.Internal, "cannot resolve image: %v", err)
 	}

components/image-builder-mk3/pkg/resolve/resolve.go

@@ -32,6 +32,14 @@ var (
 
 	// ErrNotFound is returned when we're not authorized to return the reference
 	ErrUnauthorized = xerrors.Errorf("not authorized")
+
+	// TooManyRequestsMatcher returns true if an error is a code 429 "Too Many Requests" error
+	TooManyRequestsMatcher = func(err error) bool {
+		if err == nil {
+			return false
+		}
+		return strings.Contains(err.Error(), "429 Too Many Requests")
+	}
 )
 
 // StandaloneRefResolver can resolve image references without a Docker daemon

components/image-builder-mk3/pkg/resolve/resolve_test.go

@@ -20,14 +20,16 @@ import (
 	"github.com/gitpod-io/gitpod/image-builder/pkg/resolve"
 	"github.com/golang/mock/gomock"
 	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/opencontainers/go-digest"
 	ociv1 "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 func TestStandaloneRefResolverResolve(t *testing.T) {
 	type Expectation struct {
-		Ref   string
-		Error string
+		Ref          string
+		Error        string
+		ErrorMatcher func(error) bool
 	}
 	type ResolveResponse struct {
 		Error         error
@@ -106,6 +108,16 @@ func TestStandaloneRefResolverResolve(t *testing.T) {
 				Error: resolve.ErrUnauthorized.Error(),
 			},
 		},
+		{
+			Name: "dockerhub rate limit",
+			Ref:  "registry-1.docker.io:5000/gitpod/gitpod/workspace-full:latest-pulled-too-often",
+			ResolveResponse: ResolveResponse{
+				Error: errors.New("httpReadSeeker: failed open: unexpected status code https://registry-1.docker.io/v2/gitpod/workspace-full/manifests/sha256:279f925ad6395f11f6b60e63d7efa5c0b26a853c6052327efbe29bbcc0bafd6a: 429 Too Many Requests - Server message: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit"),
+			},
+			Expectation: Expectation{
+				ErrorMatcher: resolve.TooManyRequestsMatcher,
+			},
+		},
 		{
 			Name: "not found",
 			Ref:  "something.com/we/dont:find",
@@ -195,7 +207,16 @@ func TestStandaloneRefResolverResolve(t *testing.T) {
 				act.Error = err.Error()
 			}
 
-			if diff := cmp.Diff(test.Expectation, act); diff != "" {
+			// ErrorMatcher?
+			if err != nil && test.Expectation.ErrorMatcher != nil {
+				if test.Expectation.ErrorMatcher(err) {
+					test.Expectation.Error = act.Error
+				} else {
+					test.Expectation.Error = "ErrorMatcher failed"
+				}
+			}
+
+			if diff := cmp.Diff(test.Expectation, act, cmpopts.IgnoreFields(Expectation{}, "ErrorMatcher")); diff != "" {
 				t.Errorf("Resolve() mismatch (-want +got):\n%s", diff)
 			}
 		})