OCPBUGS-45280: Allow more time for Service Account Creation

** GCP Service accounts were created correctly (and they could be found in the api), but Google
states (https://cloud.google.com/iam/docs/retry-strategy) that it could take up to 60 seconds
before the service account is able to be used. The exponential backoff is adjusted to actually
increase in time each iteration. The context was also adjusted to ensure that we give the backoff
enough time to finish. Google sends a 400 Error when the service account policies could NOT be
applied, if this happens DURING the policy setting attempt again as long as we have time left
in the context.

Author: barbacbd

pkg/infrastructure/gcp/clusterapi/iam.go

@@ -10,7 +10,7 @@ import (
 	"github.com/sirupsen/logrus"
 	resourcemanager "google.golang.org/api/cloudresourcemanager/v1"
 	"google.golang.org/api/googleapi"
-	iam "google.golang.org/api/iam/v1"
+	"google.golang.org/api/iam/v1"
 	"google.golang.org/api/option"
 	"k8s.io/apimachinery/pkg/util/wait"
 
@@ -103,7 +103,8 @@ func CreateServiceAccount(ctx context.Context, infraID, projectID, role string)
 // AddServiceAccountRoles adds predefined roles for service account.
 func AddServiceAccountRoles(ctx context.Context, projectID, serviceAccountID string, roles []string) error {
 	// Get cloudresourcemanager service
-	ctx, cancel := context.WithTimeout(ctx, time.Minute*1)
+	// The context timeout must be greater in time than the exponential backoff below
+	ctx, cancel := context.WithTimeout(ctx, time.Minute*2)
 	defer cancel()
 
 	ssn, err := gcp.GetSession(ctx)
@@ -117,8 +118,9 @@ func AddServiceAccountRoles(ctx context.Context, projectID, serviceAccountID str
 
 	backoff := wait.Backoff{
 		Duration: 2 * time.Second,
+		Factor:   2.0,
 		Jitter:   1.0,
-		Steps:    5,
+		Steps:    retryCount,
 	}
 	// Get and set the policy in a backoff loop.
 	// If the policy set fails, the policy must be retrieved again via the get before retrying the set.
@@ -135,8 +137,7 @@ func AddServiceAccountRoles(ctx context.Context, projectID, serviceAccountID str
 
 		member := fmt.Sprintf("serviceAccount:%s", serviceAccountID)
 		for _, role := range roles {
-			err = addMemberToRole(policy, role, member)
-			if err != nil {
+			if err := addMemberToRole(policy, role, member); err != nil {
 				return false, fmt.Errorf("failed to add role %s to %s: %w", role, member, err)
 			}
 		}
@@ -147,6 +148,15 @@ func AddServiceAccountRoles(ctx context.Context, projectID, serviceAccountID str
 				lastErr = err
 				logrus.Debugf("Concurrent IAM policy changes, restarting read/modify/write")
 				return false, nil
+			} else if isBadStatusError(err) {
+				// Documented here, https://cloud.google.com/iam/docs/retry-strategy, google
+				// indicates that a service account may be created but not active for up to
+				// 60 seconds. This behavior was causing a failure here when setting the policy
+				// resulting in a 400 error from the API. If this error occurs retry with an
+				// exponential backoff.
+				lastErr = err
+				logrus.Debugf("bad request, unexpected error: %s", err.Error())
+				return false, nil
 			}
 			return false, fmt.Errorf("failed to set IAM policy, unexpected error: %w", err)
 		}
@@ -224,3 +234,8 @@ func isQuotaExceededError(err error) bool {
 	}
 	return false
 }
+
+func isBadStatusError(err error) bool {
+	var ae *googleapi.Error
+	return errors.As(err, &ae) && (ae.Code == http.StatusBadRequest)
+}