add gitlab support

Author: daniel-cit

helpers/foundation-deployer/README.md

@@ -16,7 +16,7 @@ Helper tool to deploy the Terraform example foundation using Cloud Build and Clo
 Your environment need to use the same [Terraform](https://www.terraform.io/downloads.html) version used on the build pipeline.
 Otherwise, you might experience Terraform state snapshot lock errors.
 
-Version 1.5.7 is the last version before the license model change. To use a later version of Terraform, ensure that the Terraform version used in the Operational System to manually execute part of the steps in `3-networks` and `4-projects` is the same version configured in the following code
+Version 1.5.7 is the last version before the license model change. To use a later version of Terraform, ensure that the Terraform version used in the Operating System to manually execute part of the steps in `3-networks` and `4-projects` is the same version configured in the following code
 
 - 0-bootstrap/modules/jenkins-agent/variables.tf
    ```
@@ -73,7 +73,7 @@ Version 1.5.7 is the last version before the license model change. To use a late
 
 ### Prepare the deploy environment
 
-- Create a directory in the file system to host the Cloud Source repositories the will be created and a copy of the terraform example foundation.
+- Create a directory in the file system to host the Git Repositories that will be created (Cloud Source repositories, Github , or GitLab) and a copy of the terraform example foundation repository.
 - Clone the `terraform-example-foundation` repository on this directory.
 
     ```text
@@ -106,10 +106,6 @@ Version 1.5.7 is the last version before the license model change. To use a late
 
 By default the foundation regional resources are deployed in `us-west1` and `us-central1` regions and multi-regional resources are deployed in the `US` multi-region.
 
-In addition to the variables declared in the file `global.tfvars` for configuring location, there are two locals, `default_region1` and `default_region2`, in each one of the environments (`production`, `nonproduction`, and `development`) in the network steps (`3-networks-svpc` and `3-networks-hub-and-spoke`).
-They are located in the [main.tf](../../3-networks-svpc/envs/production/main.tf#L20-L21) files for each environments.
-Change the two locals **before** starting the deployment to deploy in other regions.
-
 **Note:** the region used for the variable `default_region` in the file `global.tfvars` **MUST** be one of the regions used for the `default_region1` and `default_region2` locals.
 
 ### Application default credentials

helpers/foundation-deployer/gcp/gcp.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"regexp"
 	"strings"
 	"time"
 
@@ -27,6 +26,8 @@ import (
 	"github.com/mitchellh/go-testing-interface"
 	"github.com/tidwall/gjson"
 
+	localutil "github.com/terraform-google-modules/terraform-example-foundation/helpers/foundation-deployer/utils"
+
 	"github.com/terraform-google-modules/terraform-example-foundation/test/integration/testutils"
 
 	"google.golang.org/api/cloudbuild/v1"
@@ -51,20 +52,6 @@ type Build struct {
 	CreateTime string `json:"createTime"`
 }
 
-var (
-	retryRegexp = map[*regexp.Regexp]string{}
-)
-
-func init() {
-	for e, m := range testutils.RetryableTransientErrors {
-		r, err := regexp.Compile(fmt.Sprintf("(?s)%s", e)) //(?s) enables dot (.) to match newline.
-		if err != nil {
-			panic(fmt.Sprintf("failed to compile regex %s: %s", e, err.Error()))
-		}
-		retryRegexp[r] = m
-	}
-}
-
 type GCP struct {
 	Runf            func(t testing.TB, cmd string, args ...interface{}) gjson.Result
 	RunCmd          func(t testing.TB, cmd string, args ...interface{}) string
@@ -209,7 +196,8 @@ func (g GCP) WaitBuildSuccess(t testing.TB, project, region, repo, commitSha, fa
 		}
 
 		if status != StatusSuccess {
-			if !g.IsRetryableError(t, project, region, build) {
+			logs := g.GetBuildLogs(t, project, region, build)
+			if !localutil.IsRetryableError(t, logs) {
 				return fmt.Errorf("%s\nSee:\nhttps://console.cloud.google.com/cloud-build/builds;region=%s/%s?project=%s\nfor details", failureMsg, region, build, project)
 			}
 			fmt.Println("build failed with retryable error. a new build will be triggered.")
@@ -230,21 +218,6 @@ func (g GCP) WaitBuildSuccess(t testing.TB, project, region, repo, commitSha, fa
 	return fmt.Errorf("%s\nbuild failed after %d retries.\nSee Cloud Build logs for details", failureMsg, maxErrorRetries)
 }
 
-// IsRetryableError checks the logs of a failed Cloud Build build
-// and verify if the error is a transient one and can be retried
-func (g GCP) IsRetryableError(t testing.TB, projectID, region, build string) bool {
-	logs := g.GetBuildLogs(t, projectID, region, build)
-	found := false
-	for pattern, msg := range retryRegexp {
-		if pattern.MatchString(logs) {
-			found = true
-			fmt.Printf("error '%s' is worth of a retry\n", msg)
-			break
-		}
-	}
-	return found
-}
-
 // HasSccNotification checks if a Security Command Center notification exists
 func (g GCP) HasSccNotification(t testing.TB, orgID, sccName string) bool {
 	filter := fmt.Sprintf("name=organizations/%s/locations/global/notificationConfigs/%s", orgID, sccName)

helpers/foundation-deployer/github/github.go

@@ -25,6 +25,9 @@ import (
 	"github.com/google/go-github/v58/github"
 	"github.com/mitchellh/go-testing-interface"
 	"github.com/terraform-google-modules/terraform-example-foundation/test/integration/testutils"
+
+	localutil "github.com/terraform-google-modules/terraform-example-foundation/helpers/foundation-deployer/utils"
+
 	"golang.org/x/oauth2"
 )
 
@@ -62,25 +65,30 @@ type GH struct {
 	sleepTime       time.Duration
 }
 
+// NewGH creates a new wrapper for The GitHub API
 func NewGH() GH {
 	return GH{
 		TriggerNewBuild: triggerNewBuild,
 		sleepTime:       20,
 	}
 }
+// triggerNewBuild triggers a new action execution
 func triggerNewBuild(t testing.TB, ctx context.Context, owner, repo, token string, runID int64) (int64, string, string, error) {
-	if token == "" {
-		return 0, "", "", fmt.Errorf("GITHUB_TOKEN not set")
-	}
-
 	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
 	tc := oauth2.NewClient(ctx, ts)
 	client := github.NewClient(tc)
 
-	_, err := client.Actions.RerunWorkflowByID(ctx, owner, repo, runID)
+	resp, err := client.Actions.RerunWorkflowByID(ctx, owner, repo, runID)
 	if err != nil {
 		return 0, "", "", fmt.Errorf("Error re-running workflow: %v", err)
 	}
+	if resp.StatusCode != http.StatusCreated {
+		bodyBytes, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return 0, "", "", fmt.Errorf("Error re-running workflow status: %d body parsing error: %v", resp.StatusCode, err)
+		}
+		return 0, "", "", fmt.Errorf("Error re-running workflow status: %d body: %s", resp.StatusCode, string(bodyBytes))
+	}
 	opts := &github.ListWorkflowRunsOptions{
 
 		ListOptions: github.ListOptions{
@@ -120,7 +128,8 @@ func triggerNewBuild(t testing.TB, ctx context.Context, owner, repo, token strin
 	return newRunID, status, conclusion, nil
 }
 
-func (g GH) GetLastBuildState(t testing.TB, ctx context.Context, owner, repo, token string) (int64, string, string, error) {
+// GetLastActionState returns the state of the latest action
+func (g GH) GetLastActionState(t testing.TB, ctx context.Context, owner, repo, token string) (int64, string, string, error) {
 	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
 	tc := oauth2.NewClient(ctx, ts)
 	client := github.NewClient(tc)
@@ -158,6 +167,7 @@ func (g GH) GetLastBuildState(t testing.TB, ctx context.Context, owner, repo, to
 	return runID, status, conclusion, nil
 }
 
+// GetActionState returns the state of a given action
 func (g GH) GetActionState(t testing.TB, ctx context.Context, owner, repo, token string, runID int64) (string, string, error) {
 	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
 	tc := oauth2.NewClient(ctx, ts)
@@ -180,6 +190,7 @@ func (g GH) GetActionState(t testing.TB, ctx context.Context, owner, repo, token
 	return status, conclusion, nil
 }
 
+// GetBuildLogs returns the execution logs of an action
 func (g GH) GetBuildLogs(t testing.TB, ctx context.Context, owner, repo, token string, runID int64) (string, error) {
 	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
 	tc := oauth2.NewClient(ctx, ts)
@@ -229,6 +240,7 @@ func (g GH) GetBuildLogs(t testing.TB, ctx context.Context, owner, repo, token s
 	return "", nil
 }
 
+// GetFinalActionState returns the final state of an action
 func (g GH) GetFinalActionState(t testing.TB, ctx context.Context, owner, repo, token string, runID int64, maxBuildRetry int) (string, string, error) {
 	var status, conclusion string
 	var err error
@@ -265,7 +277,7 @@ func (g GH) WaitBuildSuccess(t testing.TB, owner, repo, token string, failureMsg
 	// wait action creation
 	time.Sleep(30 * time.Second)
 
-	runID, status, conclusion, err = g.GetLastBuildState(t, ctx, owner, repo, token)
+	runID, status, conclusion, err = g.GetLastActionState(t, ctx, owner, repo, token)
 	if err != nil {
 		return err
 	}
@@ -282,7 +294,7 @@ func (g GH) WaitBuildSuccess(t testing.TB, owner, repo, token string, failureMsg
 			if err != nil {
 				return err
 			}
-			if !g.IsRetryableError(t, logs) {
+			if localutil.IsRetryableError(t, logs) {
 				return fmt.Errorf("%s\nSee:\nhttps://github.com/%s/%s/actions/runs/%d\nfor details", failureMsg, owner, repo, runID)
 			}
 			fmt.Println("build failed with retryable error. a new build will be triggered.")
@@ -302,17 +314,3 @@ func (g GH) WaitBuildSuccess(t testing.TB, owner, repo, token string, failureMsg
 	}
 	return fmt.Errorf("%s action failed after %d retries", failureMsg, maxErrorRetries)
 }
-
-// IsRetryableError checks the logs of a failed Cloud Build build
-// and verify if the error is a transient one and can be retried
-func (g GH) IsRetryableError(t testing.TB, logs string) bool {
-	found := false
-	for pattern, msg := range retryRegexp {
-		if pattern.MatchString(logs) {
-			found = true
-			fmt.Printf("error '%s' is worth of a retry\n", msg)
-			break
-		}
-	}
-	return found
-}