Merge branch 'GoogleCloudPlatform:main' into fix/25305-take2

Author: tamir-deep

.ci/containers/build-environment/Dockerfile

@@ -17,7 +17,7 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 1777 "$GOPATH"
 WORKDIR $GOPATH
 
 # terraform binary used by tfv/tgc
-COPY --from=hashicorp/terraform:1.11.0 /bin/terraform /bin/terraform
+COPY --from=hashicorp/terraform:1.14.3 /bin/terraform /bin/terraform
 
 SHELL ["/bin/bash", "-c"]
 

.ci/containers/go-plus/Dockerfile

@@ -29,7 +29,7 @@ RUN apt-get update && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-RUN wget https://releases.hashicorp.com/terraform/1.11.0/terraform_1.11.0_linux_amd64.zip \
-    && unzip terraform_1.11.0_linux_amd64.zip \
-    && rm terraform_1.11.0_linux_amd64.zip \
+RUN wget https://releases.hashicorp.com/terraform/1.14.3/terraform_1.14.3_linux_amd64.zip \
+    && unzip terraform_1.14.3_linux_amd64.zip \
+    && rm terraform_1.14.3_linux_amd64.zip \
     && mv ./terraform /bin/terraform

.ci/gcb-ingest-test-data.yaml

@@ -0,0 +1,19 @@
+---
+steps:
+    - name: 'gcr.io/graphite-docker-images/go-plus'
+      id: collect-nightly-test-status
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
+      secretEnv: ["TEAMCITY_TOKEN"]
+      args:
+        - 'collect-nightly-test-status'
+        - $_CUSTOM_DATE
+
+timeout: 3600s
+options:
+    machineType: 'N1_HIGHCPU_32'
+
+logsBucket: 'gs://cloudbuild-ingest-test-data-logs'
+availableSecrets:
+  secretManager:
+    - versionName: projects/673497134629/secrets/teamcity-token/versions/latest
+      env: TEAMCITY_TOKEN

.ci/gcb-test-failure-ticket.yaml

@@ -1,21 +1,11 @@
 ---
 steps:
-    - name: 'gcr.io/graphite-docker-images/go-plus'
-      id: collect-nightly-test-status
-      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
-      secretEnv: ["TEAMCITY_TOKEN"]
-      args:
-        - 'collect-nightly-test-status'
-        - $_CUSTOM_DATE
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: create-test-failure-ticket
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       secretEnv: ["GITHUB_TOKEN"]
-      waitFor: ["collect-nightly-test-status"]
       args:
         - 'create-test-failure-ticket'
-    - name: 'ubuntu'
-      args: ['sleep', '120']
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: manage-test-failure-ticket
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
@@ -31,7 +21,5 @@ options:
 logsBucket: 'gs://cloudbuild-test-failure-ticket-logs'
 availableSecrets:
   secretManager:
-    - versionName: projects/673497134629/secrets/teamcity-token/versions/latest
-      env: TEAMCITY_TOKEN
     - versionName: projects/673497134629/secrets/github-classic--repo-workflow/versions/latest
       env: GITHUB_TOKEN

.ci/infra/terraform/main.tf

@@ -295,6 +295,7 @@ module "project-services" {
     "firebaseremoteconfig.googleapis.com",
     "firebaserules.googleapis.com",
     "firebasestorage.googleapis.com",
+    "firebasevertexai.googleapis.com",
     "firestore.googleapis.com",
     "firestorekeyvisualizer.googleapis.com",
     "gkebackup.googleapis.com",

.ci/magician/cmd/collect_nightly_test_status.go

@@ -21,7 +21,9 @@ import (
 	"magician/provider"
 	"magician/teamcity"
 	utils "magician/utility"
+	"net/url"
 	"os"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -43,7 +45,9 @@ type TestInfo struct {
 	Status          string    `json:"status"`
 	Service         string    `json:"service"`
 	Resource        string    `json:"resource"`
+	CommitSha       string    `json:"commit_sha"`
 	ErrorMessage    string    `json:"error_message"`
+	ErrorType       string    `json:"error_type"`
 	LogLink         string    `json:"log_link"`
 	ProviderVersion string    `json:"provider_version"`
 	QueuedDate      time.Time `json:"queued_date"`
@@ -83,24 +87,27 @@ var collectNightlyTestStatusCmd = &cobra.Command{
 		tc := teamcity.NewClient(env["TEAMCITY_TOKEN"])
 		gcs := cloudstorage.NewClient()
 
-		now := time.Now()
-
 		loc, err := time.LoadLocation("America/Los_Angeles")
 		if err != nil {
 			return fmt.Errorf("Error loading location: %s", err)
 		}
-		date := now.In(loc)
+
+		now := time.Now().In(loc)
+		year, month, day := now.Date()
+
 		customDate := args[0]
 		// check if a specific date is provided
 		if customDate != "" {
 			parsedDate, err := time.Parse("2006-01-02", customDate) // input format YYYY-MM-DD
-			// Set the time to 7pm PT
-			date = time.Date(parsedDate.Year(), parsedDate.Month(), parsedDate.Day(), 19, 0, 0, 0, loc)
 			if err != nil {
 				return fmt.Errorf("invalid input time format: %w", err)
 			}
+			year, month, day = parsedDate.Date()
 		}
 
+		// Set the time to 7pm PT
+		date := time.Date(year, month, day, 19, 0, 0, 0, loc)
+
 		return execCollectNightlyTestStatus(date, tc, gcs)
 	},
 }
@@ -133,10 +140,40 @@ func execCollectNightlyTestStatus(now time.Time, tc TeamcityClient, gcs Cloudsto
 }
 
 func createTestReport(pVersion provider.Version, tc TeamcityClient, gcs CloudstorageClient, formattedStartCut, formattedFinishCut, date string) error {
+
+	baseLocator := fmt.Sprintf("count:500,project:%s,branch:refs/heads/nightly-test,queuedDate:(date:%s,condition:before),queuedDate:(date:%s,condition:after)", pVersion.TeamCityNightlyProjectName(), formattedFinishCut, formattedStartCut)
+	fields := "build(id,buildTypeId,buildConfName,webUrl,number,queuedDate,startDate,finishDate)"
+	params := url.Values{}
+
+	// Check Queued Builds
+	params.Set("locator", fmt.Sprintf("%s,state:queued", baseLocator))
+	queuedBuilds, err := tc.GetBuilds(params)
+	if err != nil {
+		return fmt.Errorf("failed to get queued builds: %w", err)
+	}
+	if len(queuedBuilds.Builds) > 0 {
+		fmt.Printf("%s Test unfinished: there are still %d builds queued.\n", strings.ToUpper(pVersion.String()), len(queuedBuilds.Builds))
+		return nil
+	}
+
+	// Check Running Builds
+	params.Set("locator", fmt.Sprintf("%s,state:running,tag:cron-trigger", baseLocator))
+	params.Set("fields", fields)
+	runningBuilds, err := tc.GetBuilds(params)
+	if err != nil {
+		return fmt.Errorf("failed to get running builds: %w", err)
+	}
+	if len(runningBuilds.Builds) > 0 {
+		fmt.Printf("%s Test unfinished: there are still %d builds running.\n", strings.ToUpper(pVersion.String()), len(runningBuilds.Builds))
+		return nil
+	}
+
 	// Get all service test builds
-	builds, err := tc.GetBuilds(pVersion.TeamCityNightlyProjectName(), formattedFinishCut, formattedStartCut)
+	params.Set("locator", fmt.Sprintf("%s,state:finished,tag:cron-trigger", baseLocator))
+	params.Set("fields", fields)
+	builds, err := tc.GetBuilds(params)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to get finished builds: %w", err)
 	}
 
 	var testInfoList []TestInfo
@@ -163,12 +200,14 @@ func createTestReport(pVersion provider.Version, tc TeamcityClient, gcs Cloudsto
 
 		for _, testResult := range serviceTestResults.TestResults {
 			var errorMessage string
+			var errorType string
 			// Get test debug log gcs link
 			logLink := fmt.Sprintf("https://storage.cloud.google.com/teamcity-logs/nightly/%s/%s/%s/debug-%s-%s-%s-%s.txt", pVersion.TeamCityNightlyProjectName(), date, build.Number, pVersion.ProviderName(), build.Number, strconv.Itoa(build.Id), testResult.Name)
 			// Get concise error message for failed and skipped tests
 			// Skipped tests have a status of "UNKNOWN" on TC
 			if testResult.Status == "FAILURE" || testResult.Status == "UNKNOWN" {
 				errorMessage = convertErrorMessage(testResult.ErrorMessage)
+				errorType = categorizeError(errorMessage)
 			}
 
 			queuedTime, err := time.Parse(tcTimeFormat, build.QueuedDate)
@@ -189,7 +228,9 @@ func createTestReport(pVersion provider.Version, tc TeamcityClient, gcs Cloudsto
 				Status:          testResult.Status,
 				Service:         serviceName,
 				Resource:        convertTestNameToResource(testResult.Name),
+				CommitSha:       build.Number,
 				ErrorMessage:    errorMessage,
+				ErrorType:       errorType,
 				LogLink:         logLink,
 				ProviderVersion: strings.ToUpper(pVersion.String()),
 				Duration:        testResult.Duration,
@@ -251,6 +292,145 @@ func convertErrorMessage(rawErrorMessage string) string {
 	return strings.TrimSpace(rawErrorMessage[startIndex:endIndex])
 }
 
+var (
+	reSubnetNotReady   = regexp.MustCompile(`The resource '[^']+/subnetworks/[^']+' is not ready`)
+	reApiEnv           = regexp.MustCompile(`has not been used in project (ci-test-project-188019|1067888929963|ci-test-project-nightly-ga|594424405950|ci-test-project-nightly-beta|653407317329|tf-vcr-private|808590572184) before or it is disabled`)
+	reAttrSet          = regexp.MustCompile(`Attribute '[^']+' expected to be set`)
+	reQuotaLimit       = regexp.MustCompile(`Quota limit '[^']+' has been exceeded`)
+	reGoogleApi4xx     = regexp.MustCompile(`googleapi: Error 4\d\d`)
+	reGoogleApi5xx     = regexp.MustCompile(`googleapi: Error 5\d\d`)
+	reGoogleApiGeneric = regexp.MustCompile(`googleapi: Error`)
+)
+
+func categorizeError(errMsg string) string {
+	if strings.Contains(errMsg, "Error code 13") {
+		return "Error code 13"
+	}
+	if strings.Contains(errMsg, "Precondition check failed") {
+		return "Precondition check failed"
+	}
+
+	// Diff Category
+	if strings.Contains(errMsg, "After applying this test step, the plan was not empty") ||
+		strings.Contains(errMsg, "After applying this test step and performing a `terraform refresh`") ||
+		strings.Contains(errMsg, "Expected a non-empty plan, but got an empty plan") ||
+		strings.Contains(errMsg, "error: Check failed") {
+		return "Diff"
+	}
+
+	if strings.Contains(errMsg, "timeout while waiting for state") {
+		return "Operation timeout"
+	}
+
+	// Regex: Subnetwork not ready
+	if reSubnetNotReady.MatchString(errMsg) {
+		return "Subnetwork not ready"
+	}
+
+	// ImportStateVerify Category
+	if strings.Contains(errMsg, "ImportStateVerify attributes not equivalent") ||
+		strings.Contains(errMsg, "Cannot import non-existent remote object") ||
+		strings.Contains(errMsg, "Error: Unexpected Import Identifier") {
+		return "ImportStateVerify"
+	}
+
+	// Deprecated (Case-insensitive check)
+	if strings.Contains(strings.ToLower(errMsg), "deprecated") {
+		return "Deprecated"
+	}
+
+	if strings.Contains(errMsg, "Provider produced inconsistent result after apply") &&
+		strings.Contains(errMsg, "Root object was present, but now absent") {
+		return "Root object was present, but now absent"
+	}
+
+	if strings.Contains(errMsg, "Provider produced inconsistent final plan") {
+		return "Provider produced inconsistent final plan"
+	}
+
+	// API Enablement
+	if reApiEnv.MatchString(errMsg) {
+		return "API enablement (Test environment)"
+	}
+	if strings.Contains(errMsg, "has not been used in project") && strings.Contains(errMsg, "before or it is disabled") {
+		return "API enablement (Created project)"
+	}
+
+	if strings.Contains(errMsg, "does not have required permissions") {
+		return "Permissions"
+	}
+	if strings.Contains(errMsg, "bootstrap_iam_test_utils.go") {
+		return "Bootstrapping"
+	}
+
+	// Bad Config Category
+	if strings.Contains(errMsg, "Inconsistent dependency lock file") ||
+		strings.Contains(errMsg, "Invalid resource type") ||
+		strings.Contains(errMsg, "Blocks of type") && strings.Contains(errMsg, "are not expected here") ||
+		strings.Contains(errMsg, "Conflicting configuration arguments") ||
+		reAttrSet.MatchString(errMsg) {
+		return "Bad config"
+	}
+
+	// Quota Category
+	if strings.Contains(errMsg, "Quota exhausted") ||
+		strings.Contains(errMsg, "Quota exceeded") ||
+		strings.Contains(errMsg, "You do not have quota") ||
+		reQuotaLimit.MatchString(errMsg) {
+		return "Quota"
+	}
+
+	if strings.Contains(errMsg, "does not have enough resources available") {
+		return "Resource availability"
+	}
+
+	// API Create/Read/Update/Delete
+	if strings.Contains(errMsg, "Error: Error waiting to create") ||
+		strings.Contains(errMsg, "Error: Error waiting for Create") ||
+		strings.Contains(errMsg, "Error: Error waiting for creating") ||
+		strings.Contains(errMsg, "Error: Error creating") ||
+		strings.Contains(errMsg, "was created in the error state") ||
+		strings.Contains(errMsg, "Error: Error changing instance status after creation:") {
+		return "API Create"
+	}
+
+	if strings.Contains(errMsg, "Error: Error reading") {
+		return "API Read"
+	}
+
+	if strings.Contains(errMsg, "Error setting IAM policy") ||
+		strings.Contains(errMsg, "Error applying IAM policy") {
+		return "API IAM"
+	}
+
+	if strings.Contains(errMsg, "Error: Error waiting for Updating") ||
+		strings.Contains(errMsg, "Error: Error updating") {
+		return "API Update"
+	}
+
+	if strings.Contains(errMsg, "Error: Error waiting for Deleting") ||
+		strings.Contains(errMsg, "Error running post-test destroy") {
+		return "API Delete"
+	}
+
+	// Google API Errors (Order matters: check specific codes before generic)
+	if reGoogleApi4xx.MatchString(errMsg) {
+		return "API (4xx)"
+	}
+	if reGoogleApi5xx.MatchString(errMsg) {
+		return "API (5xx)"
+	}
+	if reGoogleApiGeneric.MatchString(errMsg) ||
+		strings.Contains(errMsg, "Error: Error when reading or editing") ||
+		strings.Contains(errMsg, "Error: Error waiting") ||
+		strings.Contains(errMsg, "unable to queue the operation") ||
+		strings.Contains(errMsg, "Error waiting for Switching runtime") {
+		return "API (Other)"
+	}
+
+	return "Other"
+}
+
 func init() {
 	rootCmd.AddCommand(collectNightlyTestStatusCmd)
 }