Split nightly test data ingestion from ticket creation (#16240)

Author: shuyama1

.ci/gcb-ingest-test-data.yaml

@@ -0,0 +1,19 @@
+---
+steps:
+    - name: 'gcr.io/graphite-docker-images/go-plus'
+      id: collect-nightly-test-status
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
+      secretEnv: ["TEAMCITY_TOKEN"]
+      args:
+        - 'collect-nightly-test-status'
+        - $_CUSTOM_DATE
+
+timeout: 3600s
+options:
+    machineType: 'N1_HIGHCPU_32'
+
+logsBucket: 'gs://cloudbuild-ingest-test-data-logs'
+availableSecrets:
+  secretManager:
+    - versionName: projects/673497134629/secrets/teamcity-token/versions/latest
+      env: TEAMCITY_TOKEN

.ci/gcb-test-failure-ticket.yaml

@@ -1,21 +1,12 @@
 ---
 steps:
-    - name: 'gcr.io/graphite-docker-images/go-plus'
-      id: collect-nightly-test-status
-      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
-      secretEnv: ["TEAMCITY_TOKEN"]
-      args:
-        - 'collect-nightly-test-status'
-        - $_CUSTOM_DATE
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: create-test-failure-ticket
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["collect-nightly-test-status"]
       args:
         - 'create-test-failure-ticket'
-    - name: 'ubuntu'
-      args: ['sleep', '120']
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: manage-test-failure-ticket
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
@@ -31,7 +22,5 @@ options:
 logsBucket: 'gs://cloudbuild-test-failure-ticket-logs'
 availableSecrets:
   secretManager:
-    - versionName: projects/673497134629/secrets/teamcity-token/versions/latest
-      env: TEAMCITY_TOKEN
     - versionName: projects/673497134629/secrets/github-classic--repo-workflow/versions/latest
       env: GITHUB_TOKEN

.ci/magician/cmd/collect_nightly_test_status.go

@@ -84,24 +84,27 @@ var collectNightlyTestStatusCmd = &cobra.Command{
 		tc := teamcity.NewClient(env["TEAMCITY_TOKEN"])
 		gcs := cloudstorage.NewClient()
 
-		now := time.Now()
-
 		loc, err := time.LoadLocation("America/Los_Angeles")
 		if err != nil {
 			return fmt.Errorf("Error loading location: %s", err)
 		}
-		date := now.In(loc)
+
+		now := time.Now().In(loc)
+		year, month, day := now.Date()
+
 		customDate := args[0]
 		// check if a specific date is provided
 		if customDate != "" {
 			parsedDate, err := time.Parse("2006-01-02", customDate) // input format YYYY-MM-DD
-			// Set the time to 7pm PT
-			date = time.Date(parsedDate.Year(), parsedDate.Month(), parsedDate.Day(), 19, 0, 0, 0, loc)
 			if err != nil {
 				return fmt.Errorf("invalid input time format: %w", err)
 			}
+			year, month, day = parsedDate.Date()
 		}
 
+		// Set the time to 7pm PT
+		date := time.Date(year, month, day, 19, 0, 0, 0, loc)
+
 		return execCollectNightlyTestStatus(date, tc, gcs)
 	},
 }
@@ -134,10 +137,31 @@ func execCollectNightlyTestStatus(now time.Time, tc TeamcityClient, gcs Cloudsto
 }
 
 func createTestReport(pVersion provider.Version, tc TeamcityClient, gcs CloudstorageClient, formattedStartCut, formattedFinishCut, date string) error {
+
+	// Check Queued Builds
+	queuedBuilds, err := tc.GetBuilds("queued", pVersion.TeamCityNightlyProjectName(), formattedFinishCut, formattedStartCut)
+	if err != nil {
+		return fmt.Errorf("failed to get queued builds: %w", err)
+	}
+	if len(queuedBuilds.Builds) > 0 {
+		fmt.Printf("%s Test unfinished: there are still %d builds queued.\n", strings.ToUpper(pVersion.String()), len(queuedBuilds.Builds))
+		return nil
+	}
+
+	// Check Running Builds
+	runningBuilds, err := tc.GetBuilds("running", pVersion.TeamCityNightlyProjectName(), formattedFinishCut, formattedStartCut)
+	if err != nil {
+		return fmt.Errorf("failed to get running builds: %w", err)
+	}
+	if len(runningBuilds.Builds) > 0 {
+		fmt.Printf("%s Test unfinished: there are still %d builds running.\n", strings.ToUpper(pVersion.String()), len(runningBuilds.Builds))
+		return nil
+	}
+
 	// Get all service test builds
-	builds, err := tc.GetBuilds(pVersion.TeamCityNightlyProjectName(), formattedFinishCut, formattedStartCut)
+	builds, err := tc.GetBuilds("finished", pVersion.TeamCityNightlyProjectName(), formattedFinishCut, formattedStartCut)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to get finished builds: %w", err)
 	}
 
 	var testInfoList []TestInfo

.ci/magician/cmd/interfaces.go

@@ -50,7 +50,7 @@ type CloudstorageClient interface {
 }
 
 type TeamcityClient interface {
-	GetBuilds(project, finishCut, startCut string) (teamcity.Builds, error)
+	GetBuilds(state, project, finishCut, startCut string) (teamcity.Builds, error)
 	GetTestResults(build teamcity.Build) (teamcity.TestResults, error)
 }
 

.ci/magician/teamcity/get.go

@@ -54,8 +54,8 @@ type FirstFailed struct {
 	Href string `json:"href"`
 }
 
-func (tc *Client) GetBuilds(project, finishCut, startCut string) (Builds, error) {
-	url := fmt.Sprintf("https://hashicorp.teamcity.com/app/rest/builds?locator=count:500,tag:cron-trigger,project:%s,branch:refs/heads/nightly-test,queuedDate:(date:%s,condition:before),queuedDate:(date:%s,condition:after)&fields=build(id,buildTypeId,buildConfName,webUrl,number,queuedDate,startDate,finishDate)", project, finishCut, startCut)
+func (tc *Client) GetBuilds(state, project, finishCut, startCut string) (Builds, error) {
+	url := fmt.Sprintf("https://hashicorp.teamcity.com/app/rest/builds?locator=state:%s,count:500,tag:cron-trigger,project:%s,branch:refs/heads/nightly-test,queuedDate:(date:%s,condition:before),queuedDate:(date:%s,condition:after)&fields=build(id,buildTypeId,buildConfName,webUrl,number,queuedDate,startDate,finishDate)", state, project, finishCut, startCut)
 
 	var builds Builds