Update nightly test data ingestion (GoogleCloudPlatform#13956)

Author: shuyama1

.ci/magician/cmd/collect_nightly_test_status.go

@@ -38,11 +38,16 @@ var cntsRequiredEnvironmentVariables = [...]string{
 }
 
 type TestInfo struct {
-	Name         string `json:"name"`
-	Status       string `json:"status"`
-	Service      string `json:"service"`
-	ErrorMessage string `json:"error_message"`
-	LogLink      string `json"log_link`
+	Name            string `json:"name"`
+	Status          string `json:"status"`
+	Service         string `json:"service"`
+	ErrorMessage    string `json:"error_message"`
+	LogLink         string `json"log_link`
+	ProviderVersion string `json:"provider_version"`
+	QueuedDate      string `json:"queuedDate"`
+	StartDate       string `json:"startDate"`
+	FinishDate      string `json:"finishDate"`
+	Duration        int    `json:"duration"`
 }
 
 // collectNightlyTestStatusCmd represents the collectNightlyTestStatus command
@@ -87,8 +92,8 @@ var collectNightlyTestStatusCmd = &cobra.Command{
 		// check if a specific date is provided
 		if customDate != "" {
 			parsedDate, err := time.Parse("2006-01-02", customDate) // input format YYYY-MM-DD
-			// Set the time to 6pm PT
-			date = time.Date(parsedDate.Year(), parsedDate.Month(), parsedDate.Day(), 18, 0, 0, 0, loc)
+			// Set the time to 7pm PT
+			date = time.Date(parsedDate.Year(), parsedDate.Month(), parsedDate.Day(), 19, 0, 0, 0, loc)
 			if err != nil {
 				return fmt.Errorf("invalid input time format: %w", err)
 			}
@@ -164,11 +169,16 @@ func createTestReport(pVersion provider.Version, tc TeamcityClient, gcs Cloudsto
 				errorMessage = convertErrorMessage(testResult.ErrorMessage)
 			}
 			testInfoList = append(testInfoList, TestInfo{
-				Name:         testResult.Name,
-				Status:       testResult.Status,
-				Service:      serviceName,
-				ErrorMessage: errorMessage,
-				LogLink:      logLink,
+				Name:            testResult.Name,
+				Status:          testResult.Status,
+				Service:         serviceName,
+				ErrorMessage:    errorMessage,
+				LogLink:         logLink,
+				ProviderVersion: pVersion.String(),
+				Duration:        testResult.Duration,
+				QueuedDate:      build.QueuedDate,
+				StartDate:       build.StartDate,
+				FinishDate:      build.FinishDate,
 			})
 		}
 	}

.ci/magician/teamcity/get.go

@@ -28,6 +28,9 @@ type Build struct {
 	BuildConfName string `json:"buildConfName"`
 	WebUrl        string `json:"webUrl"`
 	Number        string `json:"number"`
+	QueuedDate    string `json:"queuedDate"`
+	StartDate     string `json:"startDate"`
+	FinishDate    string `json:"finishDate"`
 }
 
 type Builds struct {
@@ -52,7 +55,7 @@ type FirstFailed struct {
 }
 
 func (tc *Client) GetBuilds(project, finishCut, startCut string) (Builds, error) {
-	url := fmt.Sprintf("https://hashicorp.teamcity.com/app/rest/builds?locator=count:500,tag:cron-trigger,project:%s,branch:refs/heads/nightly-test,finishDate:(date:%s,condition:before),startDate:(date:%s,condition:after)", project, finishCut, startCut)
+	url := fmt.Sprintf("https://hashicorp.teamcity.com/app/rest/builds?locator=count:500,tag:cron-trigger,project:%s,branch:refs/heads/nightly-test,queuedDate:(date:%s,condition:before),queuedDate:(date:%s,condition:after)&fields=build(id,buildTypeId,buildConfName,webUrl,number,queuedDate,startDate,finishDate)", project, finishCut, startCut)
 
 	var builds Builds
 
@@ -62,7 +65,7 @@ func (tc *Client) GetBuilds(project, finishCut, startCut string) (Builds, error)
 }
 
 func (tc *Client) GetTestResults(build Build) (TestResults, error) {
-	url := fmt.Sprintf("https://hashicorp.teamcity.com/app/rest/testOccurrences?locator=count:5000,build:(id:%d)&fields=testOccurrence(id,name,status,duration,firstFailed(href),details,build(webUrl))", build.Id)
+	url := fmt.Sprintf("https://hashicorp.teamcity.com/app/rest/testOccurrences?locator=count:5000,build:(id:%d)&fields=testOccurrence(id,name,status,duration,firstFailed(href),details)", build.Id)
 
 	var testResults TestResults