Skip to content

Split nightly test data ingestion from ticket creation #16240

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
shuyama1 merged 7 commits into from
Jan 28, 2026
Merged

Split nightly test data ingestion from ticket creation #16240

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
882a305
split test data ingestion from ticket creation
shuyama1 Jan 27, 2026
6dc271a
correct log bucket name
shuyama1 Jan 27, 2026
e8ad020
fix
shuyama1 Jan 27, 2026
5cc562a
fix
shuyama1 Jan 27, 2026
891fcad
rename
shuyama1 Jan 27, 2026
29aa45b
add provider version to message
shuyama1 Jan 27, 2026
88f79f3
correct message]
shuyama1 Jan 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions .ci/gcb-ingest-test-data.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
---
steps:
- name: 'gcr.io/graphite-docker-images/go-plus'
id: collect-nightly-test-status
entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
secretEnv: ["TEAMCITY_TOKEN"]
args:
- 'collect-nightly-test-status'
- $_CUSTOM_DATE

timeout: 3600s
options:
machineType: 'N1_HIGHCPU_32'

logsBucket: 'gs://cloudbuild-ingest-test-data-logs'
availableSecrets:
secretManager:
- versionName: projects/673497134629/secrets/teamcity-token/versions/latest
env: TEAMCITY_TOKEN
11 changes: 0 additions & 11 deletions .ci/gcb-test-failure-ticket.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,21 +1,12 @@
---
steps:
- name: 'gcr.io/graphite-docker-images/go-plus'
id: collect-nightly-test-status
entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
secretEnv: ["TEAMCITY_TOKEN"]
args:
- 'collect-nightly-test-status'
- $_CUSTOM_DATE
- name: 'gcr.io/graphite-docker-images/go-plus'
id: create-test-failure-ticket
entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
secretEnv: ["GITHUB_TOKEN"]
waitFor: ["collect-nightly-test-status"]
args:
- 'create-test-failure-ticket'
- name: 'ubuntu'
args: ['sleep', '120']
- name: 'gcr.io/graphite-docker-images/go-plus'
id: manage-test-failure-ticket
entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
Expand All @@ -31,7 +22,5 @@ options:
logsBucket: 'gs://cloudbuild-test-failure-ticket-logs'
availableSecrets:
secretManager:
- versionName: projects/673497134629/secrets/teamcity-token/versions/latest
env: TEAMCITY_TOKEN
- versionName: projects/673497134629/secrets/github-classic--repo-workflow/versions/latest
env: GITHUB_TOKEN
38 changes: 31 additions & 7 deletions .ci/magician/cmd/collect_nightly_test_status.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,24 +84,27 @@ var collectNightlyTestStatusCmd = &cobra.Command{
tc := teamcity.NewClient(env["TEAMCITY_TOKEN"])
gcs := cloudstorage.NewClient()

now := time.Now()

loc, err := time.LoadLocation("America/Los_Angeles")
if err != nil {
return fmt.Errorf("Error loading location: %s", err)
}
date := now.In(loc)

now := time.Now().In(loc)
year, month, day := now.Date()

customDate := args[0]
// check if a specific date is provided
if customDate != "" {
parsedDate, err := time.Parse("2006-01-02", customDate) // input format YYYY-MM-DD
// Set the time to 7pm PT
date = time.Date(parsedDate.Year(), parsedDate.Month(), parsedDate.Day(), 19, 0, 0, 0, loc)
if err != nil {
return fmt.Errorf("invalid input time format: %w", err)
}
year, month, day = parsedDate.Date()
}

// Set the time to 7pm PT
date := time.Date(year, month, day, 19, 0, 0, 0, loc)

return execCollectNightlyTestStatus(date, tc, gcs)
},
}
Expand Down Expand Up @@ -134,10 +137,31 @@ func execCollectNightlyTestStatus(now time.Time, tc TeamcityClient, gcs Cloudsto
}

func createTestReport(pVersion provider.Version, tc TeamcityClient, gcs CloudstorageClient, formattedStartCut, formattedFinishCut, date string) error {

// Check Queued Builds
queuedBuilds, err := tc.GetBuilds("queued", pVersion.TeamCityNightlyProjectName(), formattedFinishCut, formattedStartCut)
if err != nil {
return fmt.Errorf("failed to get queued builds: %w", err)
}
if len(queuedBuilds.Builds) > 0 {
fmt.Printf("%s Test unfinished: there are still %d builds queued.\n", strings.ToUpper(pVersion.String()), len(queuedBuilds.Builds))
return nil
}

// Check Running Builds
runningBuilds, err := tc.GetBuilds("running", pVersion.TeamCityNightlyProjectName(), formattedFinishCut, formattedStartCut)
if err != nil {
return fmt.Errorf("failed to get running builds: %w", err)
}
if len(runningBuilds.Builds) > 0 {
fmt.Printf("%s Test unfinished: there are still %d builds running.\n", strings.ToUpper(pVersion.String()), len(runningBuilds.Builds))
return nil
}

// Get all service test builds
builds, err := tc.GetBuilds(pVersion.TeamCityNightlyProjectName(), formattedFinishCut, formattedStartCut)
builds, err := tc.GetBuilds("finished", pVersion.TeamCityNightlyProjectName(), formattedFinishCut, formattedStartCut)
if err != nil {
return err
return fmt.Errorf("failed to get finished builds: %w", err)
}

var testInfoList []TestInfo
Expand Down
2 changes: 1 addition & 1 deletion .ci/magician/cmd/interfaces.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ type CloudstorageClient interface {
}

type TeamcityClient interface {
GetBuilds(project, finishCut, startCut string) (teamcity.Builds, error)
GetBuilds(state, project, finishCut, startCut string) (teamcity.Builds, error)
GetTestResults(build teamcity.Build) (teamcity.TestResults, error)
}

Expand Down
4 changes: 2 additions & 2 deletions .ci/magician/teamcity/get.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ type FirstFailed struct {
Href string `json:"href"`
}

func (tc *Client) GetBuilds(project, finishCut, startCut string) (Builds, error) {
url := fmt.Sprintf("https://hashicorp.teamcity.com/app/rest/builds?locator=count:500,tag:cron-trigger,project:%s,branch:refs/heads/nightly-test,queuedDate:(date:%s,condition:before),queuedDate:(date:%s,condition:after)&fields=build(id,buildTypeId,buildConfName,webUrl,number,queuedDate,startDate,finishDate)", project, finishCut, startCut)
func (tc *Client) GetBuilds(state, project, finishCut, startCut string) (Builds, error) {
url := fmt.Sprintf("https://hashicorp.teamcity.com/app/rest/builds?locator=state:%s,count:500,tag:cron-trigger,project:%s,branch:refs/heads/nightly-test,queuedDate:(date:%s,condition:before),queuedDate:(date:%s,condition:after)&fields=build(id,buildTypeId,buildConfName,webUrl,number,queuedDate,startDate,finishDate)", state, project, finishCut, startCut)

var builds Builds

Expand Down
Loading