Skip to content

Commit 5a5aff2

Browse files
lukaszcllukaszcl
authored
TT-1956 Add support for e2e docker tests in Flakeguard (#1627)
* Add support for custom test commands in RunTestsCmd * Update custom test command example in RunTestsCmd help message * fix * Enhance artifact link fetching by adding artifact name check and adjusting pagination options * Add retry mechanism for fetching artifact links and increase pagination limit * Enhance artifact link fetching by implementing pagination for artifact retrieval * Refactor settings table generation to conditionally include project name * Refactor * fix * Fix * Use named return values in runCmd
1 parent 46b92fb commit 5a5aff2

File tree

5 files changed

+247
-111
lines changed

5 files changed

+247
-111
lines changed

‎tools/flakeguard/cmd/generate_report.go‎

Lines changed: 52 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -76,17 +76,13 @@ var GenerateReportCmd = &cobra.Command{
7676
hasFailedTests := aggregatedReport.SummaryData.FailedRuns > 0
7777

7878
var artifactLink string
79-
if hasFailedTests {
79+
if hasFailedTests && githubRepo != "" && githubRunID != 0 && artifactName != "" {
8080
// Fetch artifact link from GitHub API
81-
artifactLink, err = fetchArtifactLink(githubToken, githubRepo, githubRunID, artifactName)
81+
artifactLink, err = fetchArtifactLinkWithRetry(githubToken, githubRepo, githubRunID, artifactName, 5, 5*time.Second)
8282
if err != nil {
8383
log.Error().Err(err).Msg("Error fetching artifact link")
8484
os.Exit(ErrorExitCode)
8585
}
86-
} else {
87-
// No failed tests, set artifactLink to empty string
88-
artifactLink = ""
89-
log.Debug().Msg("No failed tests found. Skipping artifact link generation")
9086
}
9187

9288
// Create output directory if it doesn't exist
@@ -194,54 +190,84 @@ func init() {
194190
log.Error().Err(err).Msg("Error marking flag as required")
195191
os.Exit(ErrorExitCode)
196192
}
197-
if err := GenerateReportCmd.MarkFlagRequired("github-repository"); err != nil {
198-
log.Error().Err(err).Msg("Error marking flag as required")
199-
os.Exit(ErrorExitCode)
200-
}
201-
if err := GenerateReportCmd.MarkFlagRequired("github-run-id"); err != nil {
202-
log.Error().Err(err).Msg("Error marking flag as required")
203-
os.Exit(ErrorExitCode)
204-
}
205193
}
206194

207195
func fetchArtifactLink(githubToken, githubRepo string, githubRunID int64, artifactName string) (string, error) {
208196
if githubToken == exampleGitHubToken {
209197
return "https://example-artifact-link.com", nil
210198
}
211199
ctx := context.Background()
212-
ts := oauth2.StaticTokenSource(
213-
&oauth2.Token{AccessToken: githubToken},
214-
)
200+
ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: githubToken})
215201
tc := oauth2.NewClient(ctx, ts)
216202
client := github.NewClient(tc)
217203

218-
// Split the repository into owner and repo
204+
// Split owner/repo
219205
repoParts := strings.SplitN(githubRepo, "/", 2)
220206
if len(repoParts) != 2 {
221207
return "", fmt.Errorf("invalid format for --github-repository, expected owner/repo")
222208
}
223209
owner, repo := repoParts[0], repoParts[1]
224210

225-
// List artifacts for the workflow run
226-
opts := &github.ListOptions{PerPage: 100}
227-
artifacts, _, err := client.Actions.ListWorkflowRunArtifacts(ctx, owner, repo, githubRunID, opts)
228-
if err != nil {
229-
return "", fmt.Errorf("error listing artifacts: %w", err)
211+
opts := &github.ListOptions{PerPage: 100} // The max GitHub allows is 100 per page
212+
var allArtifacts []*github.Artifact
213+
214+
// Paginate through all artifacts
215+
for {
216+
artifacts, resp, err := client.Actions.ListWorkflowRunArtifacts(ctx, owner, repo, githubRunID, opts)
217+
if err != nil {
218+
return "", fmt.Errorf("error listing artifacts: %w", err)
219+
}
220+
221+
allArtifacts = append(allArtifacts, artifacts.Artifacts...)
222+
223+
if resp.NextPage == 0 {
224+
// No more pages
225+
break
226+
}
227+
// Move to the next page
228+
opts.Page = resp.NextPage
230229
}
231230

232231
// Find the artifact
233-
for _, artifact := range artifacts.Artifacts {
232+
for _, artifact := range allArtifacts {
234233
if artifact.GetName() == artifactName {
235-
// Construct the artifact URL using the artifact ID
236234
artifactID := artifact.GetID()
237-
artifactURL := fmt.Sprintf("https://github.com/%s/%s/actions/runs/%d/artifacts/%d", owner, repo, githubRunID, artifactID)
235+
artifactURL := fmt.Sprintf("https://github.com/%s/%s/actions/runs/%d/artifacts/%d",
236+
owner, repo, githubRunID, artifactID)
238237
return artifactURL, nil
239238
}
240239
}
241240

242241
return "", fmt.Errorf("artifact '%s' not found in the workflow run", artifactName)
243242
}
244243

244+
func fetchArtifactLinkWithRetry(
245+
githubToken, githubRepo string,
246+
githubRunID int64, artifactName string,
247+
maxRetries int, delay time.Duration,
248+
) (string, error) {
249+
var lastErr error
250+
for attempt := 1; attempt <= maxRetries; attempt++ {
251+
link, err := fetchArtifactLink(githubToken, githubRepo, githubRunID, artifactName)
252+
if err == nil {
253+
// Found the artifact link successfully
254+
return link, nil
255+
}
256+
257+
// If this was our last attempt, return the error
258+
lastErr = err
259+
if attempt == maxRetries {
260+
break
261+
}
262+
263+
// Otherwise wait and retry
264+
log.Printf("[Attempt %d/%d] Artifact not yet available. Retrying in %s...", attempt, maxRetries, delay)
265+
time.Sleep(delay)
266+
}
267+
268+
return "", fmt.Errorf("failed to fetch artifact link after %d retries: %w", maxRetries, lastErr)
269+
}
270+
245271
func generateGitHubSummaryMarkdown(report *reports.TestReport, outputPath, artifactLink, artifactName string) error {
246272
fs := reports.OSFileSystem{}
247273
mdFileName := outputPath + "-summary.md"

‎tools/flakeguard/cmd/run.go‎

Lines changed: 32 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ var RunTestsCmd = &cobra.Command{
2929
projectPath, _ := cmd.Flags().GetString("project-path")
3030
testPackagesJson, _ := cmd.Flags().GetString("test-packages-json")
3131
testPackagesArg, _ := cmd.Flags().GetStringSlice("test-packages")
32+
testCmdStrings, _ := cmd.Flags().GetStringArray("test-cmd")
3233
runCount, _ := cmd.Flags().GetInt("run-count")
3334
timeout, _ := cmd.Flags().GetDuration("timeout")
3435
tags, _ := cmd.Flags().GetStringArray("tags")
@@ -61,16 +62,19 @@ var RunTestsCmd = &cobra.Command{
6162

6263
// Determine test packages
6364
var testPackages []string
64-
if testPackagesJson != "" {
65-
if err := json.Unmarshal([]byte(testPackagesJson), &testPackages); err != nil {
66-
log.Error().Err(err).Msg("Error decoding test packages JSON")
65+
if len(testCmdStrings) == 0 {
66+
// No custom command -> parse packages
67+
if testPackagesJson != "" {
68+
if err := json.Unmarshal([]byte(testPackagesJson), &testPackages); err != nil {
69+
log.Error().Err(err).Msg("Error decoding test packages JSON")
70+
os.Exit(ErrorExitCode)
71+
}
72+
} else if len(testPackagesArg) > 0 {
73+
testPackages = testPackagesArg
74+
} else {
75+
log.Error().Msg("Error: must specify either --test-packages-json or --test-packages")
6776
os.Exit(ErrorExitCode)
6877
}
69-
} else if len(testPackagesArg) > 0 {
70-
testPackages = testPackagesArg
71-
} else {
72-
log.Error().Msg("Error: must specify either --test-packages-json or --test-packages")
73-
os.Exit(ErrorExitCode)
7478
}
7579

7680
// Initialize the runner
@@ -83,18 +87,30 @@ var RunTestsCmd = &cobra.Command{
8387
UseRace: useRace,
8488
SkipTests: skipTests,
8589
SelectTests: selectTests,
86-
SelectedTestPackages: testPackages,
8790
UseShuffle: useShuffle,
8891
ShuffleSeed: shuffleSeed,
8992
OmitOutputsOnSuccess: omitOutputsOnSuccess,
9093
MaxPassRatio: maxPassRatio,
9194
}
9295

9396
// Run the tests
94-
testReport, err := testRunner.RunTests()
95-
if err != nil {
96-
log.Error().Err(err).Msg("Error running tests")
97-
os.Exit(ErrorExitCode)
97+
var (
98+
testReport *reports.TestReport
99+
)
100+
101+
if len(testCmdStrings) > 0 {
102+
testReport, err = testRunner.RunTestCmd(testCmdStrings)
103+
if err != nil {
104+
log.Fatal().Err(err).Msg("Error running custom test command")
105+
os.Exit(ErrorExitCode)
106+
}
107+
} else {
108+
// Otherwise, use the normal go test approach
109+
testReport, err = testRunner.RunTestPackages(testPackages)
110+
if err != nil {
111+
log.Fatal().Err(err).Msg("Error running test packages")
112+
os.Exit(ErrorExitCode)
113+
}
98114
}
99115

100116
// Save the test results in JSON format
@@ -148,6 +164,9 @@ func init() {
148164
RunTestsCmd.Flags().StringP("project-path", "r", ".", "The path to the Go project. Default is the current directory. Useful for subprojects")
149165
RunTestsCmd.Flags().String("test-packages-json", "", "JSON-encoded string of test packages")
150166
RunTestsCmd.Flags().StringSlice("test-packages", nil, "Comma-separated list of test packages to run")
167+
RunTestsCmd.Flags().StringArray("test-cmd", nil,
168+
"Optional custom test command (e.g. 'go test -json github.com/smartcontractkit/chainlink/integration-tests/smoke -v -run TestForwarderOCR2Basic'), which must produce go test -json output.",
169+
)
151170
RunTestsCmd.Flags().Bool("run-all-packages", false, "Run all test packages in the project. This flag overrides --test-packages and --test-packages-json")
152171
RunTestsCmd.Flags().IntP("run-count", "c", 1, "Number of times to run the tests")
153172
RunTestsCmd.Flags().Duration("timeout", 0, "Passed on to the 'go test' command as the -timeout flag")

‎tools/flakeguard/reports/presentation.go‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -155,17 +155,23 @@ func GeneratePRCommentMarkdown(
155155
func buildSettingsTable(testReport *TestReport, maxPassRatio float64) [][]string {
156156
rows := [][]string{
157157
{"**Setting**", "**Value**"},
158-
{"Project", testReport.GoProject},
159-
{"Max Pass Ratio", fmt.Sprintf("%.2f%%", maxPassRatio*100)},
160-
{"Test Run Count", fmt.Sprintf("%d", testReport.SummaryData.TestRunCount)},
161-
{"Race Detection", fmt.Sprintf("%t", testReport.RaceDetection)},
162158
}
159+
160+
if testReport.GoProject != "" {
161+
rows = append(rows, []string{"Project", testReport.GoProject})
162+
}
163+
164+
rows = append(rows, []string{"Max Pass Ratio", fmt.Sprintf("%.2f%%", maxPassRatio*100)})
165+
rows = append(rows, []string{"Test Run Count", fmt.Sprintf("%d", testReport.SummaryData.TestRunCount)})
166+
rows = append(rows, []string{"Race Detection", fmt.Sprintf("%t", testReport.RaceDetection)})
167+
163168
if len(testReport.ExcludedTests) > 0 {
164169
rows = append(rows, []string{"Excluded Tests", strings.Join(testReport.ExcludedTests, ", ")})
165170
}
166171
if len(testReport.SelectedTests) > 0 {
167172
rows = append(rows, []string{"Selected Tests", strings.Join(testReport.SelectedTests, ", ")})
168173
}
174+
169175
return rows
170176
}
171177

‎tools/flakeguard/runner/runner.go‎

Lines changed: 100 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,18 +38,17 @@ type Runner struct {
3838
FailFast bool // Stop on first test failure.
3939
SkipTests []string // Test names to exclude.
4040
SelectTests []string // Test names to include.
41-
SelectedTestPackages []string // Explicitly selected packages to run.
4241
CollectRawOutput bool // Set to true to collect test output for later inspection.
4342
OmitOutputsOnSuccess bool // Set to true to omit test outputs on success.
4443
MaxPassRatio float64 // Maximum pass ratio threshold for a test to be considered flaky.
4544
rawOutputs map[string]*bytes.Buffer
4645
}
4746

48-
// RunTests executes the tests for each provided package and aggregates all results.
47+
// RunTestPackages executes the tests for each provided package and aggregates all results.
4948
// It returns all test results and any error encountered during testing.
50-
func (r *Runner) RunTests() (*reports.TestReport, error) {
49+
func (r *Runner) RunTestPackages(packages []string) (*reports.TestReport, error) {
5150
var jsonFilePaths []string
52-
for _, p := range r.SelectedTestPackages {
51+
for _, p := range packages {
5352
for i := 0; i < r.RunCount; i++ {
5453
if r.CollectRawOutput { // Collect raw output for debugging
5554
if r.rawOutputs == nil {
@@ -61,7 +60,7 @@ func (r *Runner) RunTests() (*reports.TestReport, error) {
6160
separator := strings.Repeat("-", 80)
6261
r.rawOutputs[p].WriteString(fmt.Sprintf("Run %d\n%s\n", i+1, separator))
6362
}
64-
jsonFilePath, passed, err := r.runTests(p)
63+
jsonFilePath, passed, err := r.runTestPackage(p)
6564
if err != nil {
6665
return nil, fmt.Errorf("failed to run tests in package %s: %w", p, err)
6766
}
@@ -88,6 +87,41 @@ func (r *Runner) RunTests() (*reports.TestReport, error) {
8887
return report, nil
8988
}
9089

90+
// RunTestCmd runs an arbitrary command testCmd (like ["go", "run", "my_test.go", ...])
91+
// that produces the same JSON lines that 'go test -json' would produce on stdout.
92+
// It captures those lines in a temp file, then parses them for pass/fail/panic/race data.
93+
func (r *Runner) RunTestCmd(testCmd []string) (*reports.TestReport, error) {
94+
var jsonFilePaths []string
95+
96+
// Run the command r.RunCount times
97+
for i := 0; i < r.RunCount; i++ {
98+
jsonFilePath, passed, err := r.runCmd(testCmd, i)
99+
if err != nil {
100+
return nil, fmt.Errorf("failed to run test command: %w", err)
101+
}
102+
jsonFilePaths = append(jsonFilePaths, jsonFilePath)
103+
if !passed && r.FailFast {
104+
break
105+
}
106+
}
107+
108+
results, err := r.parseTestResults(jsonFilePaths)
109+
if err != nil {
110+
return nil, fmt.Errorf("failed to parse test results: %w", err)
111+
}
112+
113+
report := &reports.TestReport{
114+
GoProject: r.prettyProjectPath,
115+
RaceDetection: r.UseRace,
116+
ExcludedTests: r.SkipTests,
117+
SelectedTests: r.SelectTests,
118+
Results: results,
119+
MaxPassRatio: r.MaxPassRatio,
120+
}
121+
report.GenerateSummaryData()
122+
return report, nil
123+
}
124+
91125
// RawOutputs retrieves the raw output from the test runs, if CollectRawOutput enabled.
92126
// packageName : raw output
93127
func (r *Runner) RawOutputs() map[string]*bytes.Buffer {
@@ -98,8 +132,8 @@ type exitCoder interface {
98132
ExitCode() int
99133
}
100134

101-
// runTests runs the tests for a given package and returns the path to the output file.
102-
func (r *Runner) runTests(packageName string) (string, bool, error) {
135+
// runTestPackage runs the tests for a given package and returns the path to the output file.
136+
func (r *Runner) runTestPackage(packageName string) (string, bool, error) {
103137
args := []string{"test", packageName, "-json", "-count=1"}
104138
if r.UseRace {
105139
args = append(args, "-race")
@@ -164,6 +198,65 @@ func (r *Runner) runTests(packageName string) (string, bool, error) {
164198
return tmpFile.Name(), true, nil // Test succeeded
165199
}
166200

201+
// runCmd runs the user-supplied command once, captures its JSON output,
202+
// and returns the temp file path, whether the test passed, and an error if any.
203+
func (r *Runner) runCmd(testCmd []string, runIndex int) (tempFilePath string, passed bool, err error) {
204+
// Create temp file for JSON output
205+
tmpFile, err := os.CreateTemp("", fmt.Sprintf("test-output-cmd-run%d-*.json", runIndex+1))
206+
if err != nil {
207+
err = fmt.Errorf("failed to create temp file: %w", err)
208+
return
209+
}
210+
defer tmpFile.Close()
211+
212+
if r.Verbose {
213+
log.Info().Msgf("Running custom test command (%d/%d): %s", runIndex+1, r.RunCount, strings.Join(testCmd, " "))
214+
}
215+
216+
cmd := exec.Command(testCmd[0], testCmd[1:]...)
217+
cmd.Dir = r.ProjectPath
218+
219+
// If collecting raw output, write to both file & buffer
220+
if r.CollectRawOutput {
221+
if r.rawOutputs == nil {
222+
r.rawOutputs = make(map[string]*bytes.Buffer)
223+
}
224+
key := fmt.Sprintf("customCmd-run%d", runIndex+1)
225+
if _, exists := r.rawOutputs[key]; !exists {
226+
r.rawOutputs[key] = &bytes.Buffer{}
227+
}
228+
cmd.Stdout = io.MultiWriter(tmpFile, r.rawOutputs[key])
229+
} else {
230+
cmd.Stdout = tmpFile
231+
}
232+
cmd.Stderr = os.Stderr
233+
234+
err = cmd.Run()
235+
236+
tempFilePath = tmpFile.Name()
237+
238+
// Determine pass/fail from exit code
239+
type exitCoder interface {
240+
ExitCode() int
241+
}
242+
var ec exitCoder
243+
if errors.As(err, &ec) {
244+
// Non-zero exit code => test failure
245+
passed = ec.ExitCode() == 0
246+
err = nil // Clear error since we handled it
247+
return
248+
} else if err != nil {
249+
// Some other error that doesn't implement ExitCode() => real error
250+
tempFilePath = ""
251+
err = fmt.Errorf("error running test command: %w", err)
252+
return
253+
}
254+
255+
// Otherwise, test passed
256+
passed = true
257+
return
258+
}
259+
167260
type entry struct {
168261
Action string `json:"Action"`
169262
Test string `json:"Test"`

0 commit comments

Comments
 (0)