Skip to content

Commit 35b3c4a

Browse files
committed
Fix race attribution for Flakeguard
1 parent abd0c96 commit 35b3c4a

File tree

6 files changed

+196
-198
lines changed

6 files changed

+196
-198
lines changed

tools/flakeguard/cmd/get_gh_artifact_link.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import (
77
"strings"
88
"time"
99

10-
"github.com/google/go-github/v67/github"
10+
"github.com/google/go-github/v70/github"
1111
"github.com/rs/zerolog/log"
1212
"github.com/spf13/cobra"
1313
"golang.org/x/oauth2"

tools/flakeguard/go.mod

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
module github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard
22

3-
go 1.24.1
3+
go 1.24.2
44

55
require (
66
github.com/andygrunwald/go-jira v1.16.0
77
github.com/briandowns/spinner v1.23.2
88
github.com/charmbracelet/bubbletea v1.3.4
99
github.com/charmbracelet/lipgloss v1.1.0
1010
github.com/go-resty/resty/v2 v2.16.5
11-
github.com/google/go-github/v67 v67.0.0
11+
github.com/google/go-github/v70 v70.0.0
1212
github.com/google/uuid v1.6.0
1313
github.com/rs/zerolog v1.34.0
1414
github.com/spf13/cobra v1.9.1
@@ -28,7 +28,6 @@ require (
2828
github.com/fatih/color v1.7.0 // indirect
2929
github.com/fatih/structs v1.1.0 // indirect
3030
github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
31-
github.com/google/go-cmp v0.7.0 // indirect
3231
github.com/google/go-querystring v1.1.0 // indirect
3332
github.com/inconshreveable/mousetrap v1.1.0 // indirect
3433
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect

tools/flakeguard/go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
3535
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
3636
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
3737
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
38-
github.com/google/go-github/v67 v67.0.0 h1:g11NDAmfaBaCO8qYdI9fsmbaRipHNWRIU/2YGvlh4rg=
39-
github.com/google/go-github/v67 v67.0.0/go.mod h1:zH3K7BxjFndr9QSeFibx4lTKkYS3K9nDanoI1NjaOtY=
38+
github.com/google/go-github/v70 v70.0.0 h1:/tqCp5KPrcvqCc7vIvYyFYTiCGrYvaWoYMGHSQbo55o=
39+
github.com/google/go-github/v70 v70.0.0/go.mod h1:xBUZgo8MI3lUL/hwxl3hlceJW1U8MVnXP3zUyI+rhQY=
4040
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
4141
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
4242
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=

tools/flakeguard/runner/example_test_package/example_tests_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ func TestFail(t *testing.T) {
2020

2121
func TestFailLargeOutput(t *testing.T) {
2222
t.Parallel()
23-
for i := 0; i < 1000; i++ {
23+
for range 1000 {
2424
t.Log("This is a log line")
2525
}
2626
t.Fatal("This test always fails")

tools/flakeguard/runner/runner.go

Lines changed: 31 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -375,12 +375,13 @@ func (r *Runner) parseTestResults(runPrefix string, runCount int) ([]reports.Tes
375375
}
376376
}
377377

378+
// TODO: An argument could be made to check for entryLine.Action != "output" instead, but need to check edge cases
378379
if (panicDetectionMode || raceDetectionMode) && entryLine.Action == "fail" { // End of panic or race output
380+
var outputs []string
381+
for _, entry := range detectedEntries {
382+
outputs = append(outputs, entry.Output)
383+
}
379384
if panicDetectionMode {
380-
var outputs []string
381-
for _, entry := range detectedEntries {
382-
outputs = append(outputs, entry.Output)
383-
}
384385
panicTest, timeout, err := attributePanicToTest(outputs)
385386
if err != nil {
386387
log.Warn().Err(err).Msg("Unable to attribute panic to a test")
@@ -418,9 +419,10 @@ func (r *Runner) parseTestResults(runPrefix string, runCount int) ([]reports.Tes
418419
}
419420
}
420421
} else if raceDetectionMode {
421-
raceTest, err := attributeRaceToTest(entryLine.Package, detectedEntries)
422+
raceTest, err := attributeRaceToTest(outputs)
422423
if err != nil {
423-
return nil, err
424+
log.Warn().Err(err).Msg("Unable to attribute race to a test")
425+
raceTest = "UnableToAttributeRaceTestPleaseInvestigate"
424426
}
425427
raceTestKey := fmt.Sprintf("%s/%s", entryLine.Package, raceTest)
426428

@@ -612,42 +614,46 @@ func (r *Runner) transformTestOutputFiles(filePaths []string) error {
612614
return nil
613615
}
614616

617+
var (
618+
// Regex to extract a valid test function name from a panic or race message, e.g.
619+
// github.com/smartcontractkit/chainlink/deployment/keystone/changeset_test.TestDeployBalanceReader(0xc000583c00)
620+
nestedTestNameRe = regexp.MustCompile(`\.(Test[A-Z]\w+)(?:\.[^(]+)?\s*\(`)
621+
622+
ErrFailedToAttributePanic = errors.New("failed to attribute panic to test")
623+
ErrFailedToAttributeRace = errors.New("failed to attribute race to test")
624+
)
625+
615626
// attributePanicToTest properly attributes panics to the test that caused them.
616627
func attributePanicToTest(outputs []string) (test string, timeout bool, err error) {
617-
// Regex to extract a valid test function name from a panic message, e.g.
618-
// github.com/smartcontractkit/chainlink/deployment/keystone/changeset_test.TestDeployBalanceReader(0xc000583c00)
619-
nestedTestNameRe := regexp.MustCompile(`\.(Test[A-Z]\w+)(?:\.[^(]+)?\s*\(`)
620628
// Regex to extract a valid test function name from a panic message if the panic is a timeout, e.g.
621629
// TestTimedOut (10m0s)
622630
timedOutTestNamRe := regexp.MustCompile(`^(Test\w+)\W+\(.*\)$`)
623631

624-
for _, o := range outputs {
625-
outputs = append(outputs, o)
626-
matchNestedTestName := nestedTestNameRe.FindStringSubmatch(o)
627-
matchTimedOutTestName := timedOutTestNamRe.FindStringSubmatch(o)
632+
for _, output := range outputs {
633+
matchNestedTestName := nestedTestNameRe.FindStringSubmatch(output)
634+
matchTimedOutTestName := timedOutTestNamRe.FindStringSubmatch(output)
628635
if len(matchNestedTestName) > 1 {
629636
return strings.TrimSpace(matchNestedTestName[1]), false, nil
630637
} else if len(matchTimedOutTestName) > 1 {
631638
return strings.TrimSpace(matchTimedOutTestName[1]), true, nil
632639
}
633640
}
634-
return "", false, fmt.Errorf("failed to attribute panic to test, using regex '%s' and '%s' on these strings:\n\n%s",
635-
nestedTestNameRe.String(), timedOutTestNamRe.String(), strings.Join(outputs, ""))
641+
return "", false, fmt.Errorf("%w, using regex '%s' and '%s' on these strings:\n\n%s",
642+
ErrFailedToAttributePanic, nestedTestNameRe.String(), timedOutTestNamRe.String(), strings.Join(outputs, ""),
643+
)
636644
}
637645

638646
// attributeRaceToTest properly attributes races to the test that caused them.
639-
func attributeRaceToTest(racePackage string, raceEntries []entry) (string, error) {
640-
regexSanitizeRacePackage := filepath.Base(racePackage)
641-
raceAttributionRe := regexp.MustCompile(fmt.Sprintf(`%s\.(Test[^\.\(]+)`, regexSanitizeRacePackage))
642-
entriesOutputs := []string{}
643-
for _, entry := range raceEntries {
644-
entriesOutputs = append(entriesOutputs, entry.Output)
645-
if matches := raceAttributionRe.FindStringSubmatch(entry.Output); len(matches) > 1 {
646-
testName := strings.TrimSpace(matches[1])
647-
return testName, nil
647+
func attributeRaceToTest(outputs []string) (string, error) {
648+
for _, output := range outputs {
649+
match := nestedTestNameRe.FindStringSubmatch(output)
650+
if len(match) > 1 {
651+
return strings.TrimSpace(match[1]), nil
648652
}
649653
}
650-
return "", fmt.Errorf("failed to attribute race to test, using regex: %s on these strings:\n%s", raceAttributionRe.String(), strings.Join(entriesOutputs, ""))
654+
return "", fmt.Errorf("%w, using regex '%s' on these strings:\n\n%s",
655+
ErrFailedToAttributeRace, nestedTestNameRe.String(), strings.Join(outputs, ""),
656+
)
651657
}
652658

653659
// parseSubTest checks if a test name is a subtest and returns the parent and sub names.

0 commit comments

Comments
 (0)