Skip to content

Commit 31eb9ae

Browse files
kalverrakalverra
committed
Handles timeouts
1 parent af77c7b commit 31eb9ae

File tree

6 files changed

+136
-111
lines changed

6 files changed

+136
-111
lines changed

‎tools/flakeguard/Makefile‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,16 +12,25 @@ test:
1212
example:
1313
rm -rf example_results
1414
mkdir -p example_results
15-
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_1.json
16-
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
17-
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
15+
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic,Timeout --max-pass-ratio=1 --race=false --output-json=example_results/example_run_1.json
16+
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic,Timeout --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
17+
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic,Timeout --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
1818
go run . aggregate-results --results-path ./example_results --output-results ./example_results/all_tests_example.json
1919

20-
.PHONY: example_panic
21-
example_panic:
20+
.PHONY: example_flaky_panic
21+
example_flaky_panic:
2222
rm -rf example_results
2323
mkdir -p example_results
2424
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_1.json
2525
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
2626
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
27+
go run . aggregate-results --results-path ./example_results --output-results ./example_results/all_tests_example.json
28+
29+
.PHONY: example_timeout
30+
example_timeout:
31+
rm -rf example_results
32+
mkdir -p example_results
33+
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --select-tests=TestTimeout --timeout=1s --max-pass-ratio=1 --race=false --output-json=example_results/example_run_1.json
34+
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --select-tests=TestTimeout --timeout=1s --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
35+
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --select-tests=TestTimeout --timeout=1s --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
2736
go run . aggregate-results --results-path ./example_results --output-results ./example_results/all_tests_example.json

‎tools/flakeguard/cmd/run.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ var RunTestsCmd = &cobra.Command{
2121
testPackagesJson, _ := cmd.Flags().GetString("test-packages-json")
2222
testPackagesArg, _ := cmd.Flags().GetStringSlice("test-packages")
2323
runCount, _ := cmd.Flags().GetInt("run-count")
24+
timeout, _ := cmd.Flags().GetDuration("timeout")
25+
tags, _ := cmd.Flags().GetStringArray("tags")
2426
useRace, _ := cmd.Flags().GetBool("race")
2527
outputPath, _ := cmd.Flags().GetString("output-json")
2628
maxPassRatio, _ := cmd.Flags().GetFloat64("max-pass-ratio")
@@ -50,6 +52,8 @@ var RunTestsCmd = &cobra.Command{
5052
ProjectPath: projectPath,
5153
Verbose: true,
5254
RunCount: runCount,
55+
Timeout: timeout,
56+
Tags: tags,
5357
UseRace: useRace,
5458
SkipTests: skipTests,
5559
SelectTests: selectTests,
@@ -98,6 +102,8 @@ func init() {
98102
RunTestsCmd.Flags().StringSlice("test-packages", nil, "Comma-separated list of test packages to run")
99103
RunTestsCmd.Flags().Bool("run-all-packages", false, "Run all test packages in the project. This flag overrides --test-packages and --test-packages-json")
100104
RunTestsCmd.Flags().IntP("run-count", "c", 1, "Number of times to run the tests")
105+
RunTestsCmd.Flags().Duration("timeout", 0, "Passed on to the 'go test' command as the -timeout flag")
106+
RunTestsCmd.Flags().StringArray("tags", nil, "Passed on to the 'go test' command as the -tags flag")
101107
RunTestsCmd.Flags().Bool("race", false, "Enable the race detector")
102108
RunTestsCmd.Flags().Bool("shuffle", false, "Enable test shuffling")
103109
RunTestsCmd.Flags().String("shuffle-seed", "", "Set seed for test shuffling. Must be used with --shuffle")

‎tools/flakeguard/reports/reports.go‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"encoding/json"
66
"fmt"
77
"io"
8+
"math"
89
"os"
910
"path/filepath"
1011
"sort"
@@ -18,6 +19,7 @@ type TestReport struct {
1819
TestRunCount int
1920
RaceDetection bool
2021
ExcludedTests []string
22+
SelectedTests []string
2123
Results []TestResult
2224
}
2325

@@ -27,6 +29,7 @@ type TestResult struct {
2729
TestPackage string
2830
PackagePanic bool // Indicates a package-level panic
2931
Panic bool // Indicates a test-level panic
32+
Timeout bool // Indicates if the test timed out
3033
Race bool // Indicates if the test caused a data race
3134
Skipped bool // Indicates if the test was skipped
3235
PassRatio float64 // Pass ratio in decimal format like 0.5
@@ -91,6 +94,7 @@ func AggregateTestResults(folderPath string) (*TestReport, error) {
9194
testMap = make(map[string]TestResult)
9295
fullReport = &TestReport{}
9396
excludedTests = map[string]struct{}{}
97+
selectedTests = map[string]struct{}{}
9498
)
9599

96100
// Read all JSON files in the folder
@@ -118,6 +122,9 @@ func AggregateTestResults(folderPath string) (*TestReport, error) {
118122
for _, test := range report.ExcludedTests {
119123
excludedTests[test] = struct{}{}
120124
}
125+
for _, test := range report.SelectedTests {
126+
selectedTests[test] = struct{}{}
127+
}
121128
// Process each test results
122129
for _, result := range report.Results {
123130
// Unique key for each test based on TestName and TestPackage
@@ -157,6 +164,9 @@ func AggregateTestResults(folderPath string) (*TestReport, error) {
157164
for test := range excludedTests {
158165
fullReport.ExcludedTests = append(fullReport.ExcludedTests, test)
159166
}
167+
for test := range selectedTests {
168+
fullReport.SelectedTests = append(fullReport.SelectedTests, test)
169+
}
160170

161171
var (
162172
aggregatedResults = make([]TestResult, 0, len(testMap))
@@ -185,6 +195,7 @@ func PrintTests(
185195
"**Pass Ratio**",
186196
"**Runs**",
187197
"**Test Panicked?**",
198+
"**Test Timed Out?**",
188199
"**Test Race?**",
189200
"**Successes**",
190201
"**Failures**",
@@ -203,6 +214,7 @@ func PrintTests(
203214
fmt.Sprintf("%.2f%%", test.PassRatio*100),
204215
fmt.Sprintf("%d", test.Runs),
205216
fmt.Sprintf("%t", test.Panic),
217+
fmt.Sprintf("%t", test.Timeout),
206218
fmt.Sprintf("%t", test.Race),
207219
fmt.Sprintf("%d", test.Successes),
208220
fmt.Sprintf("%d", test.Failures),
@@ -228,14 +240,23 @@ func PrintTests(
228240
}
229241
}
230242

243+
var passRatioStr string
244+
if runs == 0 || passes == runs {
245+
passRatioStr = "100%"
246+
} else {
247+
percentage := float64(passes) / float64(runs) * 100
248+
truncatedPercentage := math.Floor(percentage*100) / 100 // Truncate to 2 decimal places
249+
passRatioStr = fmt.Sprintf("%.2f%%", truncatedPercentage)
250+
}
251+
231252
// Print out summary data
232253
summaryData := [][]string{
233254
{"**Category**", "**Total**"},
234255
{"**Tests**", fmt.Sprint(len(tests))},
235256
{"**Panicked Tests**", fmt.Sprint(panickedTests)},
236257
{"**Raced Tests**", fmt.Sprint(racedTests)},
237258
{"**Flaky Tests**", fmt.Sprint(flakyTests)},
238-
{"**Pass Ratio**", fmt.Sprintf("%.2f%%", float64(passes)/float64(runs)*100)},
259+
{"**Pass Ratio**", passRatioStr},
239260
{"**Runs**", fmt.Sprint(runs)},
240261
{"**Passes**", fmt.Sprint(passes)},
241262
{"**Failures**", fmt.Sprint(fails)},
@@ -320,7 +341,12 @@ func MarkdownSummary(w io.Writer, testReport *TestReport, maxPassRatio float64)
320341
{"Max Pass Ratio", fmt.Sprintf("%.2f%%", maxPassRatio*100)},
321342
{"Test Run Count", fmt.Sprintf("%d", testReport.TestRunCount)},
322343
{"Race Detection", fmt.Sprintf("%t", testReport.RaceDetection)},
323-
{"Excluded Tests", strings.Join(testReport.ExcludedTests, ", ")},
344+
}
345+
if len(testReport.ExcludedTests) > 0 {
346+
rows = append(rows, []string{"Excluded Tests", strings.Join(testReport.ExcludedTests, ", ")})
347+
}
348+
if len(testReport.SelectedTests) > 0 {
349+
rows = append(rows, []string{"Selected Tests", strings.Join(testReport.SelectedTests, ", ")})
324350
}
325351
colWidths := make([]int, len(rows[0]))
326352

‎tools/flakeguard/runner/example_test_package/example_tests_test.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package exampletestpackage
22

33
import (
4+
"log"
45
"os"
56
"sync"
67
"testing"
8+
"time"
79
)
810

911
func TestPass(t *testing.T) {
@@ -188,3 +190,17 @@ func TestRace(t *testing.T) {
188190
// Log the result
189191
t.Logf("Final value of sharedCounter: %d", sharedCounter)
190192
}
193+
194+
func TestTimeout(t *testing.T) {
195+
t.Parallel()
196+
197+
deadline, ok := t.Deadline()
198+
if !ok {
199+
log.Fatal("This test should have a deadline")
200+
}
201+
202+
t.Logf("This test will sleep %s in order to timeout", time.Until(deadline).String())
203+
// Sleep until the deadline
204+
time.Sleep(time.Until(deadline))
205+
t.Logf("This test should have timed out")
206+
}

‎tools/flakeguard/runner/runner.go‎

Lines changed: 33 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -25,18 +25,20 @@ var (
2525
)
2626

2727
type Runner struct {
28-
ProjectPath string // Path to the Go project directory.
29-
prettyProjectPath string // Go project package path, formatted for pretty printing.
30-
Verbose bool // If true, provides detailed logging.
31-
RunCount int // Number of times to run the tests.
32-
UseRace bool // Enable race detector.
33-
UseShuffle bool // Enable test shuffling. -shuffle=on flag.
34-
ShuffleSeed string // Set seed for test shuffling -shuffle={seed} flag. Must be used with UseShuffle.
35-
FailFast bool // Stop on first test failure.
36-
SkipTests []string // Test names to exclude.
37-
SelectTests []string // Test names to include.
38-
SelectedTestPackages []string // Explicitly selected packages to run.
39-
CollectRawOutput bool // Set to true to collect test output for later inspection.
28+
ProjectPath string // Path to the Go project directory.
29+
prettyProjectPath string // Go project package path, formatted for pretty printing.
30+
Verbose bool // If true, provides detailed logging.
31+
RunCount int // Number of times to run the tests.
32+
UseRace bool // Enable race detector.
33+
Timeout time.Duration // Test timeout
34+
Tags []string // Build tags.
35+
UseShuffle bool // Enable test shuffling. -shuffle=on flag.
36+
ShuffleSeed string // Set seed for test shuffling -shuffle={seed} flag. Must be used with UseShuffle.
37+
FailFast bool // Stop on first test failure.
38+
SkipTests []string // Test names to exclude.
39+
SelectTests []string // Test names to include.
40+
SelectedTestPackages []string // Explicitly selected packages to run.
41+
CollectRawOutput bool // Set to true to collect test output for later inspection.
4042
rawOutputs map[string]*bytes.Buffer
4143
}
4244

@@ -76,6 +78,7 @@ func (r *Runner) RunTests() (*reports.TestReport, error) {
7678
TestRunCount: r.RunCount,
7779
RaceDetection: r.UseRace,
7880
ExcludedTests: r.SkipTests,
81+
SelectedTests: r.SelectTests,
7982
Results: results,
8083
}, nil
8184
}
@@ -96,6 +99,12 @@ func (r *Runner) runTests(packageName string) (string, bool, error) {
9699
if r.UseRace {
97100
args = append(args, "-race")
98101
}
102+
if r.Timeout > 0 {
103+
args = append(args, fmt.Sprintf("-timeout=%s", r.Timeout.String()))
104+
}
105+
if len(r.Tags) > 0 {
106+
args = append(args, fmt.Sprintf("-tags=%s", strings.Join(r.Tags, ",")))
107+
}
99108
if r.UseShuffle {
100109
if r.ShuffleSeed != "" {
101110
args = append(args, fmt.Sprintf("-shuffle=%s", r.ShuffleSeed))
@@ -126,7 +135,8 @@ func (r *Runner) runTests(packageName string) (string, bool, error) {
126135

127136
r.prettyProjectPath, err = prettyProjectPath(r.ProjectPath)
128137
if err != nil {
129-
return "", false, fmt.Errorf("failed to get pretty project path: %w", err)
138+
r.prettyProjectPath = r.ProjectPath
139+
log.Printf("WARN: failed to get pretty project path: %v", err)
130140
}
131141
// Run the command with output directed to the file
132142
cmd := exec.Command("go", args...)
@@ -268,12 +278,13 @@ func parseTestResults(expectedRuns int, filePaths []string) ([]reports.TestResul
268278

269279
if (panicDetectionMode || raceDetectionMode) && entryLine.Action == "fail" { // End of panic or race output
270280
if panicDetectionMode {
271-
panicTest, err := attributePanicToTest(entryLine.Package, detectedEntries)
281+
panicTest, timeout, err := attributePanicToTest(entryLine.Package, detectedEntries)
272282
if err != nil {
273283
return nil, err
274284
}
275285
panicTestKey := fmt.Sprintf("%s/%s", entryLine.Package, panicTest)
276286
testDetails[panicTestKey].Panic = true
287+
testDetails[panicTestKey].Timeout = timeout
277288
testDetails[panicTestKey].Failures++
278289
testDetails[panicTestKey].Runs++
279290
// TODO: durations and panics are weird in the same way as Runs: lots of double-counting
@@ -413,17 +424,21 @@ func parseTestResults(expectedRuns int, filePaths []string) ([]reports.TestResul
413424

414425
// properly attributes panics to the test that caused them
415426
// Go JSON output gets confused, especially when tests are run in parallel
416-
func attributePanicToTest(panicPackage string, panicEntries []entry) (string, error) {
427+
func attributePanicToTest(panicPackage string, panicEntries []entry) (test string, timeout bool, err error) {
417428
regexSanitizePanicPackage := filepath.Base(panicPackage)
418429
panicAttributionRe := regexp.MustCompile(fmt.Sprintf(`%s\.(Test[^\.\(]+)`, regexSanitizePanicPackage))
430+
timeoutAttributionRe := regexp.MustCompile(`(Test.*?)\W+\(.*\)`)
419431
entriesOutputs := []string{}
420432
for _, entry := range panicEntries {
421433
entriesOutputs = append(entriesOutputs, entry.Output)
422434
if matches := panicAttributionRe.FindStringSubmatch(entry.Output); len(matches) > 1 {
423-
return matches[1], nil
435+
return matches[1], false, nil
436+
}
437+
if matches := timeoutAttributionRe.FindStringSubmatch(entry.Output); len(matches) > 1 {
438+
return matches[1], true, nil
424439
}
425440
}
426-
return "", fmt.Errorf("failed to attribute panic to test, using regex %s on these strings:\n%s", panicAttributionRe.String(), strings.Join(entriesOutputs, "\n"))
441+
return "", false, fmt.Errorf("failed to attribute panic to test, using regex %s on these strings:\n%s", panicAttributionRe.String(), strings.Join(entriesOutputs, ""))
427442
}
428443

429444
// properly attributes races to the test that caused them
@@ -438,7 +453,7 @@ func attributeRaceToTest(racePackage string, raceEntries []entry) (string, error
438453
return matches[1], nil
439454
}
440455
}
441-
return "", fmt.Errorf("failed to attribute race to test, using regex: %s on these strings:\n%s", raceAttributionRe.String(), strings.Join(entriesOutputs, "\n"))
456+
return "", fmt.Errorf("failed to attribute race to test, using regex: %s on these strings:\n%s", raceAttributionRe.String(), strings.Join(entriesOutputs, ""))
442457
}
443458

444459
// parseSubTest checks if a test name is a subtest and returns the parent and sub names

0 commit comments

Comments
 (0)