Skip to content

Commit dfba0c5

Browse files
lukaszcllukaszcl
committed
Refactor
1 parent 3e19109 commit dfba0c5

File tree

3 files changed

+93
-105
lines changed

3 files changed

+93
-105
lines changed

‎tools/flakeguard/cmd/run.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,6 @@ var RunTestsCmd = &cobra.Command{
6464
UseRace: useRace,
6565
SkipTests: skipTests,
6666
SelectTests: selectTests,
67-
SelectedTestPackages: testPackages,
6867
UseShuffle: useShuffle,
6968
ShuffleSeed: shuffleSeed,
7069
OmitOutputsOnSuccess: omitOutputsOnSuccess,
@@ -77,13 +76,13 @@ var RunTestsCmd = &cobra.Command{
7776
)
7877

7978
if len(testCmdStrings) > 0 {
80-
testReport, err = testRunner.RunTestsByCmd(testCmdStrings)
79+
testReport, err = testRunner.RunTestCmd(testCmdStrings)
8180
if err != nil {
8281
log.Fatal().Err(err).Msg("Error running custom test command")
8382
}
8483
} else {
8584
// Otherwise, use the normal go test approach
86-
testReport, err = testRunner.RunTests()
85+
testReport, err = testRunner.RunTestPackages(testPackages)
8786
if err != nil {
8887
log.Fatal().Err(err).Msg("Error running tests")
8988
}

‎tools/flakeguard/runner/runner.go‎

Lines changed: 38 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -37,17 +37,16 @@ type Runner struct {
3737
FailFast bool // Stop on first test failure.
3838
SkipTests []string // Test names to exclude.
3939
SelectTests []string // Test names to include.
40-
SelectedTestPackages []string // Explicitly selected packages to run.
4140
CollectRawOutput bool // Set to true to collect test output for later inspection.
4241
OmitOutputsOnSuccess bool // Set to true to omit test outputs on success.
4342
rawOutputs map[string]*bytes.Buffer
4443
}
4544

46-
// RunTests executes the tests for each provided package and aggregates all results.
45+
// RunTestPackages executes the tests for each provided package and aggregates all results.
4746
// It returns all test results and any error encountered during testing.
48-
func (r *Runner) RunTests() (*reports.TestReport, error) {
47+
func (r *Runner) RunTestPackages(packages []string) (*reports.TestReport, error) {
4948
var jsonFilePaths []string
50-
for _, p := range r.SelectedTestPackages {
49+
for _, p := range packages {
5150
for i := 0; i < r.RunCount; i++ {
5251
if r.CollectRawOutput { // Collect raw output for debugging
5352
if r.rawOutputs == nil {
@@ -59,7 +58,7 @@ func (r *Runner) RunTests() (*reports.TestReport, error) {
5958
separator := strings.Repeat("-", 80)
6059
r.rawOutputs[p].WriteString(fmt.Sprintf("Run %d\n%s\n", i+1, separator))
6160
}
62-
jsonFilePath, passed, err := r.runTests(p)
61+
jsonFilePath, passed, err := r.runTestPackage(p)
6362
if err != nil {
6463
return nil, fmt.Errorf("failed to run tests in package %s: %w", p, err)
6564
}
@@ -84,6 +83,38 @@ func (r *Runner) RunTests() (*reports.TestReport, error) {
8483
}, nil
8584
}
8685

86+
// RunTestCmd runs an arbitrary command testCmd (like ["go", "run", "my_test.go", ...])
87+
// that produces the same JSON lines that 'go test -json' would produce on stdout.
88+
// It captures those lines in a temp file, then parses them for pass/fail/panic/race data.
89+
func (r *Runner) RunTestCmd(testCmd []string) (*reports.TestReport, error) {
90+
var jsonFilePaths []string
91+
92+
// Run the command r.RunCount times
93+
for i := 0; i < r.RunCount; i++ {
94+
jsonFilePath, passed, err := r.runCmd(testCmd, i)
95+
if err != nil {
96+
return nil, fmt.Errorf("failed to run test command: %w", err)
97+
}
98+
jsonFilePaths = append(jsonFilePaths, jsonFilePath)
99+
if !passed && r.FailFast {
100+
break
101+
}
102+
}
103+
104+
results, err := r.parseTestResults(jsonFilePaths)
105+
if err != nil {
106+
return nil, fmt.Errorf("failed to parse test results: %w", err)
107+
}
108+
109+
// Build a TestReport, same shape as RunTests()
110+
return &reports.TestReport{
111+
GoProject: r.prettyProjectPath,
112+
TestRunCount: r.RunCount,
113+
RaceDetection: r.UseRace,
114+
Results: results,
115+
}, nil
116+
}
117+
87118
// RawOutputs retrieves the raw output from the test runs, if CollectRawOutput enabled.
88119
// packageName : raw output
89120
func (r *Runner) RawOutputs() map[string]*bytes.Buffer {
@@ -94,8 +125,8 @@ type exitCoder interface {
94125
ExitCode() int
95126
}
96127

97-
// runTests runs the tests for a given package and returns the path to the output file.
98-
func (r *Runner) runTests(packageName string) (string, bool, error) {
128+
// runTestPackage runs the tests for a given package and returns the path to the output file.
129+
func (r *Runner) runTestPackage(packageName string) (string, bool, error) {
99130
args := []string{"test", packageName, "-json", "-count=1"}
100131
if r.UseRace {
101132
args = append(args, "-race")
@@ -160,38 +191,6 @@ func (r *Runner) runTests(packageName string) (string, bool, error) {
160191
return tmpFile.Name(), true, nil // Test succeeded
161192
}
162193

163-
// RunTestsByCmd runs an arbitrary command testCmd (like ["go", "run", "my_test.go", ...])
164-
// that produces the same JSON lines that 'go test -json' would produce on stdout.
165-
// It captures those lines in a temp file, then parses them for pass/fail/panic/race data.
166-
func (r *Runner) RunTestsByCmd(testCmd []string) (*reports.TestReport, error) {
167-
var jsonFilePaths []string
168-
169-
// Run the command r.RunCount times
170-
for i := 0; i < r.RunCount; i++ {
171-
jsonFilePath, passed, err := r.runCmd(testCmd, i)
172-
if err != nil {
173-
return nil, fmt.Errorf("failed to run test command: %w", err)
174-
}
175-
jsonFilePaths = append(jsonFilePaths, jsonFilePath)
176-
if !passed && r.FailFast {
177-
break
178-
}
179-
}
180-
181-
results, err := r.parseTestResults(jsonFilePaths)
182-
if err != nil {
183-
return nil, fmt.Errorf("failed to parse test results: %w", err)
184-
}
185-
186-
// Build a TestReport, same shape as RunTests()
187-
return &reports.TestReport{
188-
GoProject: r.prettyProjectPath,
189-
TestRunCount: r.RunCount,
190-
RaceDetection: r.UseRace,
191-
Results: results,
192-
}, nil
193-
}
194-
195194
// runCmd is a helper that runs the user-supplied command once, captures its JSON output,
196195
// and returns (tempFilePath, passed, error).
197196
func (r *Runner) runCmd(testCmd []string, runIndex int) (string, bool, error) {
@@ -222,10 +221,8 @@ func (r *Runner) runCmd(testCmd []string, runIndex int) (string, bool, error) {
222221
} else {
223222
cmd.Stdout = tmpFile
224223
}
225-
// Handle stderr however you like; here we just dump it to console
226224
cmd.Stderr = os.Stderr
227225

228-
// Run it
229226
err = cmd.Run()
230227

231228
// Determine pass/fail from exit code

‎tools/flakeguard/runner/runner_test.go‎

Lines changed: 53 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -62,14 +62,13 @@ func TestRun(t *testing.T) {
6262
{
6363
name: "default",
6464
runner: Runner{
65-
ProjectPath: "./",
66-
Verbose: true,
67-
RunCount: defaultRuns,
68-
UseRace: false,
69-
SkipTests: []string{"TestPanic", "TestFlakyPanic", "TestSubTestsSomePanic", "TestTimeout"},
70-
FailFast: false,
71-
SelectedTestPackages: []string{flakyTestPackagePath},
72-
CollectRawOutput: true,
65+
ProjectPath: "./",
66+
Verbose: true,
67+
RunCount: defaultRuns,
68+
UseRace: false,
69+
SkipTests: []string{"TestPanic", "TestFlakyPanic", "TestSubTestsSomePanic", "TestTimeout"},
70+
FailFast: false,
71+
CollectRawOutput: true,
7372
},
7473
expectedTests: map[string]*expectedTestResult{
7574
"TestFlaky": {
@@ -156,15 +155,14 @@ func TestRun(t *testing.T) {
156155
{
157156
name: "always panic",
158157
runner: Runner{
159-
ProjectPath: "./",
160-
Verbose: true,
161-
RunCount: defaultRuns,
162-
UseRace: false,
163-
SkipTests: []string{},
164-
SelectTests: []string{"TestPanic"},
165-
FailFast: false,
166-
SelectedTestPackages: []string{flakyTestPackagePath},
167-
CollectRawOutput: true,
158+
ProjectPath: "./",
159+
Verbose: true,
160+
RunCount: defaultRuns,
161+
UseRace: false,
162+
SkipTests: []string{},
163+
SelectTests: []string{"TestPanic"},
164+
FailFast: false,
165+
CollectRawOutput: true,
168166
},
169167
expectedTests: map[string]*expectedTestResult{
170168
"TestPanic": {
@@ -177,15 +175,14 @@ func TestRun(t *testing.T) {
177175
{
178176
name: "flaky panic",
179177
runner: Runner{
180-
ProjectPath: "./",
181-
Verbose: true,
182-
RunCount: defaultRuns,
183-
UseRace: false,
184-
SkipTests: []string{},
185-
SelectTests: []string{"TestFlakyPanic"},
186-
FailFast: false,
187-
SelectedTestPackages: []string{flakyTestPackagePath},
188-
CollectRawOutput: true,
178+
ProjectPath: "./",
179+
Verbose: true,
180+
RunCount: defaultRuns,
181+
UseRace: false,
182+
SkipTests: []string{},
183+
SelectTests: []string{"TestFlakyPanic"},
184+
FailFast: false,
185+
CollectRawOutput: true,
189186
},
190187
expectedTests: map[string]*expectedTestResult{
191188
"TestFlakyPanic": {
@@ -198,15 +195,14 @@ func TestRun(t *testing.T) {
198195
{
199196
name: "subtest panic",
200197
runner: Runner{
201-
ProjectPath: "./",
202-
Verbose: true,
203-
RunCount: defaultRuns,
204-
UseRace: false,
205-
SkipTests: []string{},
206-
SelectTests: []string{"TestSubTestsSomePanic"},
207-
FailFast: false,
208-
SelectedTestPackages: []string{flakyTestPackagePath},
209-
CollectRawOutput: true,
198+
ProjectPath: "./",
199+
Verbose: true,
200+
RunCount: defaultRuns,
201+
UseRace: false,
202+
SkipTests: []string{},
203+
SelectTests: []string{"TestSubTestsSomePanic"},
204+
FailFast: false,
205+
CollectRawOutput: true,
210206
},
211207
expectedTests: map[string]*expectedTestResult{
212208
"TestSubTestsSomePanic": {
@@ -229,15 +225,14 @@ func TestRun(t *testing.T) {
229225
{
230226
name: "failfast",
231227
runner: Runner{
232-
ProjectPath: "./",
233-
Verbose: true,
234-
RunCount: defaultRuns,
235-
UseRace: false,
236-
SkipTests: []string{},
237-
SelectTests: []string{"TestFail", "TestPass"},
238-
FailFast: true,
239-
SelectedTestPackages: []string{flakyTestPackagePath},
240-
CollectRawOutput: true,
228+
ProjectPath: "./",
229+
Verbose: true,
230+
RunCount: defaultRuns,
231+
UseRace: false,
232+
SkipTests: []string{},
233+
SelectTests: []string{"TestFail", "TestPass"},
234+
FailFast: true,
235+
CollectRawOutput: true,
241236
},
242237
expectedTests: map[string]*expectedTestResult{
243238
"TestFail": {
@@ -254,7 +249,7 @@ func TestRun(t *testing.T) {
254249

255250
for _, tc := range testCases {
256251
t.Run(tc.name, func(t *testing.T) {
257-
testReport, err := tc.runner.RunTests()
252+
testReport, err := tc.runner.RunTestPackages([]string{flakyTestPackagePath})
258253
require.NoError(t, err)
259254

260255
t.Cleanup(func() {
@@ -502,15 +497,14 @@ func TestFailedOutputs(t *testing.T) {
502497
t.Parallel()
503498

504499
runner := Runner{
505-
ProjectPath: "./",
506-
Verbose: true,
507-
RunCount: 1,
508-
SelectedTestPackages: []string{flakyTestPackagePath},
509-
SelectTests: []string{"TestFail"}, // This test is known to fail consistently
510-
CollectRawOutput: true,
500+
ProjectPath: "./",
501+
Verbose: true,
502+
RunCount: 1,
503+
SelectTests: []string{"TestFail"}, // This test is known to fail consistently
504+
CollectRawOutput: true,
511505
}
512506

513-
testReport, err := runner.RunTests()
507+
testReport, err := runner.RunTestPackages([]string{flakyTestPackagePath})
514508
require.NoError(t, err, "running tests should not produce an unexpected error")
515509

516510
require.Equal(t, 1, testReport.TestRunCount, "unexpected number of test runs")
@@ -537,15 +531,14 @@ func TestSkippedTests(t *testing.T) {
537531
t.Parallel()
538532

539533
runner := Runner{
540-
ProjectPath: "./",
541-
Verbose: true,
542-
RunCount: 1,
543-
SelectedTestPackages: []string{flakyTestPackagePath},
544-
SelectTests: []string{"TestSkipped"}, // Known skipping test
545-
CollectRawOutput: true,
534+
ProjectPath: "./",
535+
Verbose: true,
536+
RunCount: 1,
537+
SelectTests: []string{"TestSkipped"}, // Known skipping test
538+
CollectRawOutput: true,
546539
}
547540

548-
testReport, err := runner.RunTests()
541+
testReport, err := runner.RunTestPackages([]string{flakyTestPackagePath})
549542
require.NoError(t, err, "running tests should not produce an unexpected error")
550543

551544
require.Equal(t, 1, testReport.TestRunCount, "unexpected number of test runs")
@@ -573,13 +566,12 @@ func TestOmitOutputsOnSuccess(t *testing.T) {
573566
ProjectPath: "./",
574567
Verbose: true,
575568
RunCount: 1,
576-
SelectedTestPackages: []string{flakyTestPackagePath},
577569
SelectTests: []string{"TestPass"}, // Known passing test
578570
CollectRawOutput: true,
579571
OmitOutputsOnSuccess: true,
580572
}
581573

582-
testReport, err := runner.RunTests()
574+
testReport, err := runner.RunTestPackages([]string{flakyTestPackagePath})
583575
require.NoError(t, err, "running tests should not produce an unexpected error")
584576

585577
require.Equal(t, 1, testReport.TestRunCount, "unexpected number of test runs")

0 commit comments

Comments
 (0)