Skip to content

Commit d72264a

Browse files
lukaszcllukaszcl
committed
fixed mergeTestResults
1 parent 537b06f commit d72264a

File tree

4 files changed

+460
-64
lines changed

4 files changed

+460
-64
lines changed

‎tools/flakeguard/cmd/run.go‎

Lines changed: 54 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,32 @@ var RunTestsCmd = &cobra.Command{
3535
tags, _ := cmd.Flags().GetStringArray("tags")
3636
useRace, _ := cmd.Flags().GetBool("race")
3737
outputPath, _ := cmd.Flags().GetString("output-json")
38+
minPassRatio, _ := cmd.Flags().GetFloat64("min-pass-ratio")
39+
// For backward compatibility, check if max-pass-ratio was used
3840
maxPassRatio, _ := cmd.Flags().GetFloat64("max-pass-ratio")
41+
maxPassRatioSpecified := cmd.Flags().Changed("max-pass-ratio")
3942
skipTests, _ := cmd.Flags().GetStringSlice("skip-tests")
4043
selectTests, _ := cmd.Flags().GetStringSlice("select-tests")
4144
useShuffle, _ := cmd.Flags().GetBool("shuffle")
4245
shuffleSeed, _ := cmd.Flags().GetString("shuffle-seed")
4346
omitOutputsOnSuccess, _ := cmd.Flags().GetBool("omit-test-outputs-on-success")
4447
ignoreParentFailuresOnSubtests, _ := cmd.Flags().GetBool("ignore-parent-failures-on-subtests")
48+
rerunFailed, _ := cmd.Flags().GetInt("rerun-failed")
49+
failFast, _ := cmd.Flags().GetBool("fail-fast")
50+
51+
// Handle the compatibility between min/max pass ratio
52+
passRatioThreshold := minPassRatio
53+
if maxPassRatioSpecified && maxPassRatio != 1.0 {
54+
// If max-pass-ratio was explicitly set, use it (convert to min-pass-ratio)
55+
log.Warn().Msg("--max-pass-ratio is deprecated, please use --min-pass-ratio instead")
56+
passRatioThreshold = maxPassRatio
57+
}
58+
59+
// Validate pass ratio
60+
if passRatioThreshold < 0 || passRatioThreshold > 1 {
61+
log.Error().Float64("pass ratio", passRatioThreshold).Msg("Error: pass ratio must be between 0 and 1")
62+
os.Exit(ErrorExitCode)
63+
}
4564

4665
outputDir := filepath.Dir(outputPath)
4766
initialDirSize, err := getDirSize(outputDir)
@@ -50,11 +69,6 @@ var RunTestsCmd = &cobra.Command{
5069
// intentionally don't exit here, as we can still proceed with the run
5170
}
5271

53-
if maxPassRatio < 0 || maxPassRatio > 1 {
54-
log.Error().Float64("max pass ratio", maxPassRatio).Msg("Error: max pass ratio must be between 0 and 1")
55-
os.Exit(ErrorExitCode)
56-
}
57-
5872
// Check if project dependencies are correctly set up
5973
if err := checkDependencies(projectPath); err != nil {
6074
log.Error().Err(err).Msg("Error checking project dependencies")
@@ -90,8 +104,10 @@ var RunTestsCmd = &cobra.Command{
90104
UseShuffle: useShuffle,
91105
ShuffleSeed: shuffleSeed,
92106
OmitOutputsOnSuccess: omitOutputsOnSuccess,
93-
MaxPassRatio: maxPassRatio,
107+
MaxPassRatio: passRatioThreshold, // Use the calculated threshold
94108
IgnoreParentFailuresOnSubtests: ignoreParentFailuresOnSubtests,
109+
RerunFailed: rerunFailed,
110+
FailFast: failFast,
95111
}
96112

97113
// Run the tests
@@ -134,20 +150,37 @@ var RunTestsCmd = &cobra.Command{
134150

135151
// Filter flaky tests using FilterTests
136152
flakyTests := reports.FilterTests(testReport.Results, func(tr reports.TestResult) bool {
137-
return !tr.Skipped && tr.PassRatio < maxPassRatio
153+
return !tr.Skipped && tr.PassRatio < passRatioThreshold
138154
})
139155

140156
finalDirSize, err := getDirSize(outputDir)
141157
if err != nil {
142-
log.Error().Err(err).Str("path", outputDir).Msg("Error getting initial directory size")
158+
log.Error().Err(err).Str("path", outputDir).Msg("Error getting final directory size")
143159
// intentionally don't exit here, as we can still proceed with the run
144160
}
145161
diskSpaceUsed := byteCountSI(finalDirSize - initialDirSize)
146162

163+
// Report with more detailed information about reruns
164+
if rerunFailed > 0 {
165+
log.Info().
166+
Int("initial runs", runCount).
167+
Int("reruns for failed tests", rerunFailed).
168+
Str("disk space used", diskSpaceUsed).
169+
Msg("Test execution completed")
170+
} else {
171+
log.Info().
172+
Int("runs", runCount).
173+
Str("disk space used", diskSpaceUsed).
174+
Msg("Test execution completed")
175+
}
176+
147177
if len(flakyTests) > 0 {
148-
log.Info().Str("disk space used", diskSpaceUsed).Int("count", len(flakyTests)).Str("pass ratio threshold", fmt.Sprintf("%.2f%%", maxPassRatio*100)).Msg("Found flaky tests")
178+
log.Info().
179+
Int("count", len(flakyTests)).
180+
Str("stability threshold", fmt.Sprintf("%.0f%%", passRatioThreshold*100)).
181+
Msg("Found flaky tests")
149182
} else {
150-
log.Info().Str("disk space used", diskSpaceUsed).Msg("No flaky tests found")
183+
log.Info().Msg("All tests passed stability requirements")
151184
}
152185

153186
fmt.Printf("\nFlakeguard Summary\n")
@@ -178,9 +211,19 @@ func init() {
178211
RunTestsCmd.Flags().String("output-json", "", "Path to output the test results in JSON format")
179212
RunTestsCmd.Flags().StringSlice("skip-tests", nil, "Comma-separated list of test names to skip from running")
180213
RunTestsCmd.Flags().StringSlice("select-tests", nil, "Comma-separated list of test names to specifically run")
181-
RunTestsCmd.Flags().Float64("max-pass-ratio", 1.0, "The maximum pass ratio threshold for a test to be considered flaky. Any tests below this pass rate will be considered flaky.")
214+
215+
// Add the min-pass-ratio flag (new recommended approach)
216+
RunTestsCmd.Flags().Float64("min-pass-ratio", 1.0, "The minimum pass ratio required for a test to be considered stable (0.0-1.0)")
217+
218+
// Keep max-pass-ratio for backward compatibility but mark as deprecated
219+
RunTestsCmd.Flags().Float64("max-pass-ratio", 1.0, "DEPRECATED: Use min-pass-ratio instead")
220+
RunTestsCmd.Flags().MarkDeprecated("max-pass-ratio", "use min-pass-ratio instead")
221+
182222
RunTestsCmd.Flags().Bool("omit-test-outputs-on-success", true, "Omit test outputs and package outputs for tests that pass")
183223
RunTestsCmd.Flags().Bool("ignore-parent-failures-on-subtests", false, "Ignore failures in parent tests when only subtests fail")
224+
225+
// Add rerun failed tests flag
226+
RunTestsCmd.Flags().Int("rerun-failed", 0, "Number of times to rerun failed tests (0 disables reruns)")
184227
}
185228

186229
func checkDependencies(projectPath string) error {

‎tools/flakeguard/runner/example_test_package/example_tests_test.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package exampletestpackage
22

33
import (
44
"log"
5+
"math/rand"
56
"os"
67
"sync"
78
"testing"
@@ -212,3 +213,23 @@ func TestTimeout(t *testing.T) {
212213
time.Sleep(time.Until(deadline))
213214
t.Logf("This test should have timed out")
214215
}
216+
217+
// TestRandomFlaky is a truly random flaky test that will fail approximately 50% of the time
218+
func TestRandomFlaky(t *testing.T) {
219+
t.Parallel()
220+
221+
// Seed random number generator with current time
222+
r := rand.New(rand.NewSource(time.Now().UnixNano()))
223+
224+
// Generate a random number between 0 and 1
225+
randomValue := r.Float64()
226+
227+
t.Logf("Random value generated: %f", randomValue)
228+
229+
// Fail the test approximately 90% of the time
230+
if randomValue < 0.9 {
231+
t.Fatal("This test randomly failed (90% probability)")
232+
}
233+
234+
t.Log("This test randomly passed (90% probability)")
235+
}

‎tools/flakeguard/runner/runner.go‎

Lines changed: 75 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -736,43 +736,67 @@ func prettyProjectPath(projectPath string) (string, error) {
736736
}
737737

738738
func (r *Runner) rerunFailedTests(results []reports.TestResult) ([]reports.TestResult, error) {
739-
var failingTests []reports.TestResult
739+
// Group failing tests by package for more efficient reruns
740+
failingTestsByPackage := make(map[string][]string)
740741
for _, tr := range results {
741742
if !tr.Skipped && tr.PassRatio < 1 {
742-
failingTests = append(failingTests, tr)
743+
if _, exists := failingTestsByPackage[tr.TestPackage]; !exists {
744+
failingTestsByPackage[tr.TestPackage] = []string{}
745+
}
746+
failingTestsByPackage[tr.TestPackage] = append(failingTestsByPackage[tr.TestPackage], tr.TestName)
743747
}
744748
}
745749

746750
if r.Verbose {
747-
log.Info().Msgf("Rerunning failing tests: %v", failingTests)
751+
log.Info().Msgf("Rerunning failing tests grouped by package: %v", failingTestsByPackage)
748752
}
749753

750-
var rerunResults []reports.TestResult
754+
var rerunJsonFilePaths []string
751755

752-
// Rerun each failing test up to RerunFailed times
756+
// Rerun each failing test package up to RerunFailed times
753757
for i := 0; i < r.RerunFailed; i++ {
754-
for _, fTest := range failingTests {
755-
testCmd := r.buildGoTestCommandForTest(fTest)
756-
757-
if r.Verbose {
758-
log.Info().Msgf("Rerun iteration %d for %s: %v", i+1, fTest.TestName, testCmd)
758+
for pkg, tests := range failingTestsByPackage {
759+
// Build regex pattern to match all failing tests in this package
760+
testPattern := fmt.Sprintf("^(%s)$", strings.Join(tests, "|"))
761+
762+
cmd := []string{
763+
"go", "test",
764+
pkg,
765+
"-run", testPattern,
766+
"-json",
759767
}
760768

761-
jsonFilePath, _, err := r.runCmd(testCmd, i)
762-
if err != nil {
763-
return nil, fmt.Errorf("error on rerunCmd for test %s: %w", fTest.TestName, err)
769+
// Add other test flags
770+
if r.UseRace {
771+
cmd = append(cmd, "-race")
772+
}
773+
if r.Timeout > 0 {
774+
cmd = append(cmd, fmt.Sprintf("-timeout=%s", r.Timeout.String()))
775+
}
776+
if len(r.Tags) > 0 {
777+
cmd = append(cmd, fmt.Sprintf("-tags=%s", strings.Join(r.Tags, ",")))
778+
}
779+
if r.Verbose {
780+
cmd = append(cmd, "-v")
781+
log.Info().Msgf("Rerun iteration %d for package %s: %v", i+1, pkg, cmd)
764782
}
765783

766-
additionalResults, err := r.parseTestResults([]string{jsonFilePath}, "rerun")
784+
// Run the package tests
785+
jsonFilePath, _, err := r.runCmd(cmd, i)
767786
if err != nil {
768-
return nil, fmt.Errorf("failed to parse rerun results: %w", err)
787+
return nil, fmt.Errorf("error on rerunCmd for package %s: %w", pkg, err)
769788
}
770789

771-
// Collect these rerun results in a slice; we'll merge them later.
772-
rerunResults = append(rerunResults, additionalResults...)
790+
rerunJsonFilePaths = append(rerunJsonFilePaths, jsonFilePath)
773791
}
774792
}
775793

794+
// Parse all rerun results at once with a consistent prefix
795+
rerunResults, err := r.parseTestResults(rerunJsonFilePaths, "rerun")
796+
if err != nil {
797+
return nil, fmt.Errorf("failed to parse rerun results: %w", err)
798+
}
799+
776800
return rerunResults, nil
777801
}
778802

@@ -796,7 +820,6 @@ func (r *Runner) buildGoTestCommandForTest(t reports.TestResult) []string {
796820
return cmd
797821
}
798822

799-
// mergeTestResults merges additional test results into the existing results slice.
800823
// mergeTestResults merges additional test results into the existing results slice.
801824
func mergeTestResults(mainResults *[]reports.TestResult, additional []reports.TestResult) {
802825
for _, add := range additional {
@@ -809,30 +832,60 @@ func mergeTestResults(mainResults *[]reports.TestResult, additional []reports.Te
809832
(*mainResults)[i].Failures += add.Failures
810833
(*mainResults)[i].Skips += add.Skips
811834

835+
// Merge boolean flags (using OR operation)
836+
(*mainResults)[i].Panic = (*mainResults)[i].Panic || add.Panic
837+
(*mainResults)[i].Race = (*mainResults)[i].Race || add.Race
838+
(*mainResults)[i].Timeout = (*mainResults)[i].Timeout || add.Timeout
839+
(*mainResults)[i].Skipped = (*mainResults)[i].Skipped || add.Skipped
840+
(*mainResults)[i].PackagePanic = (*mainResults)[i].PackagePanic || add.PackagePanic
841+
812842
// Merge durations
813843
(*mainResults)[i].Durations = append((*mainResults)[i].Durations, add.Durations...)
814844

845+
// Merge maps for Outputs
846+
if (*mainResults)[i].Outputs == nil {
847+
(*mainResults)[i].Outputs = make(map[string][]string)
848+
}
849+
for runID, outputs := range add.Outputs {
850+
if existing, ok := (*mainResults)[i].Outputs[runID]; ok {
851+
(*mainResults)[i].Outputs[runID] = append(existing, outputs...)
852+
} else {
853+
(*mainResults)[i].Outputs[runID] = outputs
854+
}
855+
}
856+
815857
// Merge maps for PassedOutputs
816858
if (*mainResults)[i].PassedOutputs == nil {
817859
(*mainResults)[i].PassedOutputs = make(map[string][]string)
818860
}
819861
for runID, outputs := range add.PassedOutputs {
820-
(*mainResults)[i].PassedOutputs[runID] = append((*mainResults)[i].PassedOutputs[runID], outputs...)
862+
if existing, ok := (*mainResults)[i].PassedOutputs[runID]; ok {
863+
(*mainResults)[i].PassedOutputs[runID] = append(existing, outputs...)
864+
} else {
865+
(*mainResults)[i].PassedOutputs[runID] = outputs
866+
}
821867
}
822868

823869
// Merge maps for FailedOutputs
824870
if (*mainResults)[i].FailedOutputs == nil {
825871
(*mainResults)[i].FailedOutputs = make(map[string][]string)
826872
}
827873
for runID, outputs := range add.FailedOutputs {
828-
(*mainResults)[i].FailedOutputs[runID] = append((*mainResults)[i].FailedOutputs[runID], outputs...)
874+
if existing, ok := (*mainResults)[i].FailedOutputs[runID]; ok {
875+
(*mainResults)[i].FailedOutputs[runID] = append(existing, outputs...)
876+
} else {
877+
(*mainResults)[i].FailedOutputs[runID] = outputs
878+
}
829879
}
830880

831-
// Update pass ratio
881+
// Merge PackageOutputs
882+
(*mainResults)[i].PackageOutputs = append((*mainResults)[i].PackageOutputs, add.PackageOutputs...)
883+
884+
// Update pass ratio (consistent with parseTestResults default)
832885
if (*mainResults)[i].Runs > 0 {
833886
(*mainResults)[i].PassRatio = float64((*mainResults)[i].Successes) / float64((*mainResults)[i].Runs)
834887
} else {
835-
(*mainResults)[i].PassRatio = -1.0
888+
(*mainResults)[i].PassRatio = 1.0 // Default to 1.0 if no runs
836889
}
837890

838891
found = true

0 commit comments

Comments
 (0)