Skip to content

Commit eb1cdf6

Browse files
lukaszcllukaszcl
committed
Fix pass ratio calculations
1 parent 2185a8f commit eb1cdf6

File tree

4 files changed

+134
-101
lines changed

4 files changed

+134
-101
lines changed

‎tools/flakeguard/reports/data.go‎

Lines changed: 33 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -40,69 +40,69 @@ type TestResult struct {
4040
}
4141

4242
type SummaryData struct {
43-
TotalTests int `json:"total_tests"`
44-
PanickedTests int `json:"panicked_tests"`
45-
RacedTests int `json:"raced_tests"`
46-
FlakyTests int `json:"flaky_tests"`
47-
FlakyTestRatio string `json:"flaky_test_ratio"`
48-
TotalRuns int `json:"total_runs"`
49-
PassedRuns int `json:"passed_runs"`
50-
FailedRuns int `json:"failed_runs"`
51-
SkippedRuns int `json:"skipped_runs"`
52-
PassRatio string `json:"pass_ratio"`
53-
MaxPassRatio float64 `json:"max_pass_ratio"`
54-
AveragePassRatio float64 `json:"average_pass_ratio"`
43+
TotalTests int `json:"total_tests"`
44+
PanickedTests int `json:"panicked_tests"`
45+
RacedTests int `json:"raced_tests"`
46+
FlakyTests int `json:"flaky_tests"`
47+
FlakyTestRatio string `json:"flaky_test_ratio"`
48+
TotalRuns int `json:"total_runs"`
49+
PassedRuns int `json:"passed_runs"`
50+
FailedRuns int `json:"failed_runs"`
51+
SkippedRuns int `json:"skipped_runs"`
52+
PassRatio string `json:"pass_ratio"`
53+
MaxPassRatio float64 `json:"max_pass_ratio"`
5554
}
5655

5756
// Data Processing Functions
5857

5958
func GenerateSummaryData(tests []TestResult, maxPassRatio float64) SummaryData {
60-
var runs, passes, fails, skips, panickedTests, racedTests, flakyTests int
59+
var runs, passes, fails, skips, panickedTests, racedTests, flakyTests, skippedTests int
6160
for _, result := range tests {
6261
runs += result.Runs
6362
passes += result.Successes
6463
fails += result.Failures
6564
skips += result.Skips
6665

66+
// Count tests that were entirely skipped
67+
if result.Runs == 0 && result.Skipped {
68+
skippedTests++
69+
}
70+
6771
if result.Panic {
6872
panickedTests++
6973
flakyTests++
7074
} else if result.Race {
7175
racedTests++
7276
flakyTests++
7377
} else if !result.Skipped && result.Runs > 0 && result.PassRatio < maxPassRatio {
74-
// Exclude skipped tests and tests with no runs
7578
flakyTests++
7679
}
7780
}
7881

7982
passPercentage := 100.0
8083
flakePercentage := 0.0
81-
averagePassRatio := 1.0
8284

8385
if runs > 0 {
84-
passPercentage = math.Round((float64(passes)/float64(runs)*100)*100) / 100
85-
averagePassRatio = float64(passes) / float64(runs)
86+
passPercentage = math.Floor((float64(passes)/float64(runs)*100)*100) / 100 // Truncate to 2 decimal places
8687
}
87-
totalTests := len(tests) // Include skipped tests in total tests
88-
totalExecutedTests := totalTests - skips // Tests that were actually executed
89-
if totalExecutedTests > 0 {
90-
flakePercentage = math.Round((float64(flakyTests)/float64(totalExecutedTests)*100)*100) / 100
88+
89+
totalTests := len(tests)
90+
if totalTests > 0 {
91+
flakePercentage = math.Floor((float64(flakyTests)/float64(totalTests)*100)*100) / 100 // Truncate to 2 decimal places
9192
}
9293

9394
return SummaryData{
94-
TotalTests: totalTests,
95-
PanickedTests: panickedTests,
96-
RacedTests: racedTests,
97-
FlakyTests: flakyTests,
98-
FlakyTestRatio: fmt.Sprintf("%.2f%%", flakePercentage),
99-
TotalRuns: runs,
100-
PassedRuns: passes,
101-
FailedRuns: fails,
102-
SkippedRuns: skips,
103-
PassRatio: fmt.Sprintf("%.2f%%", passPercentage),
104-
MaxPassRatio: maxPassRatio,
105-
AveragePassRatio: averagePassRatio,
95+
TotalTests: totalTests,
96+
PanickedTests: panickedTests,
97+
RacedTests: racedTests,
98+
FlakyTests: flakyTests,
99+
FlakyTestRatio: fmt.Sprintf("%.2f%%", flakePercentage),
100+
TotalRuns: runs,
101+
PassedRuns: passes,
102+
FailedRuns: fails,
103+
SkippedRuns: skips,
104+
PassRatio: fmt.Sprintf("%.2f%%", passPercentage),
105+
MaxPassRatio: maxPassRatio,
106106
}
107107
}
108108

‎tools/flakeguard/reports/data_test.go‎

Lines changed: 88 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -24,18 +24,17 @@ func TestGenerateSummaryData(t *testing.T) {
2424
},
2525
maxPassRatio: 1.0,
2626
expected: SummaryData{
27-
TotalTests: 2,
28-
PanickedTests: 0,
29-
RacedTests: 0,
30-
FlakyTests: 0,
31-
FlakyTestRatio: "0.00%",
32-
TotalRuns: 15,
33-
PassedRuns: 15,
34-
FailedRuns: 0,
35-
SkippedRuns: 0,
36-
PassRatio: "100.00%",
37-
MaxPassRatio: 1.0,
38-
AveragePassRatio: 1.0,
27+
TotalTests: 2,
28+
PanickedTests: 0,
29+
RacedTests: 0,
30+
FlakyTests: 0,
31+
FlakyTestRatio: "0.00%",
32+
TotalRuns: 15,
33+
PassedRuns: 15,
34+
FailedRuns: 0,
35+
SkippedRuns: 0,
36+
PassRatio: "100.00%",
37+
MaxPassRatio: 1.0,
3938
},
4039
},
4140
{
@@ -47,18 +46,17 @@ func TestGenerateSummaryData(t *testing.T) {
4746
},
4847
maxPassRatio: 0.9,
4948
expected: SummaryData{
50-
TotalTests: 3,
51-
PanickedTests: 0,
52-
RacedTests: 0,
53-
FlakyTests: 2,
54-
FlakyTestRatio: "66.67%",
55-
TotalRuns: 19,
56-
PassedRuns: 15,
57-
FailedRuns: 4,
58-
SkippedRuns: 0,
59-
PassRatio: "78.95%",
60-
MaxPassRatio: 0.9,
61-
AveragePassRatio: 0.7894736842105263,
49+
TotalTests: 3,
50+
PanickedTests: 0,
51+
RacedTests: 0,
52+
FlakyTests: 2,
53+
FlakyTestRatio: "66.67%",
54+
TotalRuns: 19,
55+
PassedRuns: 15,
56+
FailedRuns: 4,
57+
SkippedRuns: 0,
58+
PassRatio: "78.95%",
59+
MaxPassRatio: 0.9,
6260
},
6361
},
6462
{
@@ -70,37 +68,78 @@ func TestGenerateSummaryData(t *testing.T) {
7068
},
7169
maxPassRatio: 1.0,
7270
expected: SummaryData{
73-
TotalTests: 3,
74-
PanickedTests: 1,
75-
RacedTests: 1,
76-
FlakyTests: 2,
77-
FlakyTestRatio: "66.67%",
78-
TotalRuns: 18,
79-
PassedRuns: 17,
80-
FailedRuns: 1,
81-
SkippedRuns: 0,
82-
PassRatio: "94.44%",
83-
MaxPassRatio: 1.0,
84-
AveragePassRatio: 0.9444444444444444,
71+
TotalTests: 3,
72+
PanickedTests: 1,
73+
RacedTests: 1,
74+
FlakyTests: 2,
75+
FlakyTestRatio: "66.67%",
76+
TotalRuns: 18,
77+
PassedRuns: 17,
78+
FailedRuns: 1,
79+
SkippedRuns: 0,
80+
PassRatio: "94.44%",
81+
MaxPassRatio: 1.0,
8582
},
8683
},
8784
{
8885
name: "No tests ran",
8986
testResults: []TestResult{},
9087
maxPassRatio: 1.0,
9188
expected: SummaryData{
92-
TotalTests: 0,
93-
PanickedTests: 0,
94-
RacedTests: 0,
95-
FlakyTests: 0,
96-
FlakyTestRatio: "0.00%",
97-
TotalRuns: 0,
98-
PassedRuns: 0,
99-
FailedRuns: 0,
100-
SkippedRuns: 0,
101-
PassRatio: "100.00%",
102-
MaxPassRatio: 1.0,
103-
AveragePassRatio: 1.0,
89+
TotalTests: 0,
90+
PanickedTests: 0,
91+
RacedTests: 0,
92+
FlakyTests: 0,
93+
FlakyTestRatio: "0.00%",
94+
TotalRuns: 0,
95+
PassedRuns: 0,
96+
FailedRuns: 0,
97+
SkippedRuns: 0,
98+
PassRatio: "100.00%",
99+
MaxPassRatio: 1.0,
100+
},
101+
},
102+
{
103+
name: "Skipped tests included in total but not executed",
104+
testResults: []TestResult{
105+
{PassRatio: -1.0, Runs: 0, Successes: 0, Skips: 1, Skipped: true},
106+
{PassRatio: 0.7, Runs: 10, Successes: 7, Failures: 3},
107+
},
108+
maxPassRatio: 0.8,
109+
expected: SummaryData{
110+
TotalTests: 2,
111+
PanickedTests: 0,
112+
RacedTests: 0,
113+
FlakyTests: 1,
114+
FlakyTestRatio: "50.00%",
115+
TotalRuns: 10,
116+
PassedRuns: 7,
117+
FailedRuns: 3,
118+
SkippedRuns: 1,
119+
PassRatio: "70.00%",
120+
MaxPassRatio: 0.8,
121+
},
122+
},
123+
{
124+
name: "Mixed skipped and executed tests",
125+
testResults: []TestResult{
126+
{PassRatio: -1.0, Runs: 0, Successes: 0, Skips: 1, Skipped: true},
127+
{PassRatio: 0.9, Runs: 10, Successes: 9, Failures: 1},
128+
{PassRatio: 0.5, Runs: 4, Successes: 2, Failures: 2},
129+
},
130+
maxPassRatio: 0.85,
131+
expected: SummaryData{
132+
TotalTests: 3,
133+
PanickedTests: 0,
134+
RacedTests: 0,
135+
FlakyTests: 1,
136+
FlakyTestRatio: "33.33%",
137+
TotalRuns: 14,
138+
PassedRuns: 11,
139+
FailedRuns: 3,
140+
SkippedRuns: 1,
141+
PassRatio: "78.57%",
142+
MaxPassRatio: 0.85,
104143
},
105144
},
106145
}
@@ -109,7 +148,7 @@ func TestGenerateSummaryData(t *testing.T) {
109148
t.Run(tc.name, func(t *testing.T) {
110149
summary := GenerateSummaryData(tc.testResults, tc.maxPassRatio)
111150
if !reflect.DeepEqual(summary, tc.expected) {
112-
t.Errorf("Expected %+v, got %+v", tc.expected, summary)
151+
t.Errorf("Test %s failed. Expected %+v, got %+v", tc.name, tc.expected, summary)
113152
}
114153
})
115154
}

‎tools/flakeguard/reports/presentation.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ func GenerateGitHubSummaryMarkdown(w io.Writer, testReport *TestReport, maxPassR
9595
}
9696

9797
summary := GenerateSummaryData(testReport.Results, maxPassRatio)
98-
if summary.AveragePassRatio < maxPassRatio {
98+
if summary.FlakyTests > 0 {
9999
fmt.Fprintln(w, "## Found Flaky Tests :x:")
100100
} else {
101101
fmt.Fprintln(w, "## No Flakes Found :white_check_mark:")
@@ -134,7 +134,7 @@ func GeneratePRCommentMarkdown(w io.Writer, testReport *TestReport, maxPassRatio
134134
}
135135

136136
// Add the flaky tests section
137-
if GenerateSummaryData(testReport.Results, maxPassRatio).AveragePassRatio < maxPassRatio {
137+
if GenerateSummaryData(testReport.Results, maxPassRatio).FlakyTests > 0 {
138138
fmt.Fprintln(w, "## Found Flaky Tests :x:")
139139
} else {
140140
fmt.Fprintln(w, "## No Flakes Found :white_check_mark:")

‎tools/flakeguard/reports/presentation_test.go‎

Lines changed: 11 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -169,8 +169,6 @@ func TestGeneratePRCommentMarkdown(t *testing.T) {
169169

170170
output := buffer.String()
171171

172-
fmt.Println(output)
173-
174172
// Check that the output includes the expected headings and links
175173
if !strings.Contains(output, "# Flakeguard Summary") {
176174
t.Error("Expected markdown summary to contain '# Flakeguard Summary'")
@@ -251,18 +249,17 @@ func TestRenderResults(t *testing.T) {
251249
},
252250
maxPassRatio: 0.9,
253251
expectedSummary: SummaryData{
254-
TotalTests: 1,
255-
PanickedTests: 0,
256-
RacedTests: 0,
257-
FlakyTests: 1,
258-
FlakyTestRatio: "100.00%",
259-
TotalRuns: 4,
260-
PassedRuns: 3,
261-
FailedRuns: 1,
262-
SkippedRuns: 0,
263-
PassRatio: "75.00%",
264-
MaxPassRatio: 0.9,
265-
AveragePassRatio: 0.75,
252+
TotalTests: 1,
253+
PanickedTests: 0,
254+
RacedTests: 0,
255+
FlakyTests: 1,
256+
FlakyTestRatio: "100.00%",
257+
TotalRuns: 4,
258+
PassedRuns: 3,
259+
FailedRuns: 1,
260+
SkippedRuns: 0,
261+
PassRatio: "75.00%",
262+
MaxPassRatio: 0.9,
266263
},
267264
expectedStringsContain: []string{"Test1", "package1", "75.00%", "false", "1.05s", "4", "0"},
268265
},
@@ -299,9 +296,6 @@ func TestRenderResults(t *testing.T) {
299296
if summary.PassRatio != tc.expectedSummary.PassRatio {
300297
t.Errorf("Expected PassRatio %v, got %v", tc.expectedSummary.PassRatio, summary.PassRatio)
301298
}
302-
if summary.AveragePassRatio != tc.expectedSummary.AveragePassRatio {
303-
t.Errorf("Expected AveragePassRatio %v, got %v", tc.expectedSummary.AveragePassRatio, summary.AveragePassRatio)
304-
}
305299

306300
// Verify output content
307301
for _, expected := range tc.expectedStringsContain {

0 commit comments

Comments
 (0)