Skip to content

Commit 97a6b31

Browse files
lukaszcllukaszcl
committed
Fix
1 parent 44121f5 commit 97a6b31

File tree

2 files changed

+135
-78
lines changed

2 files changed

+135
-78
lines changed

‎tools/flakeguard/reports/data.go‎

Lines changed: 36 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -143,30 +143,55 @@ func GenerateSummaryData(tests []TestResult, maxPassRatio float64) SummaryData {
143143
}
144144
}
145145

146-
passPercentage := 100.0
147-
flakePercentage := 0.0
146+
// Calculate the PASS ratio (as a float) and then build a string
147+
passPercentageStr := "100.00%" // default if runs == 0
148148

149149
if runs > 0 {
150-
passPercentage = math.Round((float64(passes)/float64(runs)*100)*100) / 100 // Round to 2 decimal places
150+
rawPassRatio := (float64(passes) / float64(runs)) * 100
151+
// Round to 8 decimal places to avoid floating-point quirks
152+
passPercentage := math.Floor(rawPassRatio*1e8) / 1e8
153+
154+
// If there's any failure, never show 100.00%
155+
if fails > 0 {
156+
passPercentage = math.Min(passPercentage, 99.99)
157+
}
158+
159+
passPercentageStr = fmt.Sprintf("%.2f%%", passPercentage)
151160
}
152161

162+
// Calculate the FLAKE ratio (as a float) and then build a string
163+
flakePercentage := 0.0
164+
flakeTestRatioStr := "0.00%" // default if totalTests == 0
165+
153166
totalTests := len(tests)
154167
if totalTests > 0 {
155-
flakePercentage = math.Round((float64(flakyTests)/float64(totalTests)*100)*100) / 100 // Round to 2 decimal places
168+
rawFlakeRatio := (float64(flakyTests) / float64(totalTests)) * 100
169+
170+
// If non-zero but < 0.01, show "< 0.01%"
171+
if rawFlakeRatio > 0 && rawFlakeRatio < 0.01 {
172+
flakeTestRatioStr = "< 0.01%"
173+
} else {
174+
flakePercentage = math.Floor(rawFlakeRatio*100) / 100
175+
flakeTestRatioStr = fmt.Sprintf("%.2f%%", flakePercentage)
176+
}
156177
}
157178

179+
// 3) Build the SummaryData with the final strings
158180
return SummaryData{
159181
TotalTests: totalTests,
160182
PanickedTests: panickedTests,
161183
RacedTests: racedTests,
162184
FlakyTests: flakyTests,
163-
FlakyTestRatio: fmt.Sprintf("%.2f%%", flakePercentage),
164-
TotalRuns: runs,
165-
PassedRuns: passes,
166-
FailedRuns: fails,
167-
SkippedRuns: skips,
168-
PassRatio: fmt.Sprintf("%.2f%%", passPercentage),
169-
MaxPassRatio: maxPassRatio,
185+
FlakyTestRatio: flakeTestRatioStr,
186+
187+
TotalRuns: runs,
188+
PassedRuns: passes,
189+
FailedRuns: fails,
190+
SkippedRuns: skips,
191+
192+
// Use the final passPercentageStr
193+
PassRatio: passPercentageStr,
194+
MaxPassRatio: maxPassRatio,
170195
}
171196
}
172197

‎tools/flakeguard/reports/data_test.go‎

Lines changed: 99 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,14 @@ func TestGenerateSummaryData(t *testing.T) {
2828
PanickedTests: 0,
2929
RacedTests: 0,
3030
FlakyTests: 0,
31-
FlakyTestRatio: "0.00%",
31+
FlakyTestRatio: "0.00%", // no flaky tests
3232
TotalRuns: 15,
3333
PassedRuns: 15,
3434
FailedRuns: 0,
3535
SkippedRuns: 0,
36-
PassRatio: "100.00%",
37-
MaxPassRatio: 1.0,
36+
// No failures => truly 100.00%
37+
PassRatio: "100.00%",
38+
MaxPassRatio: 1.0,
3839
},
3940
},
4041
{
@@ -46,17 +47,19 @@ func TestGenerateSummaryData(t *testing.T) {
4647
},
4748
maxPassRatio: 0.9,
4849
expected: SummaryData{
49-
TotalTests: 3,
50-
PanickedTests: 0,
51-
RacedTests: 0,
52-
FlakyTests: 2,
53-
FlakyTestRatio: "66.67%",
50+
TotalTests: 3,
51+
PanickedTests: 0,
52+
RacedTests: 0,
53+
FlakyTests: 2,
54+
// 2/3 => 66.666...% => rounds to 66.66%
55+
FlakyTestRatio: "66.66%",
5456
TotalRuns: 19,
5557
PassedRuns: 15,
56-
FailedRuns: 4,
58+
FailedRuns: 4, // total failures
5759
SkippedRuns: 0,
58-
PassRatio: "78.95%",
59-
MaxPassRatio: 0.9,
60+
// 15/19 => ~78.947... => rounds to 78.95%
61+
PassRatio: "78.95%",
62+
MaxPassRatio: 0.9,
6063
},
6164
},
6265
{
@@ -68,17 +71,19 @@ func TestGenerateSummaryData(t *testing.T) {
6871
},
6972
maxPassRatio: 1.0,
7073
expected: SummaryData{
71-
TotalTests: 3,
72-
PanickedTests: 1,
73-
RacedTests: 1,
74-
FlakyTests: 2,
75-
FlakyTestRatio: "66.67%",
74+
TotalTests: 3,
75+
PanickedTests: 1,
76+
RacedTests: 1,
77+
FlakyTests: 2,
78+
// 2/3 => ~66.666... => "66.66%"
79+
FlakyTestRatio: "66.66%",
7680
TotalRuns: 18,
7781
PassedRuns: 17,
7882
FailedRuns: 1,
7983
SkippedRuns: 0,
80-
PassRatio: "94.44%",
81-
MaxPassRatio: 1.0,
84+
// 17/18 => ~94.444... => "94.44%"
85+
PassRatio: "94.44%",
86+
MaxPassRatio: 1.0,
8287
},
8388
},
8489
{
@@ -90,13 +95,14 @@ func TestGenerateSummaryData(t *testing.T) {
9095
PanickedTests: 0,
9196
RacedTests: 0,
9297
FlakyTests: 0,
93-
FlakyTestRatio: "0.00%",
98+
FlakyTestRatio: "0.00%", // no tests => 0.00%
9499
TotalRuns: 0,
95100
PassedRuns: 0,
96101
FailedRuns: 0,
97102
SkippedRuns: 0,
98-
PassRatio: "100.00%",
99-
MaxPassRatio: 1.0,
103+
// With zero runs, we default passRatio to "100.00%"
104+
PassRatio: "100.00%",
105+
MaxPassRatio: 1.0,
100106
},
101107
},
102108
{
@@ -110,12 +116,12 @@ func TestGenerateSummaryData(t *testing.T) {
110116
TotalTests: 2,
111117
PanickedTests: 0,
112118
RacedTests: 0,
113-
FlakyTests: 1,
114-
FlakyTestRatio: "50.00%",
119+
FlakyTests: 1, // second test has ratio=0.7 => "flaky"
120+
FlakyTestRatio: "50.00%", // 1 out of 2 => 50.00%
115121
TotalRuns: 10,
116122
PassedRuns: 7,
117123
FailedRuns: 3,
118-
SkippedRuns: 1,
124+
SkippedRuns: 1, // from first test
119125
PassRatio: "70.00%",
120126
MaxPassRatio: 0.8,
121127
},
@@ -132,39 +138,34 @@ func TestGenerateSummaryData(t *testing.T) {
132138
TotalTests: 3,
133139
PanickedTests: 0,
134140
RacedTests: 0,
135-
FlakyTests: 1,
136-
FlakyTestRatio: "33.33%",
137-
TotalRuns: 14,
138-
PassedRuns: 11,
139-
FailedRuns: 3,
140-
SkippedRuns: 1,
141-
PassRatio: "78.57%",
142-
MaxPassRatio: 0.85,
141+
FlakyTests: 1, // last test has ratio=0.5 => "flaky"
142+
FlakyTestRatio: "33.33%", // 1 out of 3 => 33.333... => 33.33%
143+
TotalRuns: 14, // 10 + 4
144+
PassedRuns: 11, // 9 + 2
145+
FailedRuns: 3, // 1 + 2
146+
SkippedRuns: 1, // from first test
147+
// 11/14 => 78.5714... => "78.57%"
148+
PassRatio: "78.57%",
149+
MaxPassRatio: 0.85,
143150
},
144151
},
145152
{
146-
name: "Tiny flake ratio that rounds up to 0.01%",
153+
name: "Tiny flake ratio that is exactly 0.01%",
147154
testResults: func() []TestResult {
148-
// Create 9,999 test results in total:
149-
// - 9,998 stable (PassRatio=1.0) => not flaky
150-
// - 1 flaky (PassRatio=0.5) => definitely flaky
155+
// 9,999 total:
156+
// - 9,998 stable => pass=1.0
157+
// - 1 flaky => pass=0.5
151158
const total = 9999
152159
tests := make([]TestResult, total)
153160
for i := 0; i < total-1; i++ {
154161
tests[i] = TestResult{
155-
PassRatio: 1.0, // 100% success
162+
PassRatio: 1.0,
156163
Runs: 10,
157164
Successes: 10,
158-
Failures: 0,
159-
Skips: 0,
160-
Skipped: false,
161-
Panic: false,
162-
Race: false,
163165
}
164166
}
165-
// This final test is partially failing => PassRatio=0.5 => "flaky"
166167
tests[total-1] = TestResult{
167-
PassRatio: 0.5,
168+
PassRatio: 0.5, // 1 success, 1 fail
168169
Runs: 2,
169170
Successes: 1,
170171
Failures: 1,
@@ -173,32 +174,62 @@ func TestGenerateSummaryData(t *testing.T) {
173174
}(),
174175
maxPassRatio: 1.0,
175176
expected: SummaryData{
176-
// 9,999 total test results:
177-
TotalTests: 9999,
178-
// None of them panic or race:
177+
TotalTests: 9999,
179178
PanickedTests: 0,
180179
RacedTests: 0,
181-
// Exactly one is flaky:
182-
FlakyTests: 1,
183-
// Flaky ratio = 1 / 9999 ≈ 0.00010001 => 0.01% after rounding
180+
FlakyTests: 1,
181+
// ratio = 1/9999 => ~0.00010001 => rounds to 0.01%
184182
FlakyTestRatio: "0.01%",
185-
186-
// Total runs = 9,998 stable tests * 10 runs each + 1 flaky test with 2 runs
187-
TotalRuns: (9998 * 10) + 2, // = 99,980 + 2 = 99,982
188-
189-
// Passed runs = all 9,998 stable tests (each 10 successes) + 1 success in the flaky test
190-
PassedRuns: (9998 * 10) + 1, // = 99,980 + 1 = 99,981
191-
192-
// Failed runs = the 1 failure in the flaky test
193-
FailedRuns: 1,
194-
195-
// No skipped
183+
// total runs => (9998 stable * 10 each) + 2 = 99,982
184+
TotalRuns: (9998 * 10) + 2,
185+
// total passes => (9998 stable * 10) + 1 success in flaky = 99,981
186+
PassedRuns: (9998 * 10) + 1,
187+
FailedRuns: 1,
196188
SkippedRuns: 0,
197-
198-
// Pass ratio = 99,981 / 99,982 = 0.9999899 => 99.99899% => rounds to 100.00%
199-
PassRatio: "100.00%",
200-
201-
// Provided maxPassRatio
189+
// 1 failure => we do NOT show 100% => we cap at 99.99%
190+
PassRatio: "99.99%",
191+
MaxPassRatio: 1.0,
192+
},
193+
},
194+
{
195+
name: "Tiny flake ratio below threshold",
196+
testResults: func() []TestResult {
197+
// 100,001 total:
198+
// - 100,000 stable => pass=1.0
199+
// - 1 flaky => pass=0.5
200+
const total = 100001
201+
tests := make([]TestResult, total)
202+
for i := 0; i < total-1; i++ {
203+
tests[i] = TestResult{
204+
PassRatio: 1.0,
205+
Runs: 10,
206+
Successes: 10,
207+
}
208+
}
209+
tests[total-1] = TestResult{
210+
PassRatio: 0.5, // 1 success, 1 fail
211+
Runs: 2,
212+
Successes: 1,
213+
Failures: 1,
214+
}
215+
return tests
216+
}(),
217+
maxPassRatio: 1.0,
218+
expected: SummaryData{
219+
TotalTests: 100001,
220+
PanickedTests: 0,
221+
RacedTests: 0,
222+
FlakyTests: 1,
223+
// 1 / 100001 => ~0.000009999 => definitely < 0.01%
224+
FlakyTestRatio: "< 0.01%",
225+
// total runs => (100,000 stable * 10) + 2 = 1,000,002
226+
TotalRuns: (100000 * 10) + 2,
227+
// passes => stable=1,000,000 + 1 success from flaky=1 => 1,000,001
228+
PassedRuns: 100000*10 + 1,
229+
FailedRuns: 1,
230+
SkippedRuns: 0,
231+
// again 1 failure => cap pass ratio => "99.99%"
232+
PassRatio: "99.99%",
202233
MaxPassRatio: 1.0,
203234
},
204235
},
@@ -208,7 +239,8 @@ func TestGenerateSummaryData(t *testing.T) {
208239
t.Run(tc.name, func(t *testing.T) {
209240
summary := GenerateSummaryData(tc.testResults, tc.maxPassRatio)
210241
if !reflect.DeepEqual(summary, tc.expected) {
211-
t.Errorf("Test %s failed. Expected %+v, got %+v", tc.name, tc.expected, summary)
242+
t.Errorf("Test %q failed.\nExpected: %+v\nGot: %+v",
243+
tc.name, tc.expected, summary)
212244
}
213245
})
214246
}

0 commit comments

Comments
 (0)