Skip to content

Commit 203c96c

Browse files
kalverrakalverra
committed
More extensive runner tests
1 parent ea85f0c commit 203c96c

File tree

3 files changed

+205
-225
lines changed

3 files changed

+205
-225
lines changed

‎tools/flakeguard/reports/reports.go‎

Lines changed: 13 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,9 @@ type TestResult struct {
2727
Panics int // Count of how many times the test panicked
2828
Races int // Count of how many times the test encountered a data race
2929
Skips int // Count of how many times the test was skipped
30-
Outputs []string // Stores outputs for a test
30+
Outputs []string `json:"outputs,omitempty"` // Stores outputs for a test
3131
Durations []time.Duration // Stores elapsed time for each run of the test
32-
PackageOutputs []string // Stores package-level outputs
32+
PackageOutputs []string `json:"package_outputs,omitempty"` // Stores package-level outputs
3333
}
3434

3535
// FilterFailedTests returns a slice of TestResult where the pass ratio is below the specified threshold.
@@ -142,6 +142,7 @@ func AggregateTestResults(folderPath string) ([]TestResult, error) {
142142
}
143143

144144
// PrintTests prints tests in a pretty format
145+
// TODO: Update this with new fields
145146
func PrintTests(tests []TestResult, w io.Writer) {
146147
for i, test := range tests {
147148
fmt.Fprintf(w, "\n--- Test %d ---\n", i+1)
@@ -150,6 +151,11 @@ func PrintTests(tests []TestResult, w io.Writer) {
150151
fmt.Fprintf(w, "PassRatio: %.2f\n", test.PassRatio)
151152
fmt.Fprintf(w, "Skipped: %v\n", test.Skipped)
152153
fmt.Fprintf(w, "Runs: %d\n", test.Runs)
154+
fmt.Fprintf(w, "Successes: %d\n", test.Successes)
155+
fmt.Fprintf(w, "Failures: %d\n", test.Failures)
156+
fmt.Fprintf(w, "Panics: %d\n", test.Panics)
157+
fmt.Fprintf(w, "Races: %d\n", test.Races)
158+
fmt.Fprintf(w, "Skips: %d\n", test.Skips)
153159
durationsStr := make([]string, len(test.Durations))
154160
for i, duration := range test.Durations {
155161
durationsStr[i] = duration.String()
@@ -180,32 +186,12 @@ func SaveFilteredResultsAndLogs(outputResultsPath, outputLogsPath string, failed
180186

181187
// Helper function to save results to JSON file
182188
func saveResults(filePath string, results []TestResult) error {
183-
// Define a struct type without Outputs and PackageOutputs
184-
type filteredTestResult struct {
185-
TestName string
186-
TestPackage string
187-
Panicked bool
188-
PackagePanicked bool
189-
PassRatio float64
190-
PassRatioPercentage string
191-
Skipped bool
192-
Runs int
193-
Durations []time.Duration
194-
}
195-
196-
var filteredResults []filteredTestResult
189+
var filteredResults []TestResult
197190
for _, r := range results {
198-
filteredResults = append(filteredResults, filteredTestResult{
199-
TestName: r.TestName,
200-
TestPackage: r.TestPackage,
201-
Panicked: r.Panicked,
202-
PackagePanicked: r.PackagePanicked,
203-
PassRatio: r.PassRatio,
204-
PassRatioPercentage: r.PassRatioPercentage,
205-
Skipped: r.Skipped,
206-
Runs: r.Runs,
207-
Durations: r.Durations,
208-
})
191+
filteredResult := r
192+
filteredResult.Outputs = nil
193+
filteredResult.PackageOutputs = nil
194+
filteredResults = append(filteredResults, filteredResult)
209195
}
210196

211197
data, err := json.MarshalIndent(filteredResults, "", " ")

‎tools/flakeguard/runner/runner.go‎

Lines changed: 57 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ func (r *Runner) RunTests() ([]reports.TestResult, error) {
5050
r.rawOutputs[p] = &bytes.Buffer{}
5151
}
5252
separator := strings.Repeat("-", 80)
53-
r.rawOutputs[p].WriteString(fmt.Sprintf("%d%s\n", i, separator))
53+
r.rawOutputs[p].WriteString(fmt.Sprintf("Run %d%s\n", i+1, separator))
5454
}
5555
jsonFilePath, passed, err := r.runTests(p)
5656
if err != nil {
@@ -193,37 +193,11 @@ func parseTestResults(filePaths []string) ([]reports.TestResult, error) {
193193
result = testDetails[key]
194194
}
195195

196-
// TODO: This is a bit of a logical mess, refactor
196+
// TODO: This is a bit of a logical mess, probably worth a refactor
197197
if entryLine.Output != "" {
198198
if panicDetectionMode || raceDetectionMode { // currently collecting panic or race output
199199
detectedEntries = append(detectedEntries, entryLine)
200-
if entryLine.Action == "fail" { // End of panic output
201-
if panicDetectionMode {
202-
panicTest, err := attributePanicToTest(entryLine.Package, detectedEntries)
203-
if err != nil {
204-
return nil, err
205-
}
206-
panicTestKey := fmt.Sprintf("%s/%s", entryLine.Package, panicTest)
207-
testDetails[panicTestKey].Panicked = true
208-
testDetails[panicTestKey].Panics++
209-
testDetails[panicTestKey].Runs++
210-
testDetails[panicTestKey].Outputs = append(testDetails[panicTestKey].Outputs, entryLine.Output)
211-
} else if raceDetectionMode {
212-
raceTest, err := attributeRaceToTest(entryLine.Package, detectedEntries)
213-
if err != nil {
214-
return nil, err
215-
}
216-
raceTestKey := fmt.Sprintf("%s/%s", entryLine.Package, raceTest)
217-
testDetails[raceTestKey].Races++
218-
testDetails[raceTestKey].Runs++
219-
testDetails[raceTestKey].Outputs = append(testDetails[raceTestKey].Outputs, entryLine.Output)
220-
}
221-
222-
detectedEntries = []entry{}
223-
panicDetectionMode = false
224-
raceDetectionMode = false
225-
}
226-
continue // Don't process this entry further
200+
continue
227201
} else if startPanicRe.MatchString(entryLine.Output) { // found a panic, start collecting output
228202
panickedPackages[entryLine.Package] = struct{}{}
229203
detectedEntries = append(detectedEntries, entryLine)
@@ -244,6 +218,58 @@ func parseTestResults(filePaths []string) ([]reports.TestResult, error) {
244218
}
245219
}
246220

221+
if (panicDetectionMode || raceDetectionMode) && entryLine.Action == "fail" { // End of panic or race output
222+
if panicDetectionMode {
223+
panicTest, err := attributePanicToTest(entryLine.Package, detectedEntries)
224+
if err != nil {
225+
return nil, err
226+
}
227+
panicTestKey := fmt.Sprintf("%s/%s", entryLine.Package, panicTest)
228+
testDetails[panicTestKey].Panicked = true
229+
testDetails[panicTestKey].Panics++
230+
testDetails[panicTestKey].Runs++
231+
duration, err := time.ParseDuration(strconv.FormatFloat(entryLine.Elapsed, 'f', -1, 64) + "s")
232+
if err != nil {
233+
return nil, fmt.Errorf("failed to parse duration: %w", err)
234+
}
235+
testDetails[panicTestKey].Durations = append(testDetails[panicTestKey].Durations, duration)
236+
testDetails[panicTestKey].Outputs = append(testDetails[panicTestKey].Outputs, entryLine.Output)
237+
for _, entry := range detectedEntries {
238+
if entry.Test == "" {
239+
testDetails[panicTestKey].PackageOutputs = append(testDetails[panicTestKey].PackageOutputs, entry.Output)
240+
} else {
241+
testDetails[panicTestKey].Outputs = append(testDetails[panicTestKey].Outputs, entry.Output)
242+
}
243+
}
244+
} else if raceDetectionMode {
245+
raceTest, err := attributeRaceToTest(entryLine.Package, detectedEntries)
246+
if err != nil {
247+
return nil, err
248+
}
249+
raceTestKey := fmt.Sprintf("%s/%s", entryLine.Package, raceTest)
250+
testDetails[raceTestKey].Races++
251+
testDetails[raceTestKey].Runs++
252+
duration, err := time.ParseDuration(strconv.FormatFloat(entryLine.Elapsed, 'f', -1, 64) + "s")
253+
if err != nil {
254+
return nil, fmt.Errorf("failed to parse duration: %w", err)
255+
}
256+
testDetails[raceTestKey].Durations = append(testDetails[raceTestKey].Durations, duration)
257+
testDetails[raceTestKey].Outputs = append(testDetails[raceTestKey].Outputs, entryLine.Output)
258+
for _, entry := range detectedEntries {
259+
if entry.Test == "" {
260+
testDetails[raceTestKey].PackageOutputs = append(testDetails[raceTestKey].PackageOutputs, entry.Output)
261+
} else {
262+
testDetails[raceTestKey].Outputs = append(testDetails[raceTestKey].Outputs, entry.Output)
263+
}
264+
}
265+
}
266+
267+
detectedEntries = []entry{}
268+
panicDetectionMode = false
269+
raceDetectionMode = false
270+
continue
271+
}
272+
247273
switch entryLine.Action {
248274
case "pass":
249275
if entryLine.Test != "" {
@@ -277,7 +303,7 @@ func parseTestResults(filePaths []string) ([]reports.TestResult, error) {
277303
result.Runs++
278304
}
279305
}
280-
if entryLine.Test != "" {
306+
if entryLine.Test != "" && result.Runs > 0 {
281307
result.PassRatio = float64(result.Successes) / float64(result.Runs)
282308
result.PassRatioPercentage = fmt.Sprintf("%.0f%%", result.PassRatio*100)
283309
}
@@ -297,13 +323,13 @@ func parseTestResults(filePaths []string) ([]reports.TestResult, error) {
297323

298324
var results []reports.TestResult
299325
for _, result := range testDetails {
300-
results = append(results, *result)
301326
if _, panicked := panickedPackages[result.TestPackage]; panicked {
302327
result.PackagePanicked = true
303328
}
304329
if outputs, exists := packageLevelOutputs[result.TestPackage]; exists {
305330
result.PackageOutputs = outputs
306331
}
332+
results = append(results, *result)
307333
}
308334

309335
return results, nil

0 commit comments

Comments
 (0)