Skip to content

Commit 897bca3

Browse files
lukaszcllukaszcl
authored
flakeguard: Handle panic, save test outputs into separate file, add min-pass-ratio (#1341)
1 parent 984c3e8 commit 897bca3

File tree

5 files changed

+197
-63
lines changed

5 files changed

+197
-63
lines changed

‎tools/flakeguard/cmd/aggregate.go‎

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package cmd
22

33
import (
4-
"fmt"
54
"log"
65

76
"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
@@ -13,27 +12,24 @@ var AggregateAllCmd = &cobra.Command{
1312
Short: "Aggregate all test results and output them to a file",
1413
Run: func(cmd *cobra.Command, args []string) {
1514
resultsFolderPath, _ := cmd.Flags().GetString("results-path")
16-
outputPath, _ := cmd.Flags().GetString("output-json")
15+
outputResultsPath, _ := cmd.Flags().GetString("output-results")
16+
outputLogsPath, _ := cmd.Flags().GetString("output-logs")
1717

1818
// Aggregate all test results
1919
allResults, err := reports.AggregateTestResults(resultsFolderPath)
2020
if err != nil {
2121
log.Fatalf("Error aggregating results: %v", err)
2222
}
2323

24-
// Output all results to JSON file
25-
if outputPath != "" && len(allResults) > 0 {
26-
if err := saveResults(outputPath, allResults); err != nil {
27-
log.Fatalf("Error writing aggregated results to file: %v", err)
28-
}
29-
fmt.Printf("Aggregated test results saved to %s\n", outputPath)
30-
} else {
31-
fmt.Println("No test results found.")
24+
// Output all results to JSON files
25+
if len(allResults) > 0 {
26+
reports.SaveFilteredResultsAndLogs(outputResultsPath, outputLogsPath, allResults)
3227
}
3328
},
3429
}
3530

3631
func init() {
3732
AggregateAllCmd.Flags().String("results-path", "testresult/", "Path to the folder containing JSON test result files")
38-
AggregateAllCmd.Flags().String("output-json", "all_tests.json", "Path to output the aggregated test results in JSON format")
33+
AggregateAllCmd.Flags().String("output-results", "failed_tests.json", "Path to output the filtered failed test results in JSON format")
34+
AggregateAllCmd.Flags().String("output-logs", "failed_logs.json", "Path to output the filtered failed test logs in JSON format")
3935
}

‎tools/flakeguard/cmd/aggregate_failed.go‎

Lines changed: 11 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,7 @@
11
package cmd
22

33
import (
4-
"encoding/json"
5-
"fmt"
64
"log"
7-
"os"
85

96
"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
107
"github.com/spf13/cobra"
@@ -15,46 +12,36 @@ var AggregateFailedCmd = &cobra.Command{
1512
Short: "Aggregate all test results, then filter and output only failed tests based on a threshold",
1613
Run: func(cmd *cobra.Command, args []string) {
1714
resultsFolderPath, _ := cmd.Flags().GetString("results-path")
18-
outputPath, _ := cmd.Flags().GetString("output-json")
15+
outputResultsPath, _ := cmd.Flags().GetString("output-results")
16+
outputLogsPath, _ := cmd.Flags().GetString("output-logs")
1917
threshold, _ := cmd.Flags().GetFloat64("threshold")
18+
minPassRatio, _ := cmd.Flags().GetFloat64("min-pass-ratio")
2019

2120
// Aggregate all test results
2221
allResults, err := reports.AggregateTestResults(resultsFolderPath)
2322
if err != nil {
2423
log.Fatalf("Error aggregating results: %v", err)
2524
}
2625

27-
// Filter to only include failed tests based on threshold
26+
// Filter to only include failed tests based on threshold and minPassRatio
2827
var failedResults []reports.TestResult
2928
for _, result := range allResults {
30-
if result.PassRatio < threshold && !result.Skipped {
29+
if result.PassRatio < threshold && result.PassRatio > minPassRatio && !result.Skipped {
3130
failedResults = append(failedResults, result)
3231
}
3332
}
3433

35-
// Output failed results to JSON file
36-
if outputPath != "" && len(failedResults) > 0 {
37-
if err := saveResults(outputPath, failedResults); err != nil {
38-
log.Fatalf("Error writing failed results to file: %v", err)
39-
}
40-
fmt.Printf("Filtered failed test results saved to %s\n", outputPath)
41-
} else {
42-
fmt.Println("No failed tests found based on the specified threshold.")
34+
// Output results to JSON files
35+
if len(failedResults) > 0 {
36+
reports.SaveFilteredResultsAndLogs(outputResultsPath, outputLogsPath, failedResults)
4337
}
4438
},
4539
}
4640

4741
func init() {
4842
AggregateFailedCmd.Flags().String("results-path", "testresult/", "Path to the folder containing JSON test result files")
49-
AggregateFailedCmd.Flags().String("output-json", "failed_tests.json", "Path to output the filtered failed test results in JSON format")
43+
AggregateFailedCmd.Flags().String("output-results", "failed_tests.json", "Path to output the filtered failed test results in JSON format")
44+
AggregateFailedCmd.Flags().String("output-logs", "failed_logs.json", "Path to output the filtered failed test logs in JSON format")
5045
AggregateFailedCmd.Flags().Float64("threshold", 0.8, "Threshold for considering a test as failed")
51-
}
52-
53-
// Helper function to save results to JSON file
54-
func saveResults(filePath string, results []reports.TestResult) error {
55-
data, err := json.MarshalIndent(results, "", " ")
56-
if err != nil {
57-
return fmt.Errorf("error marshaling results: %v", err)
58-
}
59-
return os.WriteFile(filePath, data, 0644)
46+
AggregateFailedCmd.Flags().Float64("min-pass-ratio", 0.001, "Minimum pass ratio for considering a test as flaky. Used to distinguish between tests that are truly flaky (with inconsistent results) and those that are consistently failing.")
6047
}

‎tools/flakeguard/cmd/run.go‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ var RunTestsCmd = &cobra.Command{
2626
threshold, _ := cmd.Flags().GetFloat64("threshold")
2727
skipTests, _ := cmd.Flags().GetStringSlice("skip-tests")
2828
printFailedTests, _ := cmd.Flags().GetBool("print-failed-tests")
29+
minPassRatio, _ := cmd.Flags().GetFloat64("min-pass-ratio")
2930

3031
// Check if project dependencies are correctly set up
3132
if err := checkDependencies(projectPath); err != nil {
@@ -48,7 +49,6 @@ var RunTestsCmd = &cobra.Command{
4849
Verbose: true,
4950
RunCount: runCount,
5051
UseRace: useRace,
51-
FailFast: threshold == 1.0, // Fail test on first test run if threshold is 1.0
5252
SkipTests: skipTests,
5353
SelectedTestPackages: testPackages,
5454
}
@@ -62,14 +62,17 @@ var RunTestsCmd = &cobra.Command{
6262
passedTests := reports.FilterPassedTests(testResults, threshold)
6363
failedTests := reports.FilterFailedTests(testResults, threshold)
6464
skippedTests := reports.FilterSkippedTests(testResults)
65+
flakyTests := reports.FilterFlakyTests(testResults, minPassRatio, threshold)
6566

67+
// Print all failed tests including flaky tests
6668
if len(failedTests) > 0 && printFailedTests {
69+
fmt.Printf("MinPassRatio threshold for flaky tests: %.2f\n", minPassRatio)
6770
fmt.Printf("PassRatio threshold for flaky tests: %.2f\n", threshold)
6871
fmt.Printf("%d failed tests:\n", len(failedTests))
6972
reports.PrintTests(failedTests, os.Stdout)
7073
}
7174

72-
fmt.Printf("Summary: %d passed, %d skipped, %d failed\n", len(passedTests), len(skippedTests), len(failedTests))
75+
fmt.Printf("Summary: %d passed, %d skipped, %d failed, %d flaky\n", len(passedTests), len(skippedTests), len(failedTests), len(flakyTests))
7376

7477
// Save the test results in JSON format
7578
if outputPath != "" && len(testResults) > 0 {
@@ -83,8 +86,8 @@ var RunTestsCmd = &cobra.Command{
8386
fmt.Printf("All test results saved to %s\n", outputPath)
8487
}
8588

86-
if len(failedTests) > 0 {
87-
// Fail if any tests failed
89+
if len(flakyTests) > 0 {
90+
// Exit with error code if there are flaky tests
8891
os.Exit(1)
8992
} else if len(testResults) == 0 {
9093
fmt.Printf("No tests were run for the specified packages.\n")
@@ -104,6 +107,7 @@ func init() {
104107
RunTestsCmd.Flags().Float64("threshold", 0.8, "Threshold for considering a test as flaky")
105108
RunTestsCmd.Flags().StringSlice("skip-tests", nil, "Comma-separated list of test names to skip from running")
106109
RunTestsCmd.Flags().Bool("print-failed-tests", true, "Print failed test results to the console")
110+
RunTestsCmd.Flags().Float64("min-pass-ratio", 0.001, "Minimum pass ratio for considering a test as flaky. Used to distinguish between tests that are truly flaky (with inconsistent results) and those that are consistently failing.")
107111
}
108112

109113
func checkDependencies(projectPath string) error {

‎tools/flakeguard/reports/reports.go‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"encoding/json"
55
"fmt"
66
"io"
7+
"log"
78
"os"
89
"path/filepath"
910
"sort"
@@ -13,12 +14,15 @@ import (
1314
type TestResult struct {
1415
TestName string
1516
TestPackage string
17+
Panicked bool // Indicates a test-level panic
18+
PackagePanicked bool // Indicates a package-level panic
1619
PassRatio float64 // Pass ratio in decimal format like 0.5
1720
PassRatioPercentage string // Pass ratio in percentage format like "50%"
1821
Skipped bool // Indicates if the test was skipped
1922
Runs int
2023
Outputs []string // Stores outputs for a test
2124
Durations []float64 // Stores elapsed time in seconds for each run of the test
25+
PackageOutputs []string // Stores package-level outputs
2226
}
2327

2428
// FilterFailedTests returns a slice of TestResult where the pass ratio is below the specified threshold.
@@ -32,6 +36,17 @@ func FilterFailedTests(results []TestResult, threshold float64) []TestResult {
3236
return failedTests
3337
}
3438

39+
// FilterFlakyTests returns a slice of TestResult where the pass ratio is between the min pass ratio and the threshold.
40+
func FilterFlakyTests(testResults []TestResult, minPassRatio, threshold float64) []TestResult {
41+
var flakyTests []TestResult
42+
for _, test := range testResults {
43+
if test.PassRatio >= minPassRatio && test.PassRatio < threshold && !test.Skipped {
44+
flakyTests = append(flakyTests, test)
45+
}
46+
}
47+
return flakyTests
48+
}
49+
3550
// FilterPassedTests returns a slice of TestResult where the tests passed and were not skipped.
3651
func FilterPassedTests(results []TestResult, threshold float64) []TestResult {
3752
var passedTests []TestResult
@@ -136,3 +151,87 @@ func PrintTests(tests []TestResult, w io.Writer) {
136151
fmt.Fprintf(w, "Outputs:\n%s\n", strings.Join(test.Outputs, ""))
137152
}
138153
}
154+
155+
// Helper function to save filtered results and logs to specified paths
156+
func SaveFilteredResultsAndLogs(outputResultsPath, outputLogsPath string, failedResults []TestResult) {
157+
if outputResultsPath != "" {
158+
if err := saveResults(outputResultsPath, failedResults); err != nil {
159+
log.Fatalf("Error writing failed results to file: %v", err)
160+
}
161+
fmt.Printf("Filtered failed test results saved to %s\n", outputResultsPath)
162+
} else {
163+
fmt.Println("No failed tests found based on the specified threshold and min pass ratio.")
164+
}
165+
166+
if outputLogsPath != "" {
167+
if err := saveTestOutputs(outputLogsPath, failedResults); err != nil {
168+
log.Fatalf("Error writing failed logs to file: %v", err)
169+
}
170+
fmt.Printf("Filtered failed test logs saved to %s\n", outputLogsPath)
171+
}
172+
}
173+
174+
// Helper function to save results to JSON file
175+
func saveResults(filePath string, results []TestResult) error {
176+
// Define a struct type without Outputs and PackageOutputs
177+
type filteredTestResult struct {
178+
TestName string
179+
TestPackage string
180+
Panicked bool
181+
PackagePanicked bool
182+
PassRatio float64
183+
PassRatioPercentage string
184+
Skipped bool
185+
Runs int
186+
Durations []float64
187+
}
188+
189+
var filteredResults []filteredTestResult
190+
for _, r := range results {
191+
filteredResults = append(filteredResults, filteredTestResult{
192+
TestName: r.TestName,
193+
TestPackage: r.TestPackage,
194+
Panicked: r.Panicked,
195+
PackagePanicked: r.PackagePanicked,
196+
PassRatio: r.PassRatio,
197+
PassRatioPercentage: r.PassRatioPercentage,
198+
Skipped: r.Skipped,
199+
Runs: r.Runs,
200+
Durations: r.Durations,
201+
})
202+
}
203+
204+
data, err := json.MarshalIndent(filteredResults, "", " ")
205+
if err != nil {
206+
return fmt.Errorf("error marshaling results: %v", err)
207+
}
208+
return os.WriteFile(filePath, data, 0644)
209+
}
210+
211+
// Helper function to save test names, packages, and outputs to JSON file
212+
func saveTestOutputs(filePath string, results []TestResult) error {
213+
// Define a struct type with only the required fields
214+
type outputOnlyResult struct {
215+
TestName string
216+
TestPackage string
217+
Outputs []string
218+
PackageOutputs []string
219+
}
220+
221+
// Convert results to the filtered struct
222+
var outputResults []outputOnlyResult
223+
for _, r := range results {
224+
outputResults = append(outputResults, outputOnlyResult{
225+
TestName: r.TestName,
226+
TestPackage: r.TestPackage,
227+
Outputs: r.Outputs,
228+
PackageOutputs: r.PackageOutputs,
229+
})
230+
}
231+
232+
data, err := json.MarshalIndent(outputResults, "", " ")
233+
if err != nil {
234+
return fmt.Errorf("error marshaling outputs: %v", err)
235+
}
236+
return os.WriteFile(filePath, data, 0644)
237+
}

0 commit comments

Comments
 (0)