Skip to content

Commit 7987e78

Browse files
lukaszcllukaszcl
authored
flakeguard: Add cmds to aggregate test results and improve memory efficiency by save test outputs to files (#1336)
1 parent d8d5cee commit 7987e78

File tree

7 files changed

+480
-54
lines changed

7 files changed

+480
-54
lines changed

‎tools/flakeguard/cmd/aggregate.go‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package cmd
2+
3+
import (
4+
"fmt"
5+
"log"
6+
7+
"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
8+
"github.com/spf13/cobra"
9+
)
10+
11+
var AggregateAllCmd = &cobra.Command{
12+
Use: "aggregate-all",
13+
Short: "Aggregate all test results and output them to a file",
14+
Run: func(cmd *cobra.Command, args []string) {
15+
resultsFolderPath, _ := cmd.Flags().GetString("results-path")
16+
outputPath, _ := cmd.Flags().GetString("output-json")
17+
18+
// Aggregate all test results
19+
allResults, err := reports.AggregateTestResults(resultsFolderPath)
20+
if err != nil {
21+
log.Fatalf("Error aggregating results: %v", err)
22+
}
23+
24+
// Output all results to JSON file
25+
if outputPath != "" && len(allResults) > 0 {
26+
if err := saveResults(outputPath, allResults); err != nil {
27+
log.Fatalf("Error writing aggregated results to file: %v", err)
28+
}
29+
fmt.Printf("Aggregated test results saved to %s\n", outputPath)
30+
} else {
31+
fmt.Println("No test results found.")
32+
}
33+
},
34+
}
35+
36+
func init() {
37+
AggregateAllCmd.Flags().String("results-path", "testresult/", "Path to the folder containing JSON test result files")
38+
AggregateAllCmd.Flags().String("output-json", "all_tests.json", "Path to output the aggregated test results in JSON format")
39+
}

‎tools/flakeguard/cmd/aggregate_failed.go‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package cmd
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"log"
7+
"os"
8+
9+
"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
10+
"github.com/spf13/cobra"
11+
)
12+
13+
var AggregateFailedCmd = &cobra.Command{
14+
Use: "aggregate-failed",
15+
Short: "Aggregate all test results, then filter and output only failed tests based on a threshold",
16+
Run: func(cmd *cobra.Command, args []string) {
17+
resultsFolderPath, _ := cmd.Flags().GetString("results-path")
18+
outputPath, _ := cmd.Flags().GetString("output-json")
19+
threshold, _ := cmd.Flags().GetFloat64("threshold")
20+
21+
// Aggregate all test results
22+
allResults, err := reports.AggregateTestResults(resultsFolderPath)
23+
if err != nil {
24+
log.Fatalf("Error aggregating results: %v", err)
25+
}
26+
27+
// Filter to only include failed tests based on threshold
28+
var failedResults []reports.TestResult
29+
for _, result := range allResults {
30+
if result.PassRatio < threshold && !result.Skipped {
31+
failedResults = append(failedResults, result)
32+
}
33+
}
34+
35+
// Output failed results to JSON file
36+
if outputPath != "" && len(failedResults) > 0 {
37+
if err := saveResults(outputPath, failedResults); err != nil {
38+
log.Fatalf("Error writing failed results to file: %v", err)
39+
}
40+
fmt.Printf("Filtered failed test results saved to %s\n", outputPath)
41+
} else {
42+
fmt.Println("No failed tests found based on the specified threshold.")
43+
}
44+
},
45+
}
46+
47+
func init() {
48+
AggregateFailedCmd.Flags().String("results-path", "testresult/", "Path to the folder containing JSON test result files")
49+
AggregateFailedCmd.Flags().String("output-json", "failed_tests.json", "Path to output the filtered failed test results in JSON format")
50+
AggregateFailedCmd.Flags().Float64("threshold", 0.8, "Threshold for considering a test as failed")
51+
}
52+
53+
// Helper function to save results to JSON file
54+
func saveResults(filePath string, results []reports.TestResult) error {
55+
data, err := json.MarshalIndent(results, "", " ")
56+
if err != nil {
57+
return fmt.Errorf("error marshaling results: %v", err)
58+
}
59+
return os.WriteFile(filePath, data, 0644)
60+
}

‎tools/flakeguard/cmd/run.go‎

Lines changed: 34 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
package cmd
22

33
import (
4+
"bytes"
45
"encoding/json"
56
"fmt"
67
"log"
78
"os"
9+
"os/exec"
810

911
"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
1012
"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/runner"
@@ -23,6 +25,12 @@ var RunTestsCmd = &cobra.Command{
2325
outputPath, _ := cmd.Flags().GetString("output-json")
2426
threshold, _ := cmd.Flags().GetFloat64("threshold")
2527
skipTests, _ := cmd.Flags().GetStringSlice("skip-tests")
28+
printFailedTests, _ := cmd.Flags().GetBool("print-failed-tests")
29+
30+
// Check if project dependencies are correctly set up
31+
if err := checkDependencies(projectPath); err != nil {
32+
log.Fatalf("Error: %v", err)
33+
}
2634

2735
var testPackages []string
2836
if testPackagesJson != "" {
@@ -36,15 +44,16 @@ var RunTestsCmd = &cobra.Command{
3644
}
3745

3846
runner := runner.Runner{
39-
ProjectPath: projectPath,
40-
Verbose: true,
41-
RunCount: runCount,
42-
UseRace: useRace,
43-
FailFast: threshold == 1.0, // Fail test on first test run if threshold is 1.0
44-
SkipTests: skipTests,
47+
ProjectPath: projectPath,
48+
Verbose: true,
49+
RunCount: runCount,
50+
UseRace: useRace,
51+
FailFast: threshold == 1.0, // Fail test on first test run if threshold is 1.0
52+
SkipTests: skipTests,
53+
SelectedTestPackages: testPackages,
4554
}
4655

47-
testResults, err := runner.RunTests(testPackages)
56+
testResults, err := runner.RunTests()
4857
if err != nil {
4958
fmt.Printf("Error running tests: %v\n", err)
5059
os.Exit(1)
@@ -54,7 +63,7 @@ var RunTestsCmd = &cobra.Command{
5463
failedTests := reports.FilterFailedTests(testResults, threshold)
5564
skippedTests := reports.FilterSkippedTests(testResults)
5665

57-
if len(failedTests) > 0 {
66+
if len(failedTests) > 0 && printFailedTests {
5867
fmt.Printf("PassRatio threshold for flaky tests: %.2f\n", threshold)
5968
fmt.Printf("%d failed tests:\n", len(failedTests))
6069
reports.PrintTests(failedTests, os.Stdout)
@@ -87,10 +96,27 @@ func init() {
8796
RunTestsCmd.Flags().StringP("project-path", "r", ".", "The path to the Go project. Default is the current directory. Useful for subprojects")
8897
RunTestsCmd.Flags().String("test-packages-json", "", "JSON-encoded string of test packages")
8998
RunTestsCmd.Flags().StringSlice("test-packages", nil, "Comma-separated list of test packages to run")
99+
RunTestsCmd.Flags().Bool("run-all-packages", false, "Run all test packages in the project. This flag overrides --test-packages and --test-packages-json")
90100
RunTestsCmd.Flags().IntP("run-count", "c", 1, "Number of times to run the tests")
91101
RunTestsCmd.Flags().Bool("race", false, "Enable the race detector")
92102
RunTestsCmd.Flags().Bool("fail-fast", false, "Stop on the first test failure")
93103
RunTestsCmd.Flags().String("output-json", "", "Path to output the test results in JSON format")
94104
RunTestsCmd.Flags().Float64("threshold", 0.8, "Threshold for considering a test as flaky")
95105
RunTestsCmd.Flags().StringSlice("skip-tests", nil, "Comma-separated list of test names to skip from running")
106+
RunTestsCmd.Flags().Bool("print-failed-tests", true, "Print failed test results to the console")
107+
}
108+
109+
func checkDependencies(projectPath string) error {
110+
cmd := exec.Command("go", "mod", "tidy")
111+
cmd.Dir = projectPath
112+
113+
var out bytes.Buffer
114+
cmd.Stdout = &out
115+
cmd.Stderr = &out
116+
117+
if err := cmd.Run(); err != nil {
118+
return fmt.Errorf("dependency check failed: %v\n%s\nPlease run 'go mod tidy' to fix missing or unused dependencies", err, out.String())
119+
}
120+
121+
return nil
96122
}

‎tools/flakeguard/main.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ func init() {
2828

2929
rootCmd.AddCommand(cmd.FindTestsCmd)
3030
rootCmd.AddCommand(cmd.RunTestsCmd)
31+
rootCmd.AddCommand(cmd.AggregateAllCmd)
32+
rootCmd.AddCommand(cmd.AggregateFailedCmd)
3133
}
3234

3335
func main() {

‎tools/flakeguard/reports/reports.go‎

Lines changed: 77 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,24 @@
11
package reports
22

33
import (
4+
"encoding/json"
45
"fmt"
56
"io"
7+
"os"
8+
"path/filepath"
9+
"sort"
610
"strings"
711
)
812

913
type TestResult struct {
10-
TestName string
11-
TestPackage string
12-
PassRatio float64
13-
Skipped bool // Indicates if the test was skipped
14-
Runs int
15-
Outputs []string // Stores outputs for a test
16-
Durations []float64 // Stores elapsed time in seconds for each run of the test
14+
TestName string
15+
TestPackage string
16+
PassRatio float64 // Pass ratio in decimal format like 0.5
17+
PassRatioPercentage string // Pass ratio in percentage format like "50%"
18+
Skipped bool // Indicates if the test was skipped
19+
Runs int
20+
Outputs []string // Stores outputs for a test
21+
Durations []float64 // Stores elapsed time in seconds for each run of the test
1722
}
1823

1924
// FilterFailedTests returns a slice of TestResult where the pass ratio is below the specified threshold.
@@ -49,6 +54,71 @@ func FilterSkippedTests(results []TestResult) []TestResult {
4954
return skippedTests
5055
}
5156

57+
// AggregateTestResults aggregates all JSON test results.
58+
func AggregateTestResults(folderPath string) ([]TestResult, error) {
59+
// Map to hold unique tests based on their TestName and TestPackage
60+
testMap := make(map[string]TestResult)
61+
62+
err := filepath.Walk(folderPath, func(path string, info os.FileInfo, err error) error {
63+
if err != nil {
64+
return err
65+
}
66+
if !info.IsDir() && filepath.Ext(path) == ".json" {
67+
// Read file content
68+
data, readErr := os.ReadFile(path)
69+
if readErr != nil {
70+
return readErr
71+
}
72+
// Parse JSON data into TestResult slice
73+
var results []TestResult
74+
if jsonErr := json.Unmarshal(data, &results); jsonErr != nil {
75+
return jsonErr
76+
}
77+
// Process each result
78+
for _, result := range results {
79+
// Unique key for each test based on TestName and TestPackage
80+
key := result.TestName + "|" + result.TestPackage
81+
if existingResult, found := testMap[key]; found {
82+
// Aggregate runs, durations, and outputs
83+
totalRuns := existingResult.Runs + result.Runs
84+
existingResult.Durations = append(existingResult.Durations, result.Durations...)
85+
existingResult.Outputs = append(existingResult.Outputs, result.Outputs...)
86+
87+
// Calculate total successful runs for correct pass ratio calculation
88+
successfulRuns := existingResult.PassRatio*float64(existingResult.Runs) + result.PassRatio*float64(result.Runs)
89+
existingResult.Runs = totalRuns
90+
existingResult.PassRatio = successfulRuns / float64(totalRuns)
91+
existingResult.Skipped = existingResult.Skipped && result.Skipped // Mark as skipped only if all occurrences are skipped
92+
93+
// Update the map with the aggregated result
94+
testMap[key] = existingResult
95+
} else {
96+
// Add new entry to the map
97+
testMap[key] = result
98+
}
99+
}
100+
}
101+
return nil
102+
})
103+
if err != nil {
104+
return nil, fmt.Errorf("error reading files: %v", err)
105+
}
106+
107+
// Convert map to slice of TestResult and set PassRatioPercentage
108+
aggregatedResults := make([]TestResult, 0, len(testMap))
109+
for _, result := range testMap {
110+
result.PassRatioPercentage = fmt.Sprintf("%.0f%%", result.PassRatio*100)
111+
aggregatedResults = append(aggregatedResults, result)
112+
}
113+
114+
// Sort by PassRatio in ascending order
115+
sort.Slice(aggregatedResults, func(i, j int) bool {
116+
return aggregatedResults[i].PassRatio < aggregatedResults[j].PassRatio
117+
})
118+
119+
return aggregatedResults, nil
120+
}
121+
52122
// PrintTests prints tests in a pretty format
53123
func PrintTests(tests []TestResult, w io.Writer) {
54124
for i, test := range tests {

0 commit comments

Comments
 (0)