Skip to content

Commit 2033cd8

Browse files
lukaszcllukaszcl
committed
Refactor flakeguard reports
1 parent e34941d commit 2033cd8

File tree

13 files changed

+1443
-990
lines changed

13 files changed

+1443
-990
lines changed

‎tools/flakeguard/cmd/aggregate_results.go‎

Lines changed: 53 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,100 +1,90 @@
11
package cmd
22

33
import (
4-
"encoding/json"
5-
"log"
6-
"os"
7-
"path/filepath"
4+
"fmt"
85

96
"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
107
"github.com/spf13/cobra"
118
)
129

13-
var (
14-
resultsFolderPath string
15-
outputResultsPath string
16-
outputLogsPath string
17-
codeOwnersPath string
18-
projectPath string
19-
maxPassRatio float64
20-
filterFailed bool
21-
)
22-
2310
var AggregateResultsCmd = &cobra.Command{
2411
Use: "aggregate-results",
25-
Short: "Aggregate test results and optionally filter failed tests based on a threshold",
12+
Short: "Aggregate test results into a single report, with optional filtering and code owners mapping",
2613
RunE: func(cmd *cobra.Command, args []string) error {
27-
// Read test reports from files
28-
var testReports []*reports.TestReport
29-
err := filepath.Walk(resultsFolderPath, func(path string, info os.FileInfo, err error) error {
30-
if err != nil {
31-
return err
32-
}
33-
if !info.IsDir() && filepath.Ext(path) == ".json" {
34-
// Read file content
35-
data, readErr := os.ReadFile(path)
36-
if readErr != nil {
37-
return readErr
38-
}
39-
var report *reports.TestReport
40-
if jsonErr := json.Unmarshal(data, &report); jsonErr != nil {
41-
return jsonErr
42-
}
43-
testReports = append(testReports, report)
44-
}
45-
return nil
46-
})
14+
// Get flag values
15+
aggregateResultsPath, _ := cmd.Flags().GetString("results-path")
16+
aggregateOutputPath, _ := cmd.Flags().GetString("output-path")
17+
includeOutputs, _ := cmd.Flags().GetBool("include-outputs")
18+
includePackageOutputs, _ := cmd.Flags().GetBool("include-package-outputs")
19+
filterFailed, _ := cmd.Flags().GetBool("filter-failed")
20+
maxPassRatio, _ := cmd.Flags().GetFloat64("max-pass-ratio")
21+
codeOwnersPath, _ := cmd.Flags().GetString("codeowners-path")
22+
repoPath, _ := cmd.Flags().GetString("repo-path")
23+
24+
// Load test reports from JSON files
25+
testReports, err := reports.LoadReports(aggregateResultsPath)
4726
if err != nil {
48-
log.Fatalf("Error reading test reports: %v", err)
27+
return fmt.Errorf("error loading test reports: %w", err)
4928
}
5029

51-
allReport, err := reports.Aggregate(testReports...)
30+
// Aggregate the reports
31+
aggregatedReport, err := reports.Aggregate(testReports...)
5232
if err != nil {
53-
log.Fatalf("Error aggregating results: %v", err)
33+
return fmt.Errorf("error aggregating test reports: %w", err)
5434
}
5535

56-
// Map test results to paths
57-
err = reports.MapTestResultsToPaths(allReport, projectPath)
36+
// Map test results to test paths
37+
err = reports.MapTestResultsToPaths(aggregatedReport, repoPath)
5838
if err != nil {
59-
log.Fatalf("Error mapping test results to paths: %v", err)
39+
return fmt.Errorf("error mapping test results to paths: %w", err)
6040
}
6141

62-
// Map test results to owners if CODEOWNERS path is provided
42+
// Map test results to code owners if codeOwnersPath is provided
6343
if codeOwnersPath != "" {
64-
err = reports.MapTestResultsToOwners(allReport, codeOwnersPath)
44+
err = reports.MapTestResultsToOwners(aggregatedReport, codeOwnersPath)
6545
if err != nil {
66-
log.Fatalf("Error mapping test results to owners: %v", err)
46+
return fmt.Errorf("error mapping test results to code owners: %w", err)
6747
}
6848
}
6949

70-
var resultsToSave []reports.TestResult
71-
50+
// Filter results if needed
7251
if filterFailed {
73-
// Filter to only include tests that failed below the threshold
74-
for _, result := range allReport.Results {
75-
if result.PassRatio < maxPassRatio && !result.Skipped {
76-
resultsToSave = append(resultsToSave, result)
52+
aggregatedReport.Results = reports.FilterTests(aggregatedReport.Results, func(tr reports.TestResult) bool {
53+
return !tr.Skipped && tr.PassRatio < maxPassRatio
54+
})
55+
}
56+
57+
// Process the aggregated results based on the flags
58+
if !includeOutputs || !includePackageOutputs {
59+
for i := range aggregatedReport.Results {
60+
if !includeOutputs {
61+
aggregatedReport.Results[i].Outputs = nil
62+
}
63+
if !includePackageOutputs {
64+
aggregatedReport.Results[i].PackageOutputs = nil
7765
}
7866
}
79-
} else {
80-
resultsToSave = allReport.Results
8167
}
82-
allReport.Results = resultsToSave
8368

84-
// Output results to JSON files
85-
if len(resultsToSave) > 0 {
86-
return reports.SaveFilteredResultsAndLogs(outputResultsPath, outputLogsPath, allReport, codeOwnersPath != "")
69+
// Save the aggregated report
70+
if err := reports.SaveReport(reports.OSFileSystem{}, aggregateOutputPath, *aggregatedReport); err != nil {
71+
return fmt.Errorf("error saving aggregated report: %w", err)
8772
}
73+
74+
fmt.Printf("Aggregated report saved to %s\n", aggregateOutputPath)
8875
return nil
8976
},
9077
}
9178

9279
func init() {
93-
AggregateResultsCmd.Flags().StringVarP(&resultsFolderPath, "results-path", "p", "", "Path to the folder containing JSON test result files")
94-
AggregateResultsCmd.Flags().StringVarP(&outputResultsPath, "output-results", "o", "./results", "Path to output the aggregated or filtered test results in JSON and markdown format")
95-
AggregateResultsCmd.Flags().StringVarP(&outputLogsPath, "output-logs", "l", "", "Path to output the filtered test logs in JSON format")
96-
AggregateResultsCmd.Flags().Float64VarP(&maxPassRatio, "max-pass-ratio", "m", 1.0, "The maximum (non-inclusive) pass ratio threshold for a test to be considered a failure. Any tests below this pass rate will be considered flaky.")
97-
AggregateResultsCmd.Flags().BoolVarP(&filterFailed, "filter-failed", "f", false, "If true, filter and output only failed tests based on the max-pass-ratio threshold")
98-
AggregateResultsCmd.Flags().StringVarP(&codeOwnersPath, "codeowners-path", "c", "", "Path to the CODEOWNERS file")
99-
AggregateResultsCmd.Flags().StringVarP(&projectPath, "project-path", "r", ".", "The path to the Go project. Default is the current directory. Useful for subprojects")
80+
AggregateResultsCmd.Flags().StringP("results-path", "p", "", "Path to the folder containing JSON test result files (required)")
81+
AggregateResultsCmd.Flags().StringP("output-path", "o", "./aggregated-results.json", "Path to output the aggregated test results")
82+
AggregateResultsCmd.Flags().Bool("include-outputs", false, "Include test outputs in the aggregated test results")
83+
AggregateResultsCmd.Flags().Bool("include-package-outputs", false, "Include test package outputs in the aggregated test results")
84+
AggregateResultsCmd.Flags().Bool("filter-failed", false, "If true, filter and output only failed tests based on the max-pass-ratio threshold")
85+
AggregateResultsCmd.Flags().Float64("max-pass-ratio", 1.0, "The maximum pass ratio threshold for a test to be considered flaky. Any tests below this pass rate will be considered flaky.")
86+
AggregateResultsCmd.Flags().String("codeowners-path", "", "Path to the CODEOWNERS file")
87+
AggregateResultsCmd.Flags().String("repo-path", ".", "The path to the root of the repository/project")
88+
AggregateResultsCmd.MarkFlagRequired("results-path")
89+
AggregateResultsCmd.MarkFlagRequired("repo-path")
10090
}

‎tools/flakeguard/cmd/check_test_owners.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ var CheckTestOwnersCmd = &cobra.Command{
2020
Use: "check-test-owners",
2121
Short: "Check which tests in the project do not have code owners",
2222
RunE: func(cmd *cobra.Command, args []string) error {
23+
projectPath, _ := cmd.Flags().GetString("project-path")
24+
2325
// Scan project for test functions
2426
testFileMap, err := reports.ScanTestFiles(projectPath)
2527
if err != nil {
@@ -79,7 +81,7 @@ var CheckTestOwnersCmd = &cobra.Command{
7981
}
8082

8183
func init() {
82-
CheckTestOwnersCmd.Flags().StringVarP(&projectPath, "project-path", "p", ".", "Path to the root of the project")
84+
CheckTestOwnersCmd.Flags().StringP("project-path", "p", ".", "Path to the root of the project")
8385
CheckTestOwnersCmd.Flags().StringVarP(&codeownersPath, "codeowners-path", "c", ".github/CODEOWNERS", "Path to the CODEOWNERS file")
8486
CheckTestOwnersCmd.Flags().BoolVarP(&printTestFunctions, "print-test-functions", "t", false, "Print all test functions without owners")
8587
}

‎tools/flakeguard/cmd/filter_results.go‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
package cmd
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
7+
8+
"github.com/spf13/cobra"
9+
)
10+
11+
var (
12+
filterInputPath string
13+
filterOutputPath string
14+
filterMaxPassRatio float64
15+
)
16+
17+
var filterCmd = &cobra.Command{
18+
Use: "filter",
19+
Short: "Filter aggregated test results based on criteria",
20+
RunE: func(cmd *cobra.Command, args []string) error {
21+
// Load the aggregated report
22+
aggregatedReport, err := reports.LoadReport(filterInputPath)
23+
if err != nil {
24+
return fmt.Errorf("error loading aggregated report: %w", err)
25+
}
26+
27+
// Filter the test results
28+
filteredReport := reports.FilterResults(aggregatedReport, filterMaxPassRatio)
29+
30+
// Save the filtered report
31+
if err := reports.SaveReport(reports.OSFileSystem{}, filterOutputPath, *filteredReport); err != nil {
32+
return fmt.Errorf("error saving filtered report: %w", err)
33+
}
34+
35+
fmt.Printf("Filtered report saved to %s\n", filterOutputPath)
36+
return nil
37+
},
38+
}
39+
40+
func init() {
41+
filterCmd.Flags().StringVarP(&filterInputPath, "input-path", "i", "", "Path to the aggregated test results file (required)")
42+
filterCmd.Flags().StringVarP(&filterOutputPath, "output-path", "o", "./filtered-results.json", "Path to output the filtered test results")
43+
filterCmd.Flags().Float64VarP(&filterMaxPassRatio, "max-pass-ratio", "m", 1.0, "Maximum pass ratio threshold for filtering tests")
44+
filterCmd.MarkFlagRequired("input-path")
45+
}

‎tools/flakeguard/cmd/generate_report.go‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
package cmd
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
7+
"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
8+
"github.com/spf13/cobra"
9+
)
10+
11+
var (
12+
reportInputPath string
13+
reportOutputPath string
14+
reportFormat string
15+
reportCodeOwnersPath string
16+
reportProjectPath string
17+
)
18+
19+
var GenerateReportCmd = &cobra.Command{
20+
Use: "generate-report",
21+
Short: "Generate reports from test results",
22+
RunE: func(cmd *cobra.Command, args []string) error {
23+
// Load the test results
24+
testReport, err := reports.LoadReport(reportInputPath)
25+
if err != nil {
26+
return fmt.Errorf("error loading test report: %w", err)
27+
}
28+
29+
// Generate the report
30+
if err := generateReport(testReport, reportFormat, reportOutputPath); err != nil {
31+
return fmt.Errorf("error generating report: %w", err)
32+
}
33+
34+
fmt.Printf("Report generated at %s\n", reportOutputPath)
35+
return nil
36+
},
37+
}
38+
39+
func init() {
40+
GenerateReportCmd.Flags().StringVarP(&reportInputPath, "aggregated-report-path", "i", "", "Path to the aggregated test results file (required)")
41+
GenerateReportCmd.Flags().StringVarP(&reportOutputPath, "output-path", "o", "./report", "Path to output the generated report (without extension)")
42+
GenerateReportCmd.Flags().StringVarP(&reportFormat, "format", "f", "markdown", "Format of the report (markdown, json)")
43+
GenerateReportCmd.MarkFlagRequired("aggregated-report-path")
44+
}
45+
46+
func generateReport(report *reports.TestReport, format, outputPath string) error {
47+
fs := reports.OSFileSystem{}
48+
switch strings.ToLower(format) {
49+
case "markdown":
50+
mdFileName := outputPath + ".md"
51+
mdFile, err := fs.Create(mdFileName)
52+
if err != nil {
53+
return fmt.Errorf("error creating markdown file: %w", err)
54+
}
55+
defer mdFile.Close()
56+
reports.GenerateMarkdownSummary(mdFile, report, 1.0)
57+
fmt.Printf("Markdown report saved to %s\n", mdFileName)
58+
case "json":
59+
jsonFileName := outputPath + ".json"
60+
if err := reports.SaveReportNoLogs(fs, jsonFileName, *report); err != nil {
61+
return fmt.Errorf("error saving JSON report: %w", err)
62+
}
63+
fmt.Printf("JSON report saved to %s\n", jsonFileName)
64+
default:
65+
return fmt.Errorf("unsupported report format: %s", format)
66+
}
67+
68+
// Generate summary JSON
69+
summaryData := reports.GenerateSummaryData(report.Results, 1.0)
70+
summaryFileName := outputPath + "-summary.json"
71+
if err := reports.SaveSummaryAsJSON(fs, summaryFileName, summaryData); err != nil {
72+
return fmt.Errorf("error saving summary JSON: %w", err)
73+
}
74+
fmt.Printf("Summary JSON saved to %s\n", summaryFileName)
75+
76+
return nil
77+
}

‎tools/flakeguard/cmd/run.go‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ var RunTestsCmd = &cobra.Command{
1717
Use: "run",
1818
Short: "Run tests to check if they are flaky",
1919
Run: func(cmd *cobra.Command, args []string) {
20+
// Retrieve flags
2021
projectPath, _ := cmd.Flags().GetString("project-path")
2122
testPackagesJson, _ := cmd.Flags().GetString("test-packages-json")
2223
testPackagesArg, _ := cmd.Flags().GetStringSlice("test-packages")
@@ -37,6 +38,7 @@ var RunTestsCmd = &cobra.Command{
3738
log.Fatalf("Error: %v", err)
3839
}
3940

41+
// Determine test packages
4042
var testPackages []string
4143
if testPackagesJson != "" {
4244
if err := json.Unmarshal([]byte(testPackagesJson), &testPackages); err != nil {
@@ -48,7 +50,8 @@ var RunTestsCmd = &cobra.Command{
4850
log.Fatalf("Error: must specify either --test-packages-json or --test-packages")
4951
}
5052

51-
runner := runner.Runner{
53+
// Initialize the runner
54+
testRunner := runner.Runner{
5255
ProjectPath: projectPath,
5356
Verbose: true,
5457
RunCount: runCount,
@@ -62,7 +65,8 @@ var RunTestsCmd = &cobra.Command{
6265
ShuffleSeed: shuffleSeed,
6366
}
6467

65-
testReport, err := runner.RunTests()
68+
// Run the tests
69+
testReport, err := testRunner.RunTests()
6670
if err != nil {
6771
fmt.Printf("Error running tests: %v\n", err)
6872
os.Exit(1)
@@ -71,7 +75,8 @@ var RunTestsCmd = &cobra.Command{
7175
// Print all failed tests including flaky tests
7276
if printFailedTests {
7377
fmt.Printf("PassRatio threshold for flaky tests: %.2f\n", maxPassRatio)
74-
reports.PrintResults(os.Stdout, testReport.Results, maxPassRatio, false, false)
78+
// Use RenderResults instead of PrintResults
79+
reports.RenderResults(os.Stdout, testReport.Results, maxPassRatio, false)
7580
}
7681

7782
// Save the test results in JSON format
@@ -80,14 +85,20 @@ var RunTestsCmd = &cobra.Command{
8085
if err != nil {
8186
log.Fatalf("Error marshaling test results to JSON: %v", err)
8287
}
83-
if err := os.WriteFile(outputPath, jsonData, 0644); err != nil { //nolint:gosec
88+
if err := os.WriteFile(outputPath, jsonData, 0644); err != nil {
8489
log.Fatalf("Error writing test results to file: %v", err)
8590
}
8691
fmt.Printf("All test results saved to %s\n", outputPath)
8792
}
8893

89-
flakyTests := reports.FilterFlakyTests(testReport.Results, maxPassRatio)
94+
// Filter flaky tests using FilterTests
95+
flakyTests := reports.FilterTests(testReport.Results, func(tr reports.TestResult) bool {
96+
return !tr.Skipped && tr.PassRatio < maxPassRatio
97+
})
98+
9099
if len(flakyTests) > 0 {
100+
fmt.Printf("Found %d flaky tests below the pass ratio threshold of %.2f:\n", len(flakyTests), maxPassRatio)
101+
reports.RenderResults(os.Stdout, flakyTests, maxPassRatio, false)
91102
// Exit with error code if there are flaky tests
92103
os.Exit(1)
93104
} else if len(testReport.Results) == 0 {
@@ -112,7 +123,7 @@ func init() {
112123
RunTestsCmd.Flags().StringSlice("skip-tests", nil, "Comma-separated list of test names to skip from running")
113124
RunTestsCmd.Flags().StringSlice("select-tests", nil, "Comma-separated list of test names to specifically run")
114125
RunTestsCmd.Flags().Bool("print-failed-tests", true, "Print failed test results to the console")
115-
RunTestsCmd.Flags().Float64("max-pass-ratio", 1.0, "The maximum (non-inclusive) pass ratio threshold for a test to be considered a failure. Any tests below this pass rate will be considered flaky.")
126+
RunTestsCmd.Flags().Float64("max-pass-ratio", 1.0, "The maximum pass ratio threshold for a test to be considered flaky. Any tests below this pass rate will be considered flaky.")
116127
}
117128

118129
func checkDependencies(projectPath string) error {

‎tools/flakeguard/main.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ func init() {
3030
rootCmd.AddCommand(cmd.RunTestsCmd)
3131
rootCmd.AddCommand(cmd.AggregateResultsCmd)
3232
rootCmd.AddCommand(cmd.CheckTestOwnersCmd)
33+
rootCmd.AddCommand(cmd.GenerateReportCmd)
3334
}
3435

3536
func main() {

0 commit comments

Comments
 (0)