Skip to content

Commit 2a40cb6

Browse files
lukaszcllukaszcl
committed
Rename aggregate cmd to generate-test-report
1 parent 61140fb commit 2a40cb6

File tree

6 files changed

+166
-400
lines changed

6 files changed

+166
-400
lines changed

tools/flakeguard/cmd/generate_report.go renamed to tools/flakeguard/cmd/generate_github_report.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import (
1414
)
1515

1616
var GenerateReportCmd = &cobra.Command{
17-
Use: "generate-github-reports",
17+
Use: "generate-github-report",
1818
Short: "Generate Github reports from Flakeguard test report",
1919
Run: func(cmd *cobra.Command, args []string) {
2020
fs := reports.OSFileSystem{}

tools/flakeguard/cmd/aggregate_results.go renamed to tools/flakeguard/cmd/generate_test_report.go

Lines changed: 81 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -9,32 +9,41 @@ import (
99
"github.com/briandowns/spinner"
1010
"github.com/rs/zerolog/log"
1111
"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
12+
"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/utils"
1213
"github.com/spf13/cobra"
1314
)
1415

1516
var AggregateResultsCmd = &cobra.Command{
16-
Use: "aggregate-results",
17-
Short: "Aggregate test results into a single JSON report",
17+
Use: "generate-test-report",
18+
Short: "Generate test report based on test results",
1819
Run: func(cmd *cobra.Command, args []string) {
1920
fs := reports.OSFileSystem{}
2021

2122
// Get flag values
22-
resultsPath, _ := cmd.Flags().GetString("results-path")
23+
testResultsDir, _ := cmd.Flags().GetString("test-results-dir")
2324
outputDir, _ := cmd.Flags().GetString("output-path")
2425
maxPassRatio, _ := cmd.Flags().GetFloat64("max-pass-ratio")
25-
codeOwnersPath, _ := cmd.Flags().GetString("codeowners-path")
26+
projectPath, _ := cmd.Flags().GetString("project-path")
2627
repoPath, _ := cmd.Flags().GetString("repo-path")
28+
codeOwnersPath, _ := cmd.Flags().GetString("codeowners-path")
29+
useRace, _ := cmd.Flags().GetBool("race")
2730
repoURL, _ := cmd.Flags().GetString("repo-url")
2831
branchName, _ := cmd.Flags().GetString("branch-name")
2932
headSHA, _ := cmd.Flags().GetString("head-sha")
3033
baseSHA, _ := cmd.Flags().GetString("base-sha")
3134
githubWorkflowName, _ := cmd.Flags().GetString("github-workflow-name")
3235
githubWorkflowRunURL, _ := cmd.Flags().GetString("github-workflow-run-url")
3336
reportID, _ := cmd.Flags().GetString("report-id")
37+
genReportID, _ := cmd.Flags().GetBool("gen-report-id")
38+
39+
goProject, err := utils.GetGoProjectName(projectPath)
40+
if err != nil {
41+
log.Warn().Err(err).Str("projectPath", goProject).Msg("Failed to get pretty project path")
42+
}
3443

35-
initialDirSize, err := getDirSize(resultsPath)
44+
initialDirSize, err := getDirSize(testResultsDir)
3645
if err != nil {
37-
log.Error().Err(err).Str("path", resultsPath).Msg("Error getting initial directory size")
46+
log.Error().Err(err).Str("path", testResultsDir).Msg("Error getting initial directory size")
3847
// intentionally don't exit here, as we can still proceed with the aggregation
3948
}
4049

@@ -44,39 +53,28 @@ var AggregateResultsCmd = &cobra.Command{
4453
os.Exit(ErrorExitCode)
4554
}
4655

47-
// Start spinner for loading test reports
56+
// Start spinner for loading test results
4857
s := spinner.New(spinner.CharSets[11], 100*time.Millisecond)
49-
s.Suffix = " Aggregating test reports..."
58+
s.Suffix = " Aggregating test results..."
5059
s.Start()
5160
fmt.Println()
5261

53-
// Load test reports from JSON files and aggregate them
54-
aggregatedReport, err := reports.LoadAndAggregate(
55-
resultsPath,
56-
reports.WithRepoPath(repoPath),
57-
reports.WithCodeOwnersPath(codeOwnersPath),
58-
reports.WithReportID(reportID),
59-
reports.WithBranchName(branchName),
60-
reports.WithBaseSha(baseSHA),
61-
reports.WithHeadSha(headSHA),
62-
reports.WithRepoURL(repoURL),
63-
reports.WithGitHubWorkflowName(githubWorkflowName),
64-
reports.WithGitHubWorkflowRunURL(githubWorkflowRunURL),
65-
)
62+
// Load test results from JSON files and aggregate them
63+
aggregatedResults, err := reports.LoadAndAggregate(testResultsDir)
6664
if err != nil {
6765
s.Stop()
68-
log.Error().Err(err).Stack().Msg("Error aggregating test reports")
66+
log.Error().Err(err).Stack().Msg("Error aggregating test results")
6967
os.Exit(ErrorExitCode)
7068
}
7169
s.Stop()
72-
log.Debug().Msg("Successfully loaded and aggregated test reports")
70+
log.Debug().Msg("Successfully loaded and aggregated test results")
7371

7472
// Start spinner for mapping test results to paths
7573
s = spinner.New(spinner.CharSets[11], 100*time.Millisecond)
7674
s.Suffix = " Filter failed tests..."
7775
s.Start()
7876

79-
failedTests := reports.FilterTests(aggregatedReport.Results, func(tr reports.TestResult) bool {
77+
failedTests := reports.FilterTests(aggregatedResults, func(tr reports.TestResult) bool {
8078
return !tr.Skipped && tr.PassRatio < maxPassRatio
8179
})
8280
s.Stop()
@@ -86,25 +84,33 @@ var AggregateResultsCmd = &cobra.Command{
8684
log.Info().Int("count", len(failedTests)).Msg("Found failed tests")
8785

8886
// Create a new report for failed tests with logs
89-
failedReportWithLogs := &reports.TestReport{
90-
GoProject: aggregatedReport.GoProject,
91-
SummaryData: aggregatedReport.SummaryData,
92-
RaceDetection: aggregatedReport.RaceDetection,
93-
ExcludedTests: aggregatedReport.ExcludedTests,
94-
SelectedTests: aggregatedReport.SelectedTests,
95-
HeadSHA: aggregatedReport.HeadSHA,
96-
BaseSHA: aggregatedReport.BaseSHA,
97-
GitHubWorkflowName: aggregatedReport.GitHubWorkflowName,
98-
Results: failedTests,
87+
failedReportWithLogs, err := reports.NewTestReport(failedTests,
88+
reports.WithGoProject(goProject),
89+
reports.WithProjectPath(projectPath),
90+
reports.WithRepoPath(repoPath),
91+
reports.WithCodeOwnersPath(codeOwnersPath),
92+
reports.WithReportID(reportID),
93+
reports.WithGeneratedReportID(genReportID),
94+
reports.WithGoRaceDetection(useRace),
95+
reports.WithBranchName(branchName),
96+
reports.WithBaseSha(baseSHA),
97+
reports.WithHeadSha(headSHA),
98+
reports.WithRepoURL(repoURL),
99+
reports.WithGitHubWorkflowName(githubWorkflowName),
100+
reports.WithGitHubWorkflowRunURL(githubWorkflowRunURL),
101+
)
102+
if err != nil {
103+
log.Error().Stack().Err(err).Msg("Error creating failed tests report with logs")
104+
os.Exit(ErrorExitCode)
99105
}
100106

101107
// Save the failed tests report with logs
102-
failedTestsReportWithLogsPath := filepath.Join(outputDir, "failed-test-results-with-logs.json")
103-
if err := reports.SaveReport(fs, failedTestsReportWithLogsPath, *failedReportWithLogs); err != nil {
108+
failedTestsReportWithLogsPath := filepath.Join(outputDir, "failed-test-report-with-logs.json")
109+
if err := reports.SaveReport(fs, failedTestsReportWithLogsPath, failedReportWithLogs); err != nil {
104110
log.Error().Stack().Err(err).Msg("Error saving failed tests report with logs")
105111
os.Exit(ErrorExitCode)
106112
}
107-
log.Debug().Str("path", failedTestsReportWithLogsPath).Msg("Failed tests report with logs saved")
113+
log.Info().Str("path", failedTestsReportWithLogsPath).Msg("Failed tests report with logs saved")
108114

109115
// Remove logs from test results for the report without logs
110116
for i := range failedReportWithLogs.Results {
@@ -114,42 +120,64 @@ var AggregateResultsCmd = &cobra.Command{
114120
}
115121

116122
// Save the failed tests report without logs
117-
failedTestsReportNoLogsPath := filepath.Join(outputDir, "failed-test-results.json")
118-
if err := reports.SaveReport(fs, failedTestsReportNoLogsPath, *failedReportWithLogs); err != nil {
123+
failedTestsReportNoLogsPath := filepath.Join(outputDir, "failed-test-report.json")
124+
if err := reports.SaveReport(fs, failedTestsReportNoLogsPath, failedReportWithLogs); err != nil {
119125
log.Error().Stack().Err(err).Msg("Error saving failed tests report without logs")
120126
os.Exit(ErrorExitCode)
121127
}
122-
log.Debug().Str("path", failedTestsReportNoLogsPath).Msg("Failed tests report without logs saved")
128+
log.Info().Str("path", failedTestsReportNoLogsPath).Msg("Failed tests report without logs saved")
123129
} else {
124-
log.Debug().Msg("No failed tests found. Skipping generation of failed tests reports")
130+
log.Info().Msg("No failed tests found. Skipping generation of failed tests reports")
125131
}
126132

127133
// Remove logs from test results for the aggregated report
128-
for i := range aggregatedReport.Results {
129-
aggregatedReport.Results[i].PassedOutputs = nil
130-
aggregatedReport.Results[i].FailedOutputs = nil
131-
aggregatedReport.Results[i].PackageOutputs = nil
134+
for i := range aggregatedResults {
135+
aggregatedResults[i].PassedOutputs = nil
136+
aggregatedResults[i].FailedOutputs = nil
137+
aggregatedResults[i].PackageOutputs = nil
138+
}
139+
140+
aggregatedReport, err := reports.NewTestReport(aggregatedResults,
141+
reports.WithGoProject(goProject),
142+
reports.WithProjectPath(projectPath),
143+
reports.WithRepoPath(repoPath),
144+
reports.WithCodeOwnersPath(codeOwnersPath),
145+
reports.WithReportID(reportID),
146+
reports.WithGeneratedReportID(genReportID),
147+
reports.WithGoRaceDetection(useRace),
148+
reports.WithBranchName(branchName),
149+
reports.WithBaseSha(baseSHA),
150+
reports.WithHeadSha(headSHA),
151+
reports.WithRepoURL(repoURL),
152+
reports.WithGitHubWorkflowName(githubWorkflowName),
153+
reports.WithGitHubWorkflowRunURL(githubWorkflowRunURL),
154+
)
155+
if err != nil {
156+
log.Error().Stack().Err(err).Msg("Error creating aggregated test report")
157+
os.Exit(ErrorExitCode)
132158
}
133159

134160
// Save the aggregated report to the output directory
135-
aggregatedReportPath := filepath.Join(outputDir, "all-test-results.json")
136-
if err := reports.SaveReport(fs, aggregatedReportPath, *aggregatedReport); err != nil {
161+
aggregatedReportPath := filepath.Join(outputDir, "all-test-report.json")
162+
if err := reports.SaveReport(fs, aggregatedReportPath, aggregatedReport); err != nil {
137163
log.Error().Stack().Err(err).Msg("Error saving aggregated test report")
138164
os.Exit(ErrorExitCode)
139165
}
166+
log.Info().Str("path", aggregatedReportPath).Msg("All tests report without logs saved")
140167

141-
finalDirSize, err := getDirSize(resultsPath)
168+
finalDirSize, err := getDirSize(testResultsDir)
142169
if err != nil {
143-
log.Error().Err(err).Str("path", resultsPath).Msg("Error getting final directory size")
170+
log.Error().Err(err).Str("path", testResultsDir).Msg("Error getting final directory size")
144171
// intentionally don't exit here, as we can still proceed with the aggregation
145172
}
146173
diskSpaceUsed := byteCountSI(finalDirSize - initialDirSize)
147-
log.Info().Str("disk space used", diskSpaceUsed).Str("report", aggregatedReportPath).Msg("Aggregation complete")
174+
log.Info().Str("disk space used", diskSpaceUsed).Msg("Aggregation complete")
148175
},
149176
}
150177

151178
func init() {
152-
AggregateResultsCmd.Flags().StringP("results-path", "p", "", "Path to the folder containing JSON test result files (required)")
179+
AggregateResultsCmd.Flags().StringP("test-results-dir", "p", "", "Path to the folder containing JSON test result files (required)")
180+
AggregateResultsCmd.Flags().StringP("project-path", "r", ".", "The path to the Go project. Default is the current directory. Useful for subprojects")
153181
AggregateResultsCmd.Flags().StringP("output-path", "o", "./report", "Path to output the aggregated results (directory)")
154182
AggregateResultsCmd.Flags().Float64P("max-pass-ratio", "", 1.0, "The maximum pass ratio threshold for a test to be considered flaky")
155183
AggregateResultsCmd.Flags().StringP("codeowners-path", "", "", "Path to the CODEOWNERS file")
@@ -161,8 +189,9 @@ func init() {
161189
AggregateResultsCmd.Flags().String("github-workflow-name", "", "GitHub workflow name for the test report")
162190
AggregateResultsCmd.Flags().String("github-workflow-run-url", "", "GitHub workflow run URL for the test report")
163191
AggregateResultsCmd.Flags().String("report-id", "", "Optional identifier for the test report. Will be generated if not provided")
192+
AggregateResultsCmd.Flags().Bool("race", false, "Enable the race detector")
164193

165-
if err := AggregateResultsCmd.MarkFlagRequired("results-path"); err != nil {
194+
if err := AggregateResultsCmd.MarkFlagRequired("test-results-dir"); err != nil {
166195
log.Fatal().Err(err).Msg("Error marking flag as required")
167196
}
168197
}

‎tools/flakeguard/cmd/send_to_splunk.go‎

Lines changed: 0 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,6 @@ var SendToSplunkCmd = &cobra.Command{
1919
splunkToken, _ := cmd.Flags().GetString("splunk-token")
2020
splunkEvent, _ := cmd.Flags().GetString("splunk-event")
2121
failLogsURL, _ := cmd.Flags().GetString("failed-logs-url")
22-
repoURL, _ := cmd.Flags().GetString("repo-url")
23-
branchName, _ := cmd.Flags().GetString("branch-name")
24-
headSHA, _ := cmd.Flags().GetString("head-sha")
25-
baseSHA, _ := cmd.Flags().GetString("base-sha")
26-
githubWorkflowName, _ := cmd.Flags().GetString("github-workflow-name")
27-
githubWorkflowRunURL, _ := cmd.Flags().GetString("github-workflow-run-url")
28-
reportID, _ := cmd.Flags().GetString("report-id")
29-
genReportID, _ := cmd.Flags().GetBool("gen-report-id")
30-
repoPath, _ := cmd.Flags().GetString("repo-path")
31-
codeownersPath, _ := cmd.Flags().GetString("codeowners-path")
3222

3323
// Read the report file.
3424
data, err := os.ReadFile(reportPath)
@@ -43,43 +33,7 @@ var SendToSplunkCmd = &cobra.Command{
4333
log.Error().Err(err).Msg("Error unmarshalling report JSON")
4434
os.Exit(1)
4535
}
46-
testReport.GenerateSummaryData()
4736

48-
// Override report fields with flags if provided.
49-
if repoURL != "" {
50-
testReport.RepoURL = repoURL
51-
}
52-
if branchName != "" {
53-
testReport.BranchName = branchName
54-
}
55-
if headSHA != "" {
56-
testReport.HeadSHA = headSHA
57-
}
58-
if baseSHA != "" {
59-
testReport.BaseSHA = baseSHA
60-
}
61-
if githubWorkflowName != "" {
62-
testReport.GitHubWorkflowName = githubWorkflowName
63-
}
64-
if githubWorkflowRunURL != "" {
65-
testReport.GitHubWorkflowRunURL = githubWorkflowRunURL
66-
}
67-
if reportID != "" {
68-
testReport.SetReportID(reportID)
69-
}
70-
if genReportID {
71-
testReport.SetRandomReportID()
72-
}
73-
if repoPath != "" {
74-
err = reports.MapTestResultsToPaths(&testReport, repoPath)
75-
if err != nil {
76-
log.Error().Err(err).Msg("Error mapping test results to paths")
77-
os.Exit(1)
78-
}
79-
}
80-
if codeownersPath != "" && repoPath != "" {
81-
reports.MapTestResultsToOwners(&testReport, codeownersPath)
82-
}
8337
if failLogsURL != "" {
8438
testReport.FailedLogsURL = failLogsURL
8539
}
@@ -102,16 +56,6 @@ func init() {
10256
SendToSplunkCmd.Flags().String("splunk-url", "", "Optional URL to send the test results to Splunk")
10357
SendToSplunkCmd.Flags().String("splunk-token", "", "Optional Splunk HEC token to send the test results")
10458
SendToSplunkCmd.Flags().String("splunk-event", "", "Optional Splunk event to send as the triggering event for the test results")
105-
SendToSplunkCmd.Flags().String("repo-url", "", "The repository URL")
106-
SendToSplunkCmd.Flags().String("branch-name", "", "Branch name for the test report")
107-
SendToSplunkCmd.Flags().String("head-sha", "", "Head commit SHA for the test report")
108-
SendToSplunkCmd.Flags().String("base-sha", "", "Base commit SHA for the test report")
109-
SendToSplunkCmd.Flags().String("github-workflow-name", "", "GitHub workflow name for the test report")
110-
SendToSplunkCmd.Flags().String("github-workflow-run-url", "", "GitHub workflow run URL for the test report")
111-
SendToSplunkCmd.Flags().String("report-id", "", "Optional identifier for the test report. Will be generated if not provided")
112-
SendToSplunkCmd.Flags().StringP("repo-path", "", ".", "The path to the root of the repository/project")
113-
SendToSplunkCmd.Flags().StringP("codeowners-path", "", "", "Path to the CODEOWNERS file")
114-
SendToSplunkCmd.Flags().Bool("gen-report-id", false, "Generate a random report ID")
11559

11660
// Mark required flags.
11761
if err := SendToSplunkCmd.MarkFlagRequired("report-path"); err != nil {

‎tools/flakeguard/reports/data.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"encoding/json"
55
"fmt"
66
"os"
7+
"path/filepath"
78
"sort"
89
"strings"
910
"time"
@@ -36,6 +37,12 @@ type TestResult struct {
3637
}
3738

3839
func SaveTestResultsToFile(results []TestResult, filePath string) error {
40+
// Create directory path if it doesn't exist
41+
dir := filepath.Dir(filePath)
42+
if err := os.MkdirAll(dir, 0o755); err != nil {
43+
return fmt.Errorf("error creating directories: %w", err)
44+
}
45+
3946
jsonData, err := json.MarshalIndent(results, "", " ")
4047
if err != nil {
4148
return fmt.Errorf("error marshaling test results to JSON: %w", err)

0 commit comments

Comments
 (0)