Skip to content

Commit 76e3b96

Browse files
kalverrakalverra
authored
Fixes Flakeguard Reports (#1623)
1 parent 103764e commit 76e3b96

File tree

11 files changed

+151
-123
lines changed

11 files changed

+151
-123
lines changed

‎CODEOWNERS‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
* @sebawo @smartcontractkit/test-tooling-team
1+
* @smartcontractkit/test-tooling-team

‎tools/flakeguard/Makefile‎

Lines changed: 2 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -24,60 +24,8 @@ bench:
2424

2525
.PHONY: example
2626
example:
27-
rm -rf example_results
28-
mkdir -p example_results
29-
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic,Timeout --max-pass-ratio=1 --race=false --output-json=example_results/example_run_1.json
30-
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic,Timeout --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
31-
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic,Timeout --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
32-
go run . aggregate-results \
33-
--results-path ./example_results \
34-
--output-path ./example_results \
35-
--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
36-
--branch-name "example-branch" \
37-
--base-sha "abc" \
38-
--head-sha "xyz" \
39-
--github-workflow-name "ExampleWorkflowName" \
40-
--github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
41-
--splunk-url "https://splunk.example.com" \
42-
--splunk-token "splunk-token" \
43-
--splunk-event "example-splunk-event"
27+
./run_example.sh 3 Panic,Timeout
4428

4529
.PHONY: example_flaky_panic
4630
example_flaky_panic:
47-
rm -rf example_results
48-
mkdir -p example_results
49-
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_1.json
50-
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
51-
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
52-
go run . aggregate-results \
53-
--results-path ./example_results \
54-
--output-path ./example_results \
55-
--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
56-
--branch-name "example-branch" \
57-
--base-sha "abc" \
58-
--head-sha "xyz" \
59-
--github-workflow-name "ExampleWorkflowName" \
60-
--github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
61-
--splunk-url "https://splunk.example.com" \
62-
--splunk-token "splunk-token" \
63-
--splunk-event "example-splunk-event"
64-
65-
.PHONY: example_timeout
66-
example_timeout:
67-
rm -rf example_results
68-
mkdir -p example_results
69-
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --select-tests=TestTimeout --timeout=1s --max-pass-ratio=1 --race=false --output-json=example_results/example_run_1.json
70-
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --select-tests=TestTimeout --timeout=1s --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
71-
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --select-tests=TestTimeout --timeout=1s --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
72-
go run . aggregate-results \
73-
--results-path ./example_results \
74-
--output-path ./example_results \
75-
--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
76-
--branch-name "example-branch" \
77-
--base-sha "abc" \
78-
--head-sha "xyz" \
79-
--github-workflow-name "ExampleWorkflowName" \
80-
--github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
81-
--splunk-url "https://splunk.example.com" \
82-
--splunk-token "splunk-token" \
83-
--splunk-event "example-splunk-event"
31+
./run_example.sh 3 TestPanic

‎tools/flakeguard/README.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ Both `find` and `run` commands support JSON output `--json`, making it easy to i
3636
You can find example usage and see outputs with:
3737

3838
```sh
39-
make example # Run an example flow of running tests and aggregating results
40-
make example_panic # Run example flow with panicking tests
41-
ls example_results # See results of each run and aggregation
39+
make example # Run an example flow of running tests, aggregating results, and reporting them to GitHub
40+
make example_flaky_panic # Run example flow with flaky and panicking tests
41+
ls example_results # See results of each run and aggregation
4242
```

‎tools/flakeguard/cmd/generate_report.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ import (
1717
"golang.org/x/oauth2"
1818
)
1919

20+
const exampleGitHubToken = "EXAMPLE_GITHUB_TOKEN" //nolint:gosec
21+
2022
var GenerateReportCmd = &cobra.Command{
2123
Use: "generate-report",
2224
Short: "Generate test reports from aggregated results that can be posted to GitHub",
@@ -191,6 +193,9 @@ func init() {
191193
}
192194

193195
func fetchArtifactLink(githubToken, githubRepo string, githubRunID int64, artifactName string) (string, error) {
196+
if githubToken == exampleGitHubToken {
197+
return "https://example-artifact-link.com", nil
198+
}
194199
ctx := context.Background()
195200
ts := oauth2.StaticTokenSource(
196201
&oauth2.Token{AccessToken: githubToken},

‎tools/flakeguard/reports/data.go‎

Lines changed: 57 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,61 @@ type TestReport struct {
2626
MaxPassRatio float64 `json:"max_pass_ratio,omitempty"`
2727
}
2828

29+
// GenerateSummaryData generates a summary of a report's test results
30+
func (testReport *TestReport) GenerateSummaryData() {
31+
var runs, testRunCount, passes, fails, skips, panickedTests, racedTests, flakyTests, skippedTests int
32+
33+
for _, result := range testReport.Results {
34+
runs += result.Runs
35+
if result.Runs > testRunCount {
36+
testRunCount = result.Runs
37+
}
38+
passes += result.Successes
39+
fails += result.Failures
40+
skips += result.Skips
41+
42+
// Count tests that were entirely skipped
43+
if result.Runs == 0 && result.Skipped {
44+
skippedTests++
45+
}
46+
47+
if result.Panic {
48+
panickedTests++
49+
flakyTests++
50+
} else if result.Race {
51+
racedTests++
52+
flakyTests++
53+
} else if !result.Skipped && result.Runs > 0 && result.PassRatio < testReport.MaxPassRatio {
54+
flakyTests++
55+
}
56+
}
57+
58+
// Calculate the raw pass ratio
59+
passRatio := passRatio(passes, runs)
60+
61+
// Calculate the raw flake ratio
62+
totalTests := len(testReport.Results)
63+
flakeRatio := flakeRatio(flakyTests, totalTests)
64+
65+
passRatioStr := formatRatio(passRatio)
66+
flakeTestRatioStr := formatRatio(flakeRatio)
67+
68+
testReport.SummaryData = &SummaryData{
69+
UniqueTestsRun: totalTests,
70+
TestRunCount: testRunCount,
71+
PanickedTests: panickedTests,
72+
RacedTests: racedTests,
73+
FlakyTests: flakyTests,
74+
FlakyTestPercent: flakeTestRatioStr,
75+
76+
TotalRuns: runs,
77+
PassedRuns: passes,
78+
FailedRuns: fails,
79+
SkippedRuns: skips,
80+
PassPercent: passRatioStr,
81+
}
82+
}
83+
2984
// TestResult contains the results and outputs of a single test
3085
type TestResult struct {
3186
// ReportID is the ID of the report this test result belongs to
@@ -66,7 +121,7 @@ type SummaryData struct {
66121
RacedTests int `json:"raced_tests"`
67122
// FlakyTests tracks how many tests are considered flaky
68123
FlakyTests int `json:"flaky_tests"`
69-
// FlakyTestPercent is the percentage of tests that are considered flaky
124+
// FlakyTestPercent is the human-readable percentage of tests that are considered flaky
70125
FlakyTestPercent string `json:"flaky_test_percent"`
71126

72127
// Individual test run counts
@@ -79,7 +134,7 @@ type SummaryData struct {
79134
FailedRuns int `json:"failed_runs"`
80135
// SkippedRuns tracks how many test runs were skipped
81136
SkippedRuns int `json:"skipped_runs"`
82-
// PassPercent is the percentage of test runs that passed
137+
// PassPercent is the human-readable percentage of test runs that passed
83138
PassPercent string `json:"pass_percent"`
84139
}
85140

@@ -129,61 +184,6 @@ type SplunkTestResultEvent struct {
129184

130185
// Data Processing Functions
131186

132-
// GenerateSummaryData generates a summary of a report's test results
133-
func GenerateSummaryData(testReport *TestReport) {
134-
var runs, mostRuns, passes, fails, skips, panickedTests, racedTests, flakyTests, skippedTests int
135-
136-
for _, result := range testReport.Results {
137-
runs += result.Runs
138-
if result.Runs > mostRuns {
139-
mostRuns = result.Runs
140-
}
141-
passes += result.Successes
142-
fails += result.Failures
143-
skips += result.Skips
144-
145-
// Count tests that were entirely skipped
146-
if result.Runs == 0 && result.Skipped {
147-
skippedTests++
148-
}
149-
150-
if result.Panic {
151-
panickedTests++
152-
flakyTests++
153-
} else if result.Race {
154-
racedTests++
155-
flakyTests++
156-
} else if !result.Skipped && result.Runs > 0 && result.PassRatio < testReport.MaxPassRatio {
157-
flakyTests++
158-
}
159-
}
160-
161-
// Calculate the raw pass ratio
162-
passRatio := passRatio(passes, runs)
163-
164-
// Calculate the raw flake ratio
165-
totalTests := len(testReport.Results)
166-
flakeRatio := flakeRatio(flakyTests, totalTests)
167-
168-
passRatioStr := formatRatio(passRatio)
169-
flakeTestRatioStr := formatRatio(flakeRatio)
170-
171-
testReport.SummaryData = &SummaryData{
172-
UniqueTestsRun: totalTests,
173-
TestRunCount: mostRuns,
174-
PanickedTests: panickedTests,
175-
RacedTests: racedTests,
176-
FlakyTests: flakyTests,
177-
FlakyTestPercent: flakeTestRatioStr,
178-
179-
TotalRuns: runs,
180-
PassedRuns: passes,
181-
FailedRuns: fails,
182-
SkippedRuns: skips,
183-
PassPercent: passRatioStr,
184-
}
185-
}
186-
187187
func FilterResults(report *TestReport, maxPassRatio float64) *TestReport {
188188
filteredResults := FilterTests(report.Results, func(tr TestResult) bool {
189189
return !tr.Skipped && tr.PassRatio < maxPassRatio

‎tools/flakeguard/reports/data_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ func TestGenerateSummaryData(t *testing.T) {
205205

206206
for _, tc := range tests {
207207
t.Run(tc.name, func(t *testing.T) {
208-
GenerateSummaryData(tc.testReport)
208+
tc.testReport.GenerateSummaryData()
209209
assert.Equal(t, tc.expected, tc.testReport.SummaryData, "Summary data does not match expected")
210210
})
211211
}

‎tools/flakeguard/reports/io.go‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,9 @@ func LoadAndAggregate(resultsPath string, options ...AggregateOption) (*TestRepo
127127
}
128128

129129
// Apply options
130-
opts := aggregateOptions{}
130+
opts := aggregateOptions{
131+
maxPassRatio: 1.0,
132+
}
131133
for _, opt := range options {
132134
opt(&opts)
133135
}
@@ -491,10 +493,13 @@ func aggregate(reportChan <-chan *TestReport, errChan <-chan error, opts *aggreg
491493

492494
sortTestResults(aggregatedResults)
493495
fullReport.Results = aggregatedResults
494-
GenerateSummaryData(fullReport)
496+
fullReport.GenerateSummaryData()
495497

496498
if sendToSplunk {
497499
err = sendDataToSplunk(opts, *fullReport)
500+
if err != nil {
501+
return fullReport, fmt.Errorf("error sending data to Splunk: %w", err)
502+
}
498503
}
499504
return fullReport, err
500505
}
@@ -630,13 +635,15 @@ func sendDataToSplunk(opts *aggregateOptions, report TestReport) error {
630635
Int("successfully sent", successfulResultsSent).
631636
Int("total results", len(results)).
632637
Errs("errors", splunkErrs).
638+
Str("report id", report.ID).
633639
Str("duration", time.Since(start).String()).
634640
Msg("Errors occurred while sending test results to Splunk")
635641
} else {
636642
log.Debug().
637643
Int("successfully sent", successfulResultsSent).
638644
Int("total results", len(results)).
639645
Str("duration", time.Since(start).String()).
646+
Str("report id", report.ID).
640647
Msg("All results sent successfully to Splunk")
641648
}
642649

‎tools/flakeguard/reports/presentation.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,10 +90,10 @@ func GenerateGitHubSummaryMarkdown(w io.Writer, testReport *TestReport, maxPassR
9090

9191
if testReport.SummaryData.FlakyTests > 0 {
9292
fmt.Fprintln(w, "## Found Flaky Tests :x:")
93-
fmt.Fprintln(w)
9493
} else {
9594
fmt.Fprintln(w, "## No Flakes Found :white_check_mark:")
9695
}
96+
fmt.Fprintln(w)
9797

9898
RenderResults(w, testReport, true, false)
9999

@@ -199,12 +199,12 @@ func renderSummaryTable(w io.Writer, summary *SummaryData, markdown bool, collap
199199
{"Panicked Tests", fmt.Sprintf("%d", summary.PanickedTests)},
200200
{"Raced Tests", fmt.Sprintf("%d", summary.RacedTests)},
201201
{"Flaky Tests", fmt.Sprintf("%d", summary.FlakyTests)},
202-
{"Flaky Test Ratio", summary.FlakyTestPercent},
202+
{"Flaky Test Percent", summary.FlakyTestPercent},
203203
{"Total Test Runs", fmt.Sprintf("%d", summary.TotalRuns)},
204204
{"Passes", fmt.Sprintf("%d", summary.PassedRuns)},
205205
{"Failures", fmt.Sprintf("%d", summary.FailedRuns)},
206206
{"Skips", fmt.Sprintf("%d", summary.SkippedRuns)},
207-
{"Pass Ratio", summary.PassPercent},
207+
{"Pass Percent", summary.PassPercent},
208208
}
209209
if markdown {
210210
for i, row := range summaryData {

‎tools/flakeguard/reports/presentation_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,7 @@ func TestRenderResults(t *testing.T) {
241241
for _, tc := range testcases {
242242
t.Run(tc.name, func(t *testing.T) {
243243
// Generate the summary data
244-
GenerateSummaryData(tc.testReport)
244+
tc.testReport.GenerateSummaryData()
245245

246246
var buf bytes.Buffer
247247
RenderResults(&buf, tc.testReport, false, false)

‎tools/flakeguard/run_example.sh‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
#!/bin/bash
2+
# Used to run flakeguard on the example test package
3+
4+
run_flakeguard_example() {
5+
local run_count=$1
6+
local skip_tests=$2
7+
8+
echo "Running Flakeguard $run_count times"
9+
10+
for ((i=1; i<=$run_count; i++)); do
11+
output_file="example_results/example_run_$i.json"
12+
go run . run \
13+
--project-path=./runner \
14+
--test-packages=./example_test_package \
15+
--run-count=5 \
16+
--skip-tests=$skip_tests \
17+
--max-pass-ratio=1 \
18+
--race=false \
19+
--output-json=$output_file
20+
local EXIT_CODE=$?
21+
if [ $EXIT_CODE -eq 2 ]; then
22+
echo "ERROR: Flakeguard encountered an error while running tests"
23+
exit $EXIT_CODE
24+
fi
25+
done
26+
27+
go run . aggregate-results \
28+
--results-path ./example_results \
29+
--output-path ./example_results \
30+
--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
31+
--branch-name "example-branch" \
32+
--base-sha "abc" \
33+
--head-sha "xyz" \
34+
--github-workflow-name "ExampleWorkflowName" \
35+
--github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
36+
--splunk-url "https://splunk.example.com" \
37+
--splunk-token "splunk-token" \
38+
--splunk-event "example-splunk-event"
39+
local EXIT_CODE=$?
40+
if [ $EXIT_CODE -eq 2 ]; then
41+
echo "ERROR: Flakeguard encountered an error while aggregating results"
42+
exit $EXIT_CODE
43+
fi
44+
45+
GITHUB_TOKEN="EXAMPLE_GITHUB_TOKEN" go run . generate-report \
46+
--aggregated-results-path ./example_results/all-test-results.json \
47+
--output-path ./example_results \
48+
--github-repository "smartcontractkit/chainlink-testing-framework" \
49+
--github-run-id "1" \
50+
--failed-tests-artifact-name "failed-test-results-with-logs.json" \
51+
--base-branch "exampleBaseBranch" \
52+
--current-branch "exampleCurrentBranch" \
53+
--current-commit-sha "abc" \
54+
--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
55+
--action-run-id "1" \
56+
--max-pass-ratio "1.0"
57+
local EXIT_CODE=$?
58+
if [ $EXIT_CODE -eq 2 ]; then
59+
echo "ERROR: Flakeguard encountered an error while generating report for the aggregated results"
60+
exit $EXIT_CODE
61+
fi
62+
}
63+
64+
# Run the commands
65+
rm -rf example_results
66+
mkdir -p example_results
67+
68+
run_flakeguard_example "$1" "$2"

0 commit comments

Comments
 (0)