Skip to content

Commit 88d5d2a

Browse files
kalverrakalverra
committed
Cleanup makefile
1 parent 3c59074 commit 88d5d2a

File tree

5 files changed

+43
-88
lines changed

5 files changed

+43
-88
lines changed

‎tools/flakeguard/Makefile‎

Lines changed: 1 addition & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -24,88 +24,8 @@ bench:
2424

2525
.PHONY: example
2626
example:
27-
./run_example.sh 3 TestPanic
28-
go run . aggregate-results \
29-
--results-path ./example_results \
30-
--output-path ./example_results \
31-
--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
32-
--branch-name "example-branch" \
33-
--base-sha "abc" \
34-
--head-sha "xyz" \
35-
--github-workflow-name "ExampleWorkflowName" \
36-
--github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
37-
--splunk-url "https://splunk.example.com" \
38-
--splunk-token "splunk-token" \
39-
--splunk-event "example-splunk-event"
40-
GITHUB_TOKEN="EXAMPLE_GITHUB_TOKEN" go run . generate-report \
41-
--aggregated-results-path ./example_results/all-test-results.json \
42-
--output-path ./example_results \
43-
--github-repository "smartcontractkit/chainlink-testing-framework" \
44-
--github-run-id "1" \
45-
--failed-tests-artifact-name "failed-test-results-with-logs.json" \
46-
--base-branch "exampleBaseBranch" \
47-
--current-branch "exampleCurrentBranch" \
48-
--current-commit-sha "abc" \
49-
--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
50-
--action-run-id "1" \
51-
--max-pass-ratio "1.0"
27+
./run_example.sh 3 Panic,Timeout
5228

5329
.PHONY: example_flaky_panic
5430
example_flaky_panic:
5531
./run_example.sh 3 TestPanic
56-
go run . aggregate-results \
57-
--results-path ./example_results \
58-
--output-path ./example_results \
59-
--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
60-
--branch-name "example-branch" \
61-
--base-sha "abc" \
62-
--head-sha "xyz" \
63-
--github-workflow-name "ExampleWorkflowName" \
64-
--github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
65-
--splunk-url "https://splunk.example.com" \
66-
--splunk-token "splunk-token" \
67-
--splunk-event "example-splunk-event"
68-
GITHUB_TOKEN="EXAMPLE_GITHUB_TOKEN" go run . generate-report \
69-
--aggregated-results-path ./example_results/all-test-results.json \
70-
--output-path ./example_results \
71-
--github-repository "smartcontractkit/chainlink-testing-framework" \
72-
--github-run-id "1" \
73-
--failed-tests-artifact-name "failed-test-results-with-logs.json" \
74-
--base-branch "exampleBaseBranch" \
75-
--current-branch "exampleCurrentBranch" \
76-
--current-commit-sha "abc" \
77-
--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
78-
--action-run-id "1" \
79-
--max-pass-ratio "1.0"
80-
81-
.PHONY: example_timeout
82-
example_timeout:
83-
rm -rf example_results
84-
mkdir -p example_results
85-
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --select-tests=TestTimeout --timeout=1s --max-pass-ratio=1 --race=false --output-json=example_results/example_run_1.json
86-
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --select-tests=TestTimeout --timeout=1s --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
87-
- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --select-tests=TestTimeout --timeout=1s --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
88-
go run . aggregate-results \
89-
--results-path ./example_results \
90-
--output-path ./example_results \
91-
--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
92-
--branch-name "example-branch" \
93-
--base-sha "abc" \
94-
--head-sha "xyz" \
95-
--github-workflow-name "ExampleWorkflowName" \
96-
--github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
97-
--splunk-url "https://splunk.example.com" \
98-
--splunk-token "splunk-token" \
99-
--splunk-event "example-splunk-event"
100-
GITHUB_TOKEN="EXAMPLE_GITHUB_TOKEN" go run . generate-report \
101-
--aggregated-results-path ./example_results/all-test-results.json \
102-
--output-path ./example_results \
103-
--github-repository "smartcontractkit/chainlink-testing-framework" \
104-
--github-run-id "1" \
105-
--failed-tests-artifact-name "failed-test-results-with-logs.json" \
106-
--base-branch "exampleBaseBranch" \
107-
--current-branch "exampleCurrentBranch" \
108-
--current-commit-sha "abc" \
109-
--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
110-
--action-run-id "1" \
111-
--max-pass-ratio "1.0"

‎tools/flakeguard/README.md‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,5 @@ You can find example usage and see outputs with:
3838
```sh
3939
make example # Run an example flow of running tests, aggregating results, and reporting them to GitHub
4040
make example_flaky_panic # Run example flow with flaky and panicking tests
41-
make example_timeout # Run example flow tests the timeout
4241
ls example_results # See results of each run and aggregation
4342
```

‎tools/flakeguard/reports/data.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ type SummaryData struct {
121121
RacedTests int `json:"raced_tests"`
122122
// FlakyTests tracks how many tests are considered flaky
123123
FlakyTests int `json:"flaky_tests"`
124-
// FlakyTestPercent is the percentage of tests that are considered flaky
124+
// FlakyTestPercent is the human-readable percentage of tests that are considered flaky
125125
FlakyTestPercent string `json:"flaky_test_percent"`
126126

127127
// Individual test run counts
@@ -134,7 +134,7 @@ type SummaryData struct {
134134
FailedRuns int `json:"failed_runs"`
135135
// SkippedRuns tracks how many test runs were skipped
136136
SkippedRuns int `json:"skipped_runs"`
137-
// PassPercent is the percentage of test runs that passed
137+
// PassPercent is the human-readable percentage of test runs that passed
138138
PassPercent string `json:"pass_percent"`
139139
}
140140

‎tools/flakeguard/reports/presentation.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -199,12 +199,12 @@ func renderSummaryTable(w io.Writer, summary *SummaryData, markdown bool, collap
199199
{"Panicked Tests", fmt.Sprintf("%d", summary.PanickedTests)},
200200
{"Raced Tests", fmt.Sprintf("%d", summary.RacedTests)},
201201
{"Flaky Tests", fmt.Sprintf("%d", summary.FlakyTests)},
202-
{"Flaky Test Ratio", summary.FlakyTestPercent},
202+
{"Flaky Test Percent", summary.FlakyTestPercent},
203203
{"Total Test Runs", fmt.Sprintf("%d", summary.TotalRuns)},
204204
{"Passes", fmt.Sprintf("%d", summary.PassedRuns)},
205205
{"Failures", fmt.Sprintf("%d", summary.FailedRuns)},
206206
{"Skips", fmt.Sprintf("%d", summary.SkippedRuns)},
207-
{"Pass Ratio", summary.PassPercent},
207+
{"Pass Percent", summary.PassPercent},
208208
}
209209
if markdown {
210210
for i, row := range summaryData {

‎tools/flakeguard/run_example.sh‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22

3-
run_flakeguard() {
3+
run_flakeguard_example() {
44
local run_count=$1
55
local skip_tests=$2
66

@@ -22,10 +22,46 @@ run_flakeguard() {
2222
exit $EXIT_CODE
2323
fi
2424
done
25+
26+
go run . aggregate-results \
27+
--results-path ./example_results \
28+
--output-path ./example_results \
29+
--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
30+
--branch-name "example-branch" \
31+
--base-sha "abc" \
32+
--head-sha "xyz" \
33+
--github-workflow-name "ExampleWorkflowName" \
34+
--github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
35+
--splunk-url "https://splunk.example.com" \
36+
--splunk-token "splunk-token" \
37+
--splunk-event "example-splunk-event"
38+
local EXIT_CODE=$?
39+
if [ $EXIT_CODE -eq 2 ]; then
40+
echo "ERROR: Flakeguard encountered an error while aggregating results"
41+
exit $EXIT_CODE
42+
fi
43+
44+
GITHUB_TOKEN="EXAMPLE_GITHUB_TOKEN" go run . generate-report \
45+
--aggregated-results-path ./example_results/all-test-results.json \
46+
--output-path ./example_results \
47+
--github-repository "smartcontractkit/chainlink-testing-framework" \
48+
--github-run-id "1" \
49+
--failed-tests-artifact-name "failed-test-results-with-logs.json" \
50+
--base-branch "exampleBaseBranch" \
51+
--current-branch "exampleCurrentBranch" \
52+
--current-commit-sha "abc" \
53+
--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
54+
--action-run-id "1" \
55+
--max-pass-ratio "1.0"
56+
local EXIT_CODE=$?
57+
if [ $EXIT_CODE -eq 2 ]; then
58+
echo "ERROR: Flakeguard encountered an error while generating report for the aggregated results"
59+
exit $EXIT_CODE
60+
fi
2561
}
2662

2763
# Run the commands
2864
rm -rf example_results
2965
mkdir -p example_results
3066

31-
run_flakeguard "$1" "$2"
67+
run_flakeguard_example "$1" "$2"

0 commit comments

Comments
 (0)