Add reporting to example run

Author: kalverra

tools/flakeguard/Makefile

@@ -41,6 +41,18 @@ example:
 		--splunk-url "https://splunk.example.com" \
 		--splunk-token "splunk-token" \
 		--splunk-event "example-splunk-event"
+	GITHUB_TOKEN="EXAMPLE_GITHUB_TOKEN" go run . generate-report \
+		--aggregated-results-path ./example_results/all-test-results.json \
+		--output-path ./example_results \
+		--github-repository "smartcontractkit/chainlink-testing-framework" \
+		--github-run-id "1" \
+		--failed-tests-artifact-name "failed-test-results-with-logs.json" \
+		--base-branch "exampleBaseBranch" \
+		--current-branch "exampleCurrentBranch" \
+		--current-commit-sha "abc" \
+		--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
+		--action-run-id "1" \
+		--max-pass-ratio "1.0" 
 
 .PHONY: example_flaky_panic
 example_flaky_panic:
@@ -61,6 +73,18 @@ example_flaky_panic:
 		--splunk-url "https://splunk.example.com" \
 		--splunk-token "splunk-token" \
 		--splunk-event "example-splunk-event"
+	GITHUB_TOKEN="EXAMPLE_GITHUB_TOKEN" go run . generate-report \
+		--aggregated-results-path ./example_results/all-test-results.json \
+		--output-path ./example_results \
+		--github-repository "smartcontractkit/chainlink-testing-framework" \
+		--github-run-id "1" \
+		--failed-tests-artifact-name "failed-test-results-with-logs.json" \
+		--base-branch "exampleBaseBranch" \
+		--current-branch "exampleCurrentBranch" \
+		--current-commit-sha "abc" \
+		--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
+		--action-run-id "1" \
+		--max-pass-ratio "1.0" 
 
 .PHONY: example_timeout
 example_timeout:
@@ -81,3 +105,15 @@ example_timeout:
 		--splunk-url "https://splunk.example.com" \
 		--splunk-token "splunk-token" \
 		--splunk-event "example-splunk-event"
+	GITHUB_TOKEN="EXAMPLE_GITHUB_TOKEN" go run . generate-report \
+		--aggregated-results-path ./example_results/all-test-results.json \
+		--output-path ./example_results \
+		--github-repository "smartcontractkit/chainlink-testing-framework" \
+		--github-run-id "1" \
+		--failed-tests-artifact-name "failed-test-results-with-logs.json" \
+		--base-branch "exampleBaseBranch" \
+		--current-branch "exampleCurrentBranch" \
+		--current-commit-sha "abc" \
+		--repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
+		--action-run-id "1" \
+		--max-pass-ratio "1.0" 

tools/flakeguard/README.md

@@ -36,7 +36,8 @@ Both `find` and `run` commands support JSON output `--json`, making it easy to i
 You can find example usage and see outputs with:
 
 ```sh
-make example       # Run an example flow of running tests and aggregating results
-make example_panic # Run example flow with panicking tests
-ls example_results # See results of each run and aggregation
+make example             # Run an example flow of running tests, aggregating results, and reporting them to GitHub
+make example_flaky_panic # Run example flow with flaky and panicking tests
+make example_timeout     # Run example flow tests the timeout
+ls example_results       # See results of each run and aggregation
 ```

tools/flakeguard/cmd/generate_report.go

@@ -17,6 +17,8 @@ import (
 	"golang.org/x/oauth2"
 )
 
+const exampleGitHubToken = "EXAMPLE_GITHUB_TOKEN" //nolint:gosec
+
 var GenerateReportCmd = &cobra.Command{
 	Use:   "generate-report",
 	Short: "Generate test reports from aggregated results that can be posted to GitHub",
@@ -191,6 +193,9 @@ func init() {
 }
 
 func fetchArtifactLink(githubToken, githubRepo string, githubRunID int64, artifactName string) (string, error) {
+	if githubToken == exampleGitHubToken {
+		return "https://example-artifact-link.com", nil
+	}
 	ctx := context.Background()
 	ts := oauth2.StaticTokenSource(
 		&oauth2.Token{AccessToken: githubToken},

tools/flakeguard/reports/data.go

@@ -26,6 +26,61 @@ type TestReport struct {
 	MaxPassRatio float64 `json:"max_pass_ratio,omitempty"`
 }
 
+// GenerateSummaryData generates a summary of a report's test results
+func (testReport *TestReport) GenerateSummaryData() {
+	var runs, mostRuns, passes, fails, skips, panickedTests, racedTests, flakyTests, skippedTests int
+
+	for _, result := range testReport.Results {
+		runs += result.Runs
+		if result.Runs > mostRuns {
+			mostRuns = result.Runs
+		}
+		passes += result.Successes
+		fails += result.Failures
+		skips += result.Skips
+
+		// Count tests that were entirely skipped
+		if result.Runs == 0 && result.Skipped {
+			skippedTests++
+		}
+
+		if result.Panic {
+			panickedTests++
+			flakyTests++
+		} else if result.Race {
+			racedTests++
+			flakyTests++
+		} else if !result.Skipped && result.Runs > 0 && result.PassRatio < testReport.MaxPassRatio {
+			flakyTests++
+		}
+	}
+
+	// Calculate the raw pass ratio
+	passRatio := passRatio(passes, runs)
+
+	// Calculate the raw flake ratio
+	totalTests := len(testReport.Results)
+	flakeRatio := flakeRatio(flakyTests, totalTests)
+
+	passRatioStr := formatRatio(passRatio)
+	flakeTestRatioStr := formatRatio(flakeRatio)
+
+	testReport.SummaryData = &SummaryData{
+		UniqueTestsRun:   totalTests,
+		TestRunCount:     mostRuns,
+		PanickedTests:    panickedTests,
+		RacedTests:       racedTests,
+		FlakyTests:       flakyTests,
+		FlakyTestPercent: flakeTestRatioStr,
+
+		TotalRuns:   runs,
+		PassedRuns:  passes,
+		FailedRuns:  fails,
+		SkippedRuns: skips,
+		PassPercent: passRatioStr,
+	}
+}
+
 // TestResult contains the results and outputs of a single test
 type TestResult struct {
 	// ReportID is the ID of the report this test result belongs to
@@ -129,61 +184,6 @@ type SplunkTestResultEvent struct {
 
 // Data Processing Functions
 
-// GenerateSummaryData generates a summary of a report's test results
-func GenerateSummaryData(testReport *TestReport) {
-	var runs, mostRuns, passes, fails, skips, panickedTests, racedTests, flakyTests, skippedTests int
-
-	for _, result := range testReport.Results {
-		runs += result.Runs
-		if result.Runs > mostRuns {
-			mostRuns = result.Runs
-		}
-		passes += result.Successes
-		fails += result.Failures
-		skips += result.Skips
-
-		// Count tests that were entirely skipped
-		if result.Runs == 0 && result.Skipped {
-			skippedTests++
-		}
-
-		if result.Panic {
-			panickedTests++
-			flakyTests++
-		} else if result.Race {
-			racedTests++
-			flakyTests++
-		} else if !result.Skipped && result.Runs > 0 && result.PassRatio < testReport.MaxPassRatio {
-			flakyTests++
-		}
-	}
-
-	// Calculate the raw pass ratio
-	passRatio := passRatio(passes, runs)
-
-	// Calculate the raw flake ratio
-	totalTests := len(testReport.Results)
-	flakeRatio := flakeRatio(flakyTests, totalTests)
-
-	passRatioStr := formatRatio(passRatio)
-	flakeTestRatioStr := formatRatio(flakeRatio)
-
-	testReport.SummaryData = &SummaryData{
-		UniqueTestsRun:   totalTests,
-		TestRunCount:     mostRuns,
-		PanickedTests:    panickedTests,
-		RacedTests:       racedTests,
-		FlakyTests:       flakyTests,
-		FlakyTestPercent: flakeTestRatioStr,
-
-		TotalRuns:   runs,
-		PassedRuns:  passes,
-		FailedRuns:  fails,
-		SkippedRuns: skips,
-		PassPercent: passRatioStr,
-	}
-}
-
 func FilterResults(report *TestReport, maxPassRatio float64) *TestReport {
 	filteredResults := FilterTests(report.Results, func(tr TestResult) bool {
 		return !tr.Skipped && tr.PassRatio < maxPassRatio

tools/flakeguard/reports/data_test.go

@@ -205,7 +205,7 @@ func TestGenerateSummaryData(t *testing.T) {
 
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
-			GenerateSummaryData(tc.testReport)
+			tc.testReport.GenerateSummaryData()
 			assert.Equal(t, tc.expected, tc.testReport.SummaryData, "Summary data does not match expected")
 		})
 	}

tools/flakeguard/reports/io.go

@@ -491,10 +491,13 @@ func aggregate(reportChan <-chan *TestReport, errChan <-chan error, opts *aggreg
 
 	sortTestResults(aggregatedResults)
 	fullReport.Results = aggregatedResults
-	GenerateSummaryData(fullReport)
+	fullReport.GenerateSummaryData()
 
 	if sendToSplunk {
 		err = sendDataToSplunk(opts, *fullReport)
+		if err != nil {
+			return fullReport, fmt.Errorf("error sending data to Splunk: %w", err)
+		}
 	}
 	return fullReport, err
 }
@@ -630,13 +633,15 @@ func sendDataToSplunk(opts *aggregateOptions, report TestReport) error {
 			Int("successfully sent", successfulResultsSent).
 			Int("total results", len(results)).
 			Errs("errors", splunkErrs).
+			Str("report id", report.ID).
 			Str("duration", time.Since(start).String()).
 			Msg("Errors occurred while sending test results to Splunk")
 	} else {
 		log.Debug().
 			Int("successfully sent", successfulResultsSent).
 			Int("total results", len(results)).
 			Str("duration", time.Since(start).String()).
+			Str("report id", report.ID).
 			Msg("All results sent successfully to Splunk")
 	}
 

tools/flakeguard/reports/presentation.go

@@ -90,10 +90,10 @@ func GenerateGitHubSummaryMarkdown(w io.Writer, testReport *TestReport, maxPassR
 
 	if testReport.SummaryData.FlakyTests > 0 {
 		fmt.Fprintln(w, "## Found Flaky Tests :x:")
-		fmt.Fprintln(w)
 	} else {
 		fmt.Fprintln(w, "## No Flakes Found :white_check_mark:")
 	}
+	fmt.Fprintln(w)
 
 	RenderResults(w, testReport, true, false)
 

tools/flakeguard/reports/presentation_test.go

@@ -241,7 +241,7 @@ func TestRenderResults(t *testing.T) {
 	for _, tc := range testcases {
 		t.Run(tc.name, func(t *testing.T) {
 			// Generate the summary data
-			GenerateSummaryData(tc.testReport)
+			tc.testReport.GenerateSummaryData()
 
 			var buf bytes.Buffer
 			RenderResults(&buf, tc.testReport, false, false)

tools/flakeguard/runner/runner.go

@@ -84,7 +84,7 @@ func (r *Runner) RunTests() (*reports.TestReport, error) {
 		Results:       results,
 		MaxPassRatio:  r.MaxPassRatio,
 	}
-	reports.GenerateSummaryData(report)
+	report.GenerateSummaryData()
 	return report, nil
 }