[TT-1795] [TT-1834] Flakeguard for Splunk (#1504)

Author: kalverra

tools/flakeguard/Makefile

@@ -8,14 +8,34 @@ test:
 	set -euo pipefail
 	go list ./... | grep -v 'example_test_package' | xargs go test -json -cover -coverprofile unit-test-coverage.out -v 2>&1 | tee /tmp/gotest.log | gotestfmt
 
+.PHONY: test-package
+test-package:
+	go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
+	set -euo pipefail
+	go test -json -cover -coverprofile unit-test-coverage.out -v $(PKG) 2>&1 | tee /tmp/gotest.log | gotestfmt
+
+.PHONY: test-race
+test-race:
+	go list ./... | grep -v 'example_test_package' | xargs go test -count=1 -race
+
+.PHONY: bench
+bench:
+	go test -run=^$$ -bench=. -benchmem ./...
+
 .PHONY: example
 example:
 	rm -rf example_results
 	mkdir -p example_results
 	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic,Timeout --max-pass-ratio=1 --race=false --output-json=example_results/example_run_1.json
 	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic,Timeout --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
 	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic,Timeout --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
-	go run . aggregate-results --results-path ./example_results --output-results ./example_results/all_tests_example.json
+	go run . aggregate-results \
+      --results-path ./example_results \
+      --output-path ./example_results \
+      --repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
+      --base-sha "abc" \
+      --head-sha "xyz" \
+      --github-workflow-name "example"
 
 .PHONY: example_flaky_panic
 example_flaky_panic:
@@ -24,7 +44,13 @@ example_flaky_panic:
 	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_1.json
 	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
 	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
-	go run . aggregate-results --results-path ./example_results --output-results ./example_results/all_tests_example.json
+	go run . aggregate-results \
+	  --results-path ./example_results \
+	  --output-path ./example_results \
+	  --repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
+      --base-sha "abc" \
+      --head-sha "xyz" \
+      --github-workflow-name "example"
 
 .PHONY: example_timeout
 example_timeout:
@@ -33,4 +59,10 @@ example_timeout:
 	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --select-tests=TestTimeout --timeout=1s --max-pass-ratio=1 --race=false --output-json=example_results/example_run_1.json
 	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --select-tests=TestTimeout --timeout=1s --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
 	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --select-tests=TestTimeout --timeout=1s --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
-	go run . aggregate-results --results-path ./example_results --output-results ./example_results/all_tests_example.json
+	go run . aggregate-results \
+	  --results-path ./example_results \
+	  --output-path ./example_results \
+	  --repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
+      --base-sha "abc" \
+      --head-sha "xyz" \
+      --github-workflow-name "example"

tools/flakeguard/cmd/aggregate_results.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/briandowns/spinner"
+	"github.com/rs/zerolog/log"
 	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
 	"github.com/spf13/cobra"
 )
@@ -28,6 +29,11 @@ var AggregateResultsCmd = &cobra.Command{
 		headSHA, _ := cmd.Flags().GetString("head-sha")
 		baseSHA, _ := cmd.Flags().GetString("base-sha")
 		githubWorkflowName, _ := cmd.Flags().GetString("github-workflow-name")
+		githubWorkflowRunURL, _ := cmd.Flags().GetString("github-workflow-run-url")
+		reportID, _ := cmd.Flags().GetString("report-id")
+		splunkURL, _ := cmd.Flags().GetString("splunk-url")
+		splunkToken, _ := cmd.Flags().GetString("splunk-token")
+		splunkEvent, _ := cmd.Flags().GetString("splunk-event")
 
 		// Ensure the output directory exists
 		if err := fs.MkdirAll(outputDir, 0755); err != nil {
@@ -40,26 +46,32 @@ var AggregateResultsCmd = &cobra.Command{
 		s.Start()
 
 		// Load test reports from JSON files and aggregate them
-		aggregatedReport, err := reports.LoadAndAggregate(resultsPath)
+		aggregatedReport, err := reports.LoadAndAggregate(
+			resultsPath,
+			reports.WithReportID(reportID),
+			reports.WithSplunk(splunkURL, splunkToken, reports.SplunkEvent(splunkEvent)),
+		)
 		if err != nil {
 			s.Stop()
+			fmt.Println()
 			return fmt.Errorf("error loading test reports: %w", err)
 		}
 		s.Stop()
-		fmt.Println("Test reports aggregated successfully.")
+		fmt.Println()
 
 		// Add metadata to the aggregated report
 		aggregatedReport.HeadSHA = headSHA
 		aggregatedReport.BaseSHA = baseSHA
 		aggregatedReport.RepoURL = repoURL
 		aggregatedReport.GitHubWorkflowName = githubWorkflowName
+		aggregatedReport.GitHubWorkflowRunURL = githubWorkflowRunURL
 
 		if err != nil {
 			s.Stop()
 			return fmt.Errorf("error aggregating test reports: %w", err)
 		}
 		s.Stop()
-		fmt.Println("Test reports aggregated successfully.")
+		log.Debug().Msg("Successfully loaded and aggregated test reports")
 
 		// Start spinner for mapping test results to paths
 		s = spinner.New(spinner.CharSets[11], 100*time.Millisecond)
@@ -73,7 +85,7 @@ var AggregateResultsCmd = &cobra.Command{
 			return fmt.Errorf("error mapping test results to paths: %w", err)
 		}
 		s.Stop()
-		fmt.Println("Test results mapped to paths successfully.")
+		log.Debug().Msg("Successfully mapped paths to test results")
 
 		// Map test results to code owners if codeOwnersPath is provided
 		if codeOwnersPath != "" {
@@ -87,7 +99,7 @@ var AggregateResultsCmd = &cobra.Command{
 				return fmt.Errorf("error mapping test results to code owners: %w", err)
 			}
 			s.Stop()
-			fmt.Println("Test results mapped to code owners successfully.")
+			log.Debug().Msg("Successfully mapped code owners to test results")
 		}
 
 		failedTests := reports.FilterTests(aggregatedReport.Results, func(tr reports.TestResult) bool {
@@ -97,7 +109,7 @@ var AggregateResultsCmd = &cobra.Command{
 
 		// Check if there are any failed tests
 		if len(failedTests) > 0 {
-			fmt.Printf("Found %d failed test(s).\n", len(failedTests))
+			log.Info().Int("count", len(failedTests)).Msg("Found failed tests")
 
 			// Create a new report for failed tests with logs
 			failedReportWithLogs := &reports.TestReport{
@@ -117,7 +129,7 @@ var AggregateResultsCmd = &cobra.Command{
 			if err := reports.SaveReport(fs, failedTestsReportWithLogsPath, *failedReportWithLogs); err != nil {
 				return fmt.Errorf("error saving failed tests report with logs: %w", err)
 			}
-			fmt.Printf("Failed tests report with logs saved to %s\n", failedTestsReportWithLogsPath)
+			log.Debug().Str("path", failedTestsReportWithLogsPath).Msg("Failed tests report with logs saved")
 
 			// Remove logs from test results for the report without logs
 			for i := range failedReportWithLogs.Results {
@@ -131,9 +143,9 @@ var AggregateResultsCmd = &cobra.Command{
 			if err := reports.SaveReport(fs, failedTestsReportNoLogsPath, *failedReportWithLogs); err != nil {
 				return fmt.Errorf("error saving failed tests report without logs: %w", err)
 			}
-			fmt.Printf("Failed tests report without logs saved to %s\n", failedTestsReportNoLogsPath)
+			log.Debug().Str("path", failedTestsReportNoLogsPath).Msg("Failed tests report without logs saved")
 		} else {
-			fmt.Println("No failed tests found. Skipping generation of failed tests reports.")
+			log.Debug().Msg("No failed tests found. Skipping generation of failed tests reports")
 		}
 
 		// Remove logs from test results for the aggregated report
@@ -148,26 +160,26 @@ var AggregateResultsCmd = &cobra.Command{
 		if err := reports.SaveReport(fs, aggregatedReportPath, *aggregatedReport); err != nil {
 			return fmt.Errorf("error saving aggregated test report: %w", err)
 		}
-		fmt.Printf("Aggregated test report saved to %s\n", aggregatedReportPath)
+		log.Debug().Str("path", aggregatedReportPath).Msg("Aggregated test report saved")
 
 		// Generate all-tests-summary.json
+		var summaryFilePath string
 		if summaryFileName != "" {
 			s = spinner.New(spinner.CharSets[11], 100*time.Millisecond)
 			s.Suffix = " Generating summary json..."
 			s.Start()
 
-			summaryFilePath := filepath.Join(outputDir, summaryFileName)
+			summaryFilePath = filepath.Join(outputDir, summaryFileName)
 			err = generateAllTestsSummaryJSON(aggregatedReport, summaryFilePath, maxPassRatio)
 			if err != nil {
 				s.Stop()
 				return fmt.Errorf("error generating summary json: %w", err)
 			}
 			s.Stop()
-			fmt.Printf("Summary generated at %s\n", summaryFilePath)
+			log.Debug().Str("path", summaryFilePath).Msg("Summary generated")
 		}
 
-		fmt.Println("Aggregation complete.")
-
+		log.Info().Str("summary", summaryFilePath).Str("report", aggregatedReportPath).Msg("Aggregation complete")
 		return nil
 	},
 }
@@ -183,8 +195,15 @@ func init() {
 	AggregateResultsCmd.Flags().String("head-sha", "", "Head commit SHA for the test report")
 	AggregateResultsCmd.Flags().String("base-sha", "", "Base commit SHA for the test report")
 	AggregateResultsCmd.Flags().String("github-workflow-name", "", "GitHub workflow name for the test report")
-
-	AggregateResultsCmd.MarkFlagRequired("results-path")
+	AggregateResultsCmd.Flags().String("github-workflow-run-url", "", "GitHub workflow run URL for the test report")
+	AggregateResultsCmd.Flags().String("report-id", "", "Optional identifier for the test report. Will be generated if not provided")
+	AggregateResultsCmd.Flags().String("splunk-url", "", "Optional url to simultaneously send the test results to splunk")
+	AggregateResultsCmd.Flags().String("splunk-token", "", "Optional Splunk HEC token to simultaneously send the test results to splunk")
+	AggregateResultsCmd.Flags().String("splunk-event", "manual", "Optional Splunk event to send as the triggering event for the test results")
+
+	if err := AggregateResultsCmd.MarkFlagRequired("results-path"); err != nil {
+		log.Fatal().Err(err).Msg("Error marking flag as required")
+	}
 }
 
 // New function to generate all-tests-summary.json

tools/flakeguard/cmd/check_test_owners.go

@@ -1,10 +1,9 @@
 package cmd
 
 import (
-	"fmt"
-	"log"
 	"path/filepath"
 
+	"github.com/rs/zerolog/log"
 	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/codeowners"
 	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
 	"github.com/spf13/cobra"
@@ -25,21 +24,21 @@ var CheckTestOwnersCmd = &cobra.Command{
 		// Scan project for test functions
 		testFileMap, err := reports.ScanTestFiles(projectPath)
 		if err != nil {
-			log.Fatalf("Error scanning test files: %v", err)
+			log.Fatal().Err(err).Msg("Error scanning test files")
 		}
 
 		// Parse CODEOWNERS file
 		codeOwnerPatterns, err := codeowners.Parse(codeownersPath)
 		if err != nil {
-			log.Fatalf("Error parsing CODEOWNERS file: %v", err)
+			log.Fatal().Err(err).Msg("Error parsing CODEOWNERS file")
 		}
 
 		// Check for tests without code owners
 		testsWithoutOwners := make(map[string]string) // Map of test names to their relative paths
 		for testName, filePath := range testFileMap {
 			relFilePath, err := filepath.Rel(projectPath, filePath)
 			if err != nil {
-				fmt.Printf("Error getting relative path for test %s: %v\n", testName, err)
+				log.Error().Err(err).Msgf("Error getting relative path for test %s", testName)
 				continue
 			}
 			// Convert to Unix-style path for matching
@@ -60,19 +59,18 @@ var CheckTestOwnersCmd = &cobra.Command{
 		percentageWithoutOwners := float64(totalWithoutOwners) / float64(totalTests) * 100
 
 		// Report results
-		fmt.Printf("Total Test functions found: %d\n", totalTests)
-		fmt.Printf("Test functions with owners: %d (%.2f%%)\n", totalWithOwners, percentageWithOwners)
-		fmt.Printf("Test functions without owners: %d (%.2f%%)\n", totalWithoutOwners, percentageWithoutOwners)
+		log.Info().Msgf("Total Test functions found: %d", totalTests)
+		log.Info().Msgf("Test functions with owners: %d (%.2f%%)", totalWithOwners, percentageWithOwners)
+		log.Info().Msgf("Test functions without owners: %d (%.2f%%)", totalWithoutOwners, percentageWithoutOwners)
 
 		if printTestFunctions {
-
 			if totalWithoutOwners > 0 {
-				fmt.Println("\nTest functions without owners:")
+				log.Debug().Msg("Test functions without owners:")
 				for testName, relPath := range testsWithoutOwners {
-					fmt.Printf("- %s (%s)\n", testName, relPath)
+					log.Debug().Msgf("- %s (%s)", testName, relPath)
 				}
 			} else {
-				fmt.Println("All Test functions have code owners!")
+				log.Debug().Msg("All Test functions have code owners!")
 			}
 		}
 

tools/flakeguard/cmd/find.go

@@ -2,9 +2,8 @@ package cmd
 
 import (
 	"encoding/json"
-	"fmt"
-	"log"
 
+	"github.com/rs/zerolog/log"
 	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/git"
 	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/golang"
 	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/utils"
@@ -32,26 +31,26 @@ var FindTestsCmd = &cobra.Command{
 		if findByTestFilesDiff {
 			changedTestFiles, err := git.FindChangedFiles(projectPath, baseRef, "grep '_test\\.go$'")
 			if err != nil {
-				log.Fatalf("Error finding changed test files: %v", err)
+				log.Fatal().Err(err).Msg("Error finding changed test files")
 			}
 			if onlyShowChangedTestFiles {
 				outputResults(changedTestFiles, jsonOutput)
 				return
 			}
 			if verbose {
-				fmt.Println("Changed test files:", changedTestFiles)
+				log.Debug().Strs("files", changedTestFiles).Msg("Found changed test files")
 			}
 			changedTestPkgs, err = golang.GetFilePackages(changedTestFiles)
 			if err != nil {
-				log.Fatalf("Error getting package names for test files: %v", err)
+				log.Fatal().Err(err).Msg("Error getting package names for test files")
 			}
 		}
 
 		// Find all affected test packages
 		var affectedTestPkgs []string
 		if findByAffected {
 			if verbose {
-				fmt.Println("Finding affected packages...")
+				log.Debug().Msg("Finding affected packages...")
 			}
 			affectedTestPkgs = findAffectedPackages(baseRef, projectPath, excludes, levels)
 		}
@@ -63,7 +62,7 @@ var FindTestsCmd = &cobra.Command{
 		// Filter out packages that do not have tests
 		if filterEmptyTests {
 			if verbose {
-				fmt.Println("Filtering packages without tests...")
+				log.Debug().Msg("Filtering packages without tests...")
 			}
 			testPkgs = golang.FilterPackagesWithTests(testPkgs)
 		}
@@ -85,40 +84,40 @@ func init() {
 	FindTestsCmd.Flags().Bool("only-show-changed-test-files", false, "Only show the changed test files and exit")
 
 	if err := FindTestsCmd.MarkFlagRequired("base-ref"); err != nil {
-		fmt.Println("Error marking base-ref as required:", err)
+		log.Fatal().Err(err).Msg("Error marking base-ref as required")
 	}
 }
 
 func findAffectedPackages(baseRef, projectPath string, excludes []string, levels int) []string {
 	goList, err := golang.GoList()
 	if err != nil {
-		log.Fatalf("Error getting go list: %v\nStdErr: %s", err, goList.Stderr.String())
+		log.Fatal().Err(err).Msg("Error getting go list")
 	}
 	gitDiff, err := git.Diff(baseRef)
 	if err != nil {
-		log.Fatalf("Error getting the git diff: %v\nStdErr: %s", err, gitDiff.Stderr.String())
+		log.Fatal().Err(err).Msg("Error getting the git diff")
 	}
 	gitModDiff, err := git.ModDiff(baseRef, projectPath)
 	if err != nil {
-		log.Fatalf("Error getting the git mod diff: %v\nStdErr: %s", err, gitModDiff.Stderr.String())
+		log.Fatal().Err(err).Msg("Error getting the git mod diff")
 	}
 
 	packages, err := golang.ParsePackages(goList.Stdout)
 	if err != nil {
-		log.Fatalf("Error parsing packages: %v", err)
+		log.Fatal().Err(err).Msg("Error parsing packages")
 	}
 
 	fileMap := golang.GetGoFileMap(packages, true)
 
 	var changedPackages []string
 	changedPackages, err = git.GetChangedGoPackagesFromDiff(gitDiff.Stdout, projectPath, excludes, fileMap)
 	if err != nil {
-		log.Fatalf("Error getting changed packages: %v", err)
+		log.Fatal().Err(err).Msg("Error getting changed packages")
 	}
 
 	changedModPackages, err := git.GetGoModChangesFromDiff(gitModDiff.Stdout)
 	if err != nil {
-		log.Fatalf("Error getting go.mod changes: %v", err)
+		log.Fatal().Err(err).Msg("Error getting go.mod changes")
 	}
 
 	depMap := golang.GetGoDepMap(packages)
@@ -156,12 +155,12 @@ func outputResults(packages []string, jsonOutput bool) {
 	if jsonOutput {
 		data, err := json.Marshal(packages)
 		if err != nil {
-			log.Fatalf("Error marshaling test files to JSON: %v", err)
+			log.Fatal().Err(err).Msg("Error marshaling test packages to JSON")
 		}
-		fmt.Println(string(data))
+		log.Debug().Str("output", string(data)).Msg("JSON")
 	} else {
 		for _, pkg := range packages {
-			fmt.Print(pkg, " ")
+			log.Debug().Str("package", pkg).Msg("Test package")
 		}
 	}
 }