Fixes Splunk Aggregation Data (#1615)

Author: kalverra

tools/flakeguard/Makefile

@@ -35,7 +35,11 @@ example:
       --repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
       --base-sha "abc" \
       --head-sha "xyz" \
-      --github-workflow-name "example"
+      --github-workflow-name "ExampleWorkflowName" \
+	  --github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
+	  --splunk-url "https://splunk.example.com" \
+	  --splunk-token "splunk-token" \
+	  --splunk-event "example-splunk-event"
 
 .PHONY: example_flaky_panic
 example_flaky_panic:
@@ -50,7 +54,11 @@ example_flaky_panic:
 	  --repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
       --base-sha "abc" \
       --head-sha "xyz" \
-      --github-workflow-name "example"
+      --github-workflow-name "ExampleWorkflowName" \
+	  --github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
+	  --splunk-url "https://splunk.example.com" \
+	  --splunk-token "splunk-token" \
+	  --splunk-event "example-splunk-event"
 
 .PHONY: example_timeout
 example_timeout:
@@ -65,4 +73,8 @@ example_timeout:
 	  --repo-url "https://github.com/smartcontractkit/chainlink-testing-framework" \
       --base-sha "abc" \
       --head-sha "xyz" \
-      --github-workflow-name "example"
+      --github-workflow-name "ExampleWorkflowName" \
+	  --github-workflow-run-url "https://github.com/example/repo/actions/runs/1" \
+	  --splunk-url "https://splunk.example.com" \
+	  --splunk-token "splunk-token" \
+	  --splunk-event "example-splunk-event"

tools/flakeguard/cmd/aggregate_results.go

@@ -49,7 +49,12 @@ var AggregateResultsCmd = &cobra.Command{
 		aggregatedReport, err := reports.LoadAndAggregate(
 			resultsPath,
 			reports.WithReportID(reportID),
-			reports.WithSplunk(splunkURL, splunkToken, reports.SplunkEvent(splunkEvent)),
+			reports.WithSplunk(splunkURL, splunkToken, splunkEvent),
+			reports.WithBaseSha(baseSHA),
+			reports.WithHeadSha(headSHA),
+			reports.WithRepoURL(repoURL),
+			reports.WithGitHubWorkflowName(githubWorkflowName),
+			reports.WithGitHubWorkflowRunURL(githubWorkflowRunURL),
 		)
 		if err != nil {
 			s.Stop()
@@ -59,13 +64,6 @@ var AggregateResultsCmd = &cobra.Command{
 		s.Stop()
 		fmt.Println()
 
-		// Add metadata to the aggregated report
-		aggregatedReport.HeadSHA = headSHA
-		aggregatedReport.BaseSHA = baseSHA
-		aggregatedReport.RepoURL = repoURL
-		aggregatedReport.GitHubWorkflowName = githubWorkflowName
-		aggregatedReport.GitHubWorkflowRunURL = githubWorkflowRunURL
-
 		if err != nil {
 			s.Stop()
 			return fmt.Errorf("error aggregating test reports: %w", err)

tools/flakeguard/cmd/run.go

@@ -97,7 +97,7 @@ var RunTestsCmd = &cobra.Command{
 		if len(flakyTests) > 0 {
 			log.Info().Int("count", len(flakyTests)).Str("pass ratio threshold", fmt.Sprintf("%.2f%%", maxPassRatio*100)).Msg("Found flaky tests")
 			fmt.Printf("\nFlakeguard Summary\n")
-			reports.RenderResults(os.Stdout, flakyTests, maxPassRatio, false)
+			reports.RenderResults(os.Stdout, flakyTests, maxPassRatio, false, false)
 			// Exit with error code if there are flaky tests
 			os.Exit(1)
 		}

tools/flakeguard/reports/data.go

@@ -64,19 +64,11 @@ type SummaryData struct {
 	MaxPassRatio   float64 `json:"max_pass_ratio"`
 }
 
-// SplunkEvent represents a customized splunk event string that helps us distinguish what
-// triggered the test to run. This is a custom field, different from the Splunk event field.
-type SplunkEvent string
-
 // SplunkType represents what type of data is being sent to Splunk, e.g. a report or a result.
 // This is a custom field to help us distinguish what kind of data we're sending.
 type SplunkType string
 
 const (
-	Manual      SplunkEvent = "manual"
-	Scheduled   SplunkEvent = "scheduled"
-	PullRequest SplunkEvent = "pull_request"
-
 	Report SplunkType = "report"
 	Result SplunkType = "result"
 
@@ -95,9 +87,9 @@ type SplunkTestReport struct {
 
 // SplunkTestReportEvent contains the actual meat of the Splunk test report event
 type SplunkTestReportEvent struct {
-	Event SplunkEvent `json:"event"`
-	Type  SplunkType  `json:"type"`
-	Data  TestReport  `json:"data"`
+	Event string     `json:"event"`
+	Type  SplunkType `json:"type"`
+	Data  TestReport `json:"data"`
 	// Incomplete indicates that there were issues uploading test results and the report is incomplete
 	Incomplete bool `json:"incomplete"`
 }
@@ -111,9 +103,9 @@ type SplunkTestResult struct {
 
 // SplunkTestResultEvent contains the actual meat of the Splunk test result event
 type SplunkTestResultEvent struct {
-	Event SplunkEvent `json:"event"`
-	Type  SplunkType  `json:"type"`
-	Data  TestResult  `json:"data"`
+	Event string     `json:"event"`
+	Type  SplunkType `json:"type"`
+	Data  TestResult `json:"data"`
 }
 
 // Data Processing Functions

tools/flakeguard/reports/io.go

@@ -39,10 +39,15 @@ func (OSFileSystem) WriteFile(filename string, data []byte, perm os.FileMode) er
 }
 
 type aggregateOptions struct {
-	reportID    string
-	splunkURL   string
-	splunkToken string
-	splunkEvent SplunkEvent
+	reportID             string
+	splunkURL            string
+	splunkToken          string
+	splunkEvent          string
+	baseSha              string
+	headSha              string
+	repoURL              string
+	gitHubWorkflowName   string
+	gitHubWorkflowRunURL string
 }
 
 // AggregateOption is a functional option for configuring the aggregation process.
@@ -56,14 +61,49 @@ func WithReportID(reportID string) AggregateOption {
 }
 
 // WithSplunk also sends the aggregation to a Splunk instance as events.
-func WithSplunk(url, token string, event SplunkEvent) AggregateOption {
+func WithSplunk(url, token string, event string) AggregateOption {
 	return func(opts *aggregateOptions) {
 		opts.splunkURL = url
 		opts.splunkToken = token
 		opts.splunkEvent = event
 	}
 }
 
+// WithHeadSha sets the head SHA for the aggregated report.
+func WithHeadSha(headSha string) AggregateOption {
+	return func(opts *aggregateOptions) {
+		opts.headSha = headSha
+	}
+}
+
+// WithBaseSha sets the base SHA for the aggregated report.
+func WithBaseSha(baseSha string) AggregateOption {
+	return func(opts *aggregateOptions) {
+		opts.baseSha = baseSha
+	}
+}
+
+// WithRepoURL sets the repository URL for the aggregated report.
+func WithRepoURL(repoURL string) AggregateOption {
+	return func(opts *aggregateOptions) {
+		opts.repoURL = repoURL
+	}
+}
+
+// WithGitHubWorkflowName sets the GitHub workflow name for the aggregated report.
+func WithGitHubWorkflowName(githubWorkflowName string) AggregateOption {
+	return func(opts *aggregateOptions) {
+		opts.gitHubWorkflowName = githubWorkflowName
+	}
+}
+
+// WithGitHubWorkflowRunURL sets the GitHub workflow run URL for the aggregated report.
+func WithGitHubWorkflowRunURL(githubWorkflowRunURL string) AggregateOption {
+	return func(opts *aggregateOptions) {
+		opts.gitHubWorkflowRunURL = githubWorkflowRunURL
+	}
+}
+
 // LoadAndAggregate reads all JSON files in a directory and aggregates the results into a single TestReport.
 func LoadAndAggregate(resultsPath string, options ...AggregateOption) (*TestReport, error) {
 	if _, err := os.Stat(resultsPath); os.IsNotExist(err) {
@@ -115,6 +155,7 @@ func LoadAndAggregate(resultsPath string, options ...AggregateOption) (*TestRepo
 	if err != nil {
 		return nil, fmt.Errorf("error aggregating reports: %w", err)
 	}
+
 	return aggregatedReport, nil
 }
 
@@ -318,14 +359,20 @@ func SaveReport(fs FileSystem, filePath string, report TestReport) error {
 // aggregate aggregates multiple TestReport objects into a single TestReport as they are received
 func aggregate(reportChan <-chan *TestReport, errChan <-chan error, opts *aggregateOptions) (*TestReport, error) {
 	var (
+		fullReport = &TestReport{
+			ID:                   opts.reportID,
+			BaseSHA:              opts.baseSha,
+			HeadSHA:              opts.headSha,
+			RepoURL:              opts.repoURL,
+			GitHubWorkflowName:   opts.gitHubWorkflowName,
+			GitHubWorkflowRunURL: opts.gitHubWorkflowRunURL,
+		}
 		testMap       = make(map[string]TestResult)
-		fullReport    = &TestReport{}
 		excludedTests = map[string]struct{}{}
 		selectedTests = map[string]struct{}{}
 		sendToSplunk  = opts.splunkURL != ""
 	)
 
-	fullReport.ID = opts.reportID
 	for report := range reportChan {
 		if fullReport.GoProject == "" {
 			fullReport.GoProject = report.GoProject
@@ -388,23 +435,25 @@ func aggregate(reportChan <-chan *TestReport, errChan <-chan error, opts *aggreg
 // sendDataToSplunk sends a truncated TestReport and each individual TestResults to Splunk as events
 func sendDataToSplunk(opts *aggregateOptions, report TestReport, results ...TestResult) error {
 	start := time.Now()
+	// Dry-run mode for example runs
+	isExampleRun := strings.Contains(opts.splunkURL, "splunk.example.com")
+
 	client := resty.New().
 		SetBaseURL(opts.splunkURL).
 		SetAuthScheme("Splunk").
 		SetAuthToken(opts.splunkToken).
 		SetHeader("Content-Type", "application/json").
 		SetLogger(ZerologRestyLogger{})
-	client.AddRetryAfterErrorCondition().SetRetryCount(5).SetTimeout(time.Second * 10)
 
 	log.Debug().Str("report id", report.ID).Int("results", len(results)).Msg("Sending aggregated data to Splunk")
 
-	// Send results
 	var (
 		splunkErrs            = []error{}
 		resultsBatchSize      = 10
 		resultsBatch          = []SplunkTestResult{}
 		successfulResultsSent = 0
 	)
+
 	for resultCount, result := range results {
 		resultsBatch = append(resultsBatch, SplunkTestResult{
 			Event: SplunkTestResultEvent{
@@ -415,53 +464,70 @@ func sendDataToSplunk(opts *aggregateOptions, report TestReport, results ...Test
 			SourceType: SplunkSourceType,
 			Index:      SplunkIndex,
 		})
-		// Send results in batches so Splunk doesn't get mad
+
 		if len(resultsBatch) >= resultsBatchSize || resultCount == len(results)-1 {
 			batchData, testNames, err := batchSplunkResults(resultsBatch)
 			if err != nil {
 				return fmt.Errorf("error batching results: %w", err)
 			}
 
-			resp, err := client.R().
-				SetBody(batchData.String()).
-				Post("")
-			if err != nil {
-				splunkErrs = append(splunkErrs,
-					fmt.Errorf("error sending flakeguard results for [%s] to Splunk: %w", strings.Join(testNames, ", "), err),
-				)
-			}
-			if resp.IsError() {
-				splunkErrs = append(splunkErrs,
-					fmt.Errorf("error sending flakeguard result for [%s] to Splunk: %s", strings.Join(testNames, ", "), resp.String()),
-				)
-			}
-			if err == nil && !resp.IsError() {
-				successfulResultsSent += len(resultsBatch)
+			if isExampleRun {
+				log.Debug().Strs("tests", testNames).Msg("Example Run. Would send the below results to Splunk")
+				for _, result := range resultsBatch {
+					jsonResult, err := json.Marshal(result)
+					if err != nil {
+						return fmt.Errorf("error marshaling result for '%s': %w", result.Event.Data.TestName, err)
+					}
+					fmt.Println(string(jsonResult))
+				}
+			} else {
+				resp, err := client.R().SetBody(batchData.String()).Post("")
+				if err != nil {
+					splunkErrs = append(splunkErrs,
+						fmt.Errorf("error sending results for [%s] to Splunk: %w", strings.Join(testNames, ", "), err),
+					)
+				}
+				if resp.IsError() {
+					splunkErrs = append(splunkErrs,
+						fmt.Errorf("error sending result for [%s] to Splunk: %s", strings.Join(testNames, ", "), resp.String()),
+					)
+				}
+				if err == nil && !resp.IsError() {
+					successfulResultsSent += len(resultsBatch)
+				}
 			}
 			resultsBatch = []SplunkTestResult{}
 		}
 	}
 
-	// Check if errors occurred while uploading results and send report with incomplete flag
-	resp, err := client.R().
-		SetBody(SplunkTestReport{
-			Event: SplunkTestReportEvent{
-				Event:      opts.splunkEvent,
-				Type:       Report,
-				Data:       report,
-				Incomplete: len(splunkErrs) > 0,
-			},
-			SourceType: SplunkSourceType,
-			Index:      SplunkIndex,
-		}).
-		Post("")
-
-	if err != nil {
-		splunkErrs = append(splunkErrs, fmt.Errorf("error sending flakeguard report '%s' to Splunk: %w", report.ID, err))
+	reportData := SplunkTestReport{
+		Event: SplunkTestReportEvent{
+			Event:      opts.splunkEvent,
+			Type:       Report,
+			Data:       report,
+			Incomplete: len(splunkErrs) > 0,
+		},
+		SourceType: SplunkSourceType,
+		Index:      SplunkIndex,
 	}
-	if resp.IsError() {
-		splunkErrs = append(splunkErrs, fmt.Errorf("error sending flakeguard report '%s' to Splunk: %s", report.ID, resp.String()))
+
+	if isExampleRun {
+		log.Info().Msg("Example Run. Would send the below report to Splunk")
+		jsonReport, err := json.Marshal(reportData)
+		if err != nil {
+			return fmt.Errorf("error marshaling report: %w", err)
+		}
+		fmt.Println(string(jsonReport))
+	} else {
+		resp, err := client.R().SetBody(reportData).Post("")
+		if err != nil {
+			splunkErrs = append(splunkErrs, fmt.Errorf("error sending report '%s' to Splunk: %w", report.ID, err))
+		}
+		if resp.IsError() {
+			splunkErrs = append(splunkErrs, fmt.Errorf("error sending report '%s' to Splunk: %s", report.ID, resp.String()))
+		}
 	}
+
 	if len(splunkErrs) > 0 {
 		log.Error().
 			Int("successfully sent", successfulResultsSent).
@@ -476,6 +542,7 @@ func sendDataToSplunk(opts *aggregateOptions, report TestReport, results ...Test
 			Str("duration", time.Since(start).String()).
 			Msg("All results sent successfully")
 	}
+
 	return errors.Join(splunkErrs...)
 }
 

tools/flakeguard/reports/io_test.go

@@ -14,11 +14,11 @@ import (
 )
 
 const (
-	splunkToken              = "test-token"
-	splunkEvent  SplunkEvent = "test"
-	reportID                 = "123"
-	testRunCount             = 15
-	uniqueTests              = 18
+	splunkToken  = "test-token"
+	splunkEvent  = "test"
+	reportID     = "123"
+	testRunCount = 15
+	uniqueTests  = 18
 )
 
 func TestAggregateResultFilesSplunk(t *testing.T) {