DX-90 Flakeguard: include codeowners test path and failed test log url in splunk (#1671)

Author: lukaszcl

tools/flakeguard/cmd/aggregate_results.go

@@ -31,9 +31,6 @@ var AggregateResultsCmd = &cobra.Command{
 		githubWorkflowName, _ := cmd.Flags().GetString("github-workflow-name")
 		githubWorkflowRunURL, _ := cmd.Flags().GetString("github-workflow-run-url")
 		reportID, _ := cmd.Flags().GetString("report-id")
-		splunkURL, _ := cmd.Flags().GetString("splunk-url")
-		splunkToken, _ := cmd.Flags().GetString("splunk-token")
-		splunkEvent, _ := cmd.Flags().GetString("splunk-event")
 
 		initialDirSize, err := getDirSize(resultsPath)
 		if err != nil {
@@ -56,8 +53,9 @@ var AggregateResultsCmd = &cobra.Command{
 		// Load test reports from JSON files and aggregate them
 		aggregatedReport, err := reports.LoadAndAggregate(
 			resultsPath,
+			reports.WithRepoPath(repoPath),
+			reports.WithCodeOwnersPath(codeOwnersPath),
 			reports.WithReportID(reportID),
-			reports.WithSplunk(splunkURL, splunkToken, splunkEvent),
 			reports.WithBranchName(branchName),
 			reports.WithBaseSha(baseSHA),
 			reports.WithHeadSha(headSHA),
@@ -75,36 +73,9 @@ var AggregateResultsCmd = &cobra.Command{
 
 		// Start spinner for mapping test results to paths
 		s = spinner.New(spinner.CharSets[11], 100*time.Millisecond)
-		s.Suffix = " Mapping test results to paths..."
+		s.Suffix = " Filter failed tests..."
 		s.Start()
 
-		// Map test results to test paths
-		err = reports.MapTestResultsToPaths(aggregatedReport, repoPath)
-		if err != nil {
-			s.Stop()
-			log.Error().Stack().Err(err).Msg("Error mapping test results to paths")
-			os.Exit(ErrorExitCode)
-		}
-		s.Stop()
-		log.Debug().Msg("Successfully mapped paths to test results")
-
-		// Map test results to code owners if codeOwnersPath is provided
-		if codeOwnersPath != "" {
-			s = spinner.New(spinner.CharSets[11], 100*time.Millisecond)
-			s.Suffix = " Mapping test results to code owners..."
-			s.Start()
-			fmt.Println()
-
-			err = reports.MapTestResultsToOwners(aggregatedReport, codeOwnersPath)
-			if err != nil {
-				s.Stop()
-				log.Error().Stack().Err(err).Msg("Error mapping test results to code owners")
-				os.Exit(ErrorExitCode)
-			}
-			s.Stop()
-			log.Debug().Msg("Successfully mapped code owners to test results")
-		}
-
 		failedTests := reports.FilterTests(aggregatedReport.Results, func(tr reports.TestResult) bool {
 			return !tr.Skipped && tr.PassRatio < maxPassRatio
 		})
@@ -190,9 +161,6 @@ func init() {
 	AggregateResultsCmd.Flags().String("github-workflow-name", "", "GitHub workflow name for the test report")
 	AggregateResultsCmd.Flags().String("github-workflow-run-url", "", "GitHub workflow run URL for the test report")
 	AggregateResultsCmd.Flags().String("report-id", "", "Optional identifier for the test report. Will be generated if not provided")
-	AggregateResultsCmd.Flags().String("splunk-url", "", "Optional url to simultaneously send the test results to splunk")
-	AggregateResultsCmd.Flags().String("splunk-token", "", "Optional Splunk HEC token to simultaneously send the test results to splunk")
-	AggregateResultsCmd.Flags().String("splunk-event", "manual", "Optional Splunk event to send as the triggering event for the test results")
 
 	if err := AggregateResultsCmd.MarkFlagRequired("results-path"); err != nil {
 		log.Fatal().Err(err).Msg("Error marking flag as required")

tools/flakeguard/cmd/generate_report.go

@@ -1,24 +1,18 @@
 package cmd
 
 import (
-	"context"
 	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
 	"time"
 
 	"github.com/briandowns/spinner"
-	"github.com/google/go-github/v67/github"
 	"github.com/rs/zerolog/log"
 	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
 	"github.com/spf13/cobra"
-	"golang.org/x/oauth2"
 )
 
-const exampleGitHubToken = "EXAMPLE_GITHUB_TOKEN" //nolint:gosec
-
 var GenerateReportCmd = &cobra.Command{
 	Use:   "generate-report",
 	Short: "Generate test reports from aggregated results that can be posted to GitHub",
@@ -30,23 +24,16 @@ var GenerateReportCmd = &cobra.Command{
 		outputDir, _ := cmd.Flags().GetString("output-path")
 		maxPassRatio, _ := cmd.Flags().GetFloat64("max-pass-ratio")
 		generatePRComment, _ := cmd.Flags().GetBool("generate-pr-comment")
-		githubRepo, _ := cmd.Flags().GetString("github-repository")
-		githubRunID, _ := cmd.Flags().GetInt64("github-run-id")
-		artifactName, _ := cmd.Flags().GetString("failed-tests-artifact-name")
+		failedLogsURL, _ := cmd.Flags().GetString("failed-logs-url")
+
+		failedLogsArtifactName := "failed-test-results-with-logs.json"
 
 		initialDirSize, err := getDirSize(outputDir)
 		if err != nil {
 			log.Error().Err(err).Str("path", outputDir).Msg("Error getting initial directory size")
 			// intentionally don't exit here, as we can still proceed with the generation
 		}
 
-		// Get the GitHub token from environment variable
-		githubToken := os.Getenv("GITHUB_TOKEN")
-		if githubToken == "" {
-			log.Error().Msg("GITHUB_TOKEN environment variable is not set")
-			os.Exit(ErrorExitCode)
-		}
-
 		// Load the aggregated report
 		s := spinner.New(spinner.CharSets[11], 100*time.Millisecond)
 		s.Suffix = " Loading aggregated test report..."
@@ -72,19 +59,6 @@ var GenerateReportCmd = &cobra.Command{
 		fmt.Println()
 		log.Info().Msg("Successfully loaded aggregated test report")
 
-		// Check if there are failed tests
-		hasFailedTests := aggregatedReport.SummaryData.FailedRuns > 0
-
-		var artifactLink string
-		if hasFailedTests && githubRepo != "" && githubRunID != 0 && artifactName != "" {
-			// Fetch artifact link from GitHub API
-			artifactLink, err = fetchArtifactLinkWithRetry(githubToken, githubRepo, githubRunID, artifactName, 5, 5*time.Second)
-			if err != nil {
-				log.Error().Err(err).Msg("Error fetching artifact link")
-				os.Exit(ErrorExitCode)
-			}
-		}
-
 		// Create output directory if it doesn't exist
 		if err := fs.MkdirAll(outputDir, 0755); err != nil {
 			log.Error().Err(err).Msg("Error creating output directory")
@@ -96,7 +70,7 @@ var GenerateReportCmd = &cobra.Command{
 		s.Suffix = " Generating GitHub summary markdown..."
 		s.Start()
 
-		err = generateGitHubSummaryMarkdown(aggregatedReport, filepath.Join(outputDir, "all-test"), artifactLink, artifactName)
+		err = generateGitHubSummaryMarkdown(aggregatedReport, filepath.Join(outputDir, "all-test"), failedLogsURL, failedLogsArtifactName)
 		if err != nil {
 			s.Stop()
 			fmt.Println()
@@ -147,8 +121,8 @@ var GenerateReportCmd = &cobra.Command{
 				currentCommitSHA,
 				repoURL,
 				actionRunID,
-				artifactName,
-				artifactLink,
+				failedLogsArtifactName,
+				failedLogsURL,
 				maxPassRatio,
 			)
 			if err != nil {
@@ -184,90 +158,14 @@ func init() {
 	GenerateReportCmd.Flags().String("action-run-id", "", "The GitHub Actions run ID (required if generate-pr-comment is set)")
 	GenerateReportCmd.Flags().String("github-repository", "", "The GitHub repository in the format owner/repo (required)")
 	GenerateReportCmd.Flags().Int64("github-run-id", 0, "The GitHub Actions run ID (required)")
-	GenerateReportCmd.Flags().String("failed-tests-artifact-name", "failed-test-results-with-logs.json", "The name of the failed tests artifact (default 'failed-test-results-with-logs.json')")
+	GenerateReportCmd.Flags().String("failed-logs-url", "", "Optional URL linking to additional logs for failed tests")
 
 	if err := GenerateReportCmd.MarkFlagRequired("aggregated-results-path"); err != nil {
 		log.Error().Err(err).Msg("Error marking flag as required")
 		os.Exit(ErrorExitCode)
 	}
 }
 
-func fetchArtifactLink(githubToken, githubRepo string, githubRunID int64, artifactName string) (string, error) {
-	if githubToken == exampleGitHubToken {
-		return "https://example-artifact-link.com", nil
-	}
-	ctx := context.Background()
-	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: githubToken})
-	tc := oauth2.NewClient(ctx, ts)
-	client := github.NewClient(tc)
-
-	// Split owner/repo
-	repoParts := strings.SplitN(githubRepo, "/", 2)
-	if len(repoParts) != 2 {
-		return "", fmt.Errorf("invalid format for --github-repository, expected owner/repo")
-	}
-	owner, repo := repoParts[0], repoParts[1]
-
-	opts := &github.ListOptions{PerPage: 100} // The max GitHub allows is 100 per page
-	var allArtifacts []*github.Artifact
-
-	// Paginate through all artifacts
-	for {
-		artifacts, resp, err := client.Actions.ListWorkflowRunArtifacts(ctx, owner, repo, githubRunID, opts)
-		if err != nil {
-			return "", fmt.Errorf("error listing artifacts: %w", err)
-		}
-
-		allArtifacts = append(allArtifacts, artifacts.Artifacts...)
-
-		if resp.NextPage == 0 {
-			// No more pages
-			break
-		}
-		// Move to the next page
-		opts.Page = resp.NextPage
-	}
-
-	// Find the artifact
-	for _, artifact := range allArtifacts {
-		if artifact.GetName() == artifactName {
-			artifactID := artifact.GetID()
-			artifactURL := fmt.Sprintf("https://github.com/%s/%s/actions/runs/%d/artifacts/%d",
-				owner, repo, githubRunID, artifactID)
-			return artifactURL, nil
-		}
-	}
-
-	return "", fmt.Errorf("artifact '%s' not found in the workflow run", artifactName)
-}
-
-func fetchArtifactLinkWithRetry(
-	githubToken, githubRepo string,
-	githubRunID int64, artifactName string,
-	maxRetries int, delay time.Duration,
-) (string, error) {
-	var lastErr error
-	for attempt := 1; attempt <= maxRetries; attempt++ {
-		link, err := fetchArtifactLink(githubToken, githubRepo, githubRunID, artifactName)
-		if err == nil {
-			// Found the artifact link successfully
-			return link, nil
-		}
-
-		// If this was our last attempt, return the error
-		lastErr = err
-		if attempt == maxRetries {
-			break
-		}
-
-		// Otherwise wait and retry
-		log.Printf("[Attempt %d/%d] Artifact not yet available. Retrying in %s...", attempt, maxRetries, delay)
-		time.Sleep(delay)
-	}
-
-	return "", fmt.Errorf("failed to fetch artifact link after %d retries: %w", maxRetries, lastErr)
-}
-
 func generateGitHubSummaryMarkdown(report *reports.TestReport, outputPath, artifactLink, artifactName string) error {
 	fs := reports.OSFileSystem{}
 	mdFileName := outputPath + "-summary.md"

tools/flakeguard/cmd/get_gh_artifact_link.go

@@ -0,0 +1,123 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/google/go-github/v67/github"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+	"golang.org/x/oauth2"
+)
+
+// GetGHArtifactLinkCmd fetches the artifact link from GitHub API.
+var GetGHArtifactLinkCmd = &cobra.Command{
+	Use:   "get-gh-artifact",
+	Short: "Get artifact link from GitHub API",
+	Run: func(cmd *cobra.Command, args []string) {
+		// Get flag values
+		githubRepo, _ := cmd.Flags().GetString("github-repository")
+		githubRunID, _ := cmd.Flags().GetInt64("github-run-id")
+		artifactName, _ := cmd.Flags().GetString("failed-tests-artifact-name")
+
+		// Get the GitHub token from environment variable
+		githubToken := os.Getenv("GITHUB_TOKEN")
+		if githubToken == "" {
+			log.Error().Msg("GITHUB_TOKEN environment variable is not set")
+			os.Exit(ErrorExitCode)
+		}
+
+		// Fetch artifact link from GitHub API with retry logic
+		artifactLink, err := fetchArtifactLinkWithRetry(githubToken, githubRepo, githubRunID, artifactName, 5, 5*time.Second)
+		if err != nil {
+			log.Error().Err(err).Msg("Error fetching artifact link")
+			os.Exit(ErrorExitCode)
+		}
+
+		fmt.Println(artifactLink)
+	},
+}
+
+func init() {
+	GetGHArtifactLinkCmd.Flags().String("github-repository", "", "The GitHub repository in the format owner/repo (required)")
+	GetGHArtifactLinkCmd.Flags().Int64("github-run-id", 0, "The GitHub Actions run ID (required)")
+	GetGHArtifactLinkCmd.Flags().String("failed-tests-artifact-name", "failed-test-results-with-logs.json", "The name of the failed tests artifact (default 'failed-test-results-with-logs.json')")
+
+	if err := GetGHArtifactLinkCmd.MarkFlagRequired("github-repository"); err != nil {
+		log.Error().Err(err).Msg("Error marking github-repository flag as required")
+		os.Exit(ErrorExitCode)
+	}
+	if err := GetGHArtifactLinkCmd.MarkFlagRequired("github-run-id"); err != nil {
+		log.Error().Err(err).Msg("Error marking github-run-id flag as required")
+		os.Exit(ErrorExitCode)
+	}
+}
+
+// fetchArtifactLink uses the GitHub API to retrieve the artifact link.
+func fetchArtifactLink(githubToken, githubRepo string, githubRunID int64, artifactName string) (string, error) {
+	ctx := context.Background()
+	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: githubToken})
+	tc := oauth2.NewClient(ctx, ts)
+	client := github.NewClient(tc)
+
+	// Split owner and repo from the provided repository string.
+	repoParts := strings.SplitN(githubRepo, "/", 2)
+	if len(repoParts) != 2 {
+		return "", fmt.Errorf("invalid format for --github-repository, expected owner/repo")
+	}
+	owner, repo := repoParts[0], repoParts[1]
+
+	opts := &github.ListOptions{PerPage: 100} // maximum per page allowed by GitHub
+	var allArtifacts []*github.Artifact
+
+	// Paginate through all artifacts.
+	for {
+		artifacts, resp, err := client.Actions.ListWorkflowRunArtifacts(ctx, owner, repo, githubRunID, opts)
+		if err != nil {
+			return "", fmt.Errorf("error listing artifacts: %w", err)
+		}
+
+		allArtifacts = append(allArtifacts, artifacts.Artifacts...)
+
+		if resp.NextPage == 0 {
+			break
+		}
+		opts.Page = resp.NextPage
+	}
+
+	// Search for the artifact by name.
+	for _, artifact := range allArtifacts {
+		if artifact.GetName() == artifactName {
+			artifactID := artifact.GetID()
+			artifactURL := fmt.Sprintf("https://github.com/%s/%s/actions/runs/%d/artifacts/%d",
+				owner, repo, githubRunID, artifactID)
+			return artifactURL, nil
+		}
+	}
+
+	return "", fmt.Errorf("artifact '%s' not found in the workflow run", artifactName)
+}
+
+// fetchArtifactLinkWithRetry attempts to fetch the artifact link with retry logic.
+func fetchArtifactLinkWithRetry(githubToken, githubRepo string, githubRunID int64, artifactName string, maxRetries int, delay time.Duration) (string, error) {
+	var lastErr error
+	for attempt := 1; attempt <= maxRetries; attempt++ {
+		link, err := fetchArtifactLink(githubToken, githubRepo, githubRunID, artifactName)
+		if err == nil {
+			return link, nil
+		}
+
+		lastErr = err
+		if attempt == maxRetries {
+			break
+		}
+
+		log.Printf("[Attempt %d/%d] Artifact not yet available. Retrying in %s...", attempt, maxRetries, delay)
+		time.Sleep(delay)
+	}
+
+	return "", fmt.Errorf("failed to fetch artifact link after %d retries: %w", maxRetries, lastErr)
+}