Add PR comment to flakeguard report

lukaszcl · lukaszcl · commit d3a8549a0e5d · 2024-12-06T13:10:52.000+01:00
diff --git a/tools/flakeguard/cmd/report.go b/tools/flakeguard/cmd/report.go
@@ -20,13 +20,10 @@ var ReportCmd = &cobra.Command{
 		// Get flag values
 		reportResultsPath, _ := cmd.Flags().GetString("results-path")
 		reportOutputPath, _ := cmd.Flags().GetString("output-path")
-		reportFormats, _ := cmd.Flags().GetString("format")
 		reportMaxPassRatio, _ := cmd.Flags().GetFloat64("max-pass-ratio")
 		reportCodeOwnersPath, _ := cmd.Flags().GetString("codeowners-path")
 		reportRepoPath, _ := cmd.Flags().GetString("repo-path")
-
-		// Split the formats into a slice
-		formats := strings.Split(reportFormats, ",")
+		generatePRComment, _ := cmd.Flags().GetBool("generate-pr-comment")
 
 		// Start spinner for loading test reports
 		s := spinner.New(spinner.CharSets[11], 100*time.Millisecond)
@@ -104,19 +101,57 @@ var ReportCmd = &cobra.Command{
 		}
 		fmt.Printf("All tests report saved to %s\n", allTestsReportPath)
 
-		// Generate and save the summary reports (all tests) in specified formats
-		for _, format := range formats {
-			format = strings.ToLower(strings.TrimSpace(format))
+		// Generate GitHub summary markdown
+		s = spinner.New(spinner.CharSets[11], 100*time.Millisecond)
+		s.Suffix = " Generating GitHub summary markdown..."
+		s.Start()
+
+		err = generateGitHubSummaryMarkdown(aggregatedReport, filepath.Join(outputDir, "all-tests"))
+		if err != nil {
+			s.Stop()
+			return fmt.Errorf("error generating GitHub summary markdown: %w", err)
+		}
+		s.Stop()
+		fmt.Println("GitHub summary markdown generated successfully.")
+
+		if generatePRComment {
+			// Retrieve required flags
+			currentBranch, _ := cmd.Flags().GetString("current-branch")
+			currentCommitSHA, _ := cmd.Flags().GetString("current-commit-sha")
+			baseBranch, _ := cmd.Flags().GetString("base-branch")
+			repoURL, _ := cmd.Flags().GetString("repo-url")
+			actionRunID, _ := cmd.Flags().GetString("action-run-id")
+
+			// Validate that required flags are provided
+			missingFlags := []string{}
+			if currentBranch == "" {
+				missingFlags = append(missingFlags, "--current-branch")
+			}
+			if currentCommitSHA == "" {
+				missingFlags = append(missingFlags, "--current-commit-sha")
+			}
+			if repoURL == "" {
+				missingFlags = append(missingFlags, "--repo-url")
+			}
+			if actionRunID == "" {
+				missingFlags = append(missingFlags, "--action-run-id")
+			}
+			if len(missingFlags) > 0 {
+				return fmt.Errorf("the following flags are required when --generate-pr-comment is set: %s", strings.Join(missingFlags, ", "))
+			}
+
+			// Generate PR comment markdown
 			s = spinner.New(spinner.CharSets[11], 100*time.Millisecond)
-			s.Suffix = fmt.Sprintf(" Generating all tests summary in format %s...", format)
+			s.Suffix = " Generating PR comment markdown..."
 			s.Start()
 
-			if err := generateReport(aggregatedReport, format, filepath.Join(outputDir, "all-tests")); err != nil {
+			err = generatePRCommentMarkdown(aggregatedReport, filepath.Join(outputDir, "all-tests"), baseBranch, currentBranch, currentCommitSHA, repoURL, actionRunID)
+			if err != nil {
 				s.Stop()
-				return fmt.Errorf("error generating all tests summary in format %s: %w", format, err)
+				return fmt.Errorf("error generating PR comment markdown: %w", err)
 			}
 			s.Stop()
-			fmt.Printf("All tests summary in format %s generated successfully.\n", format)
+			fmt.Println("PR comment markdown generated successfully.")
 		}
 
 		// Filter failed tests (PassRatio < maxPassRatio and not skipped)
@@ -163,38 +198,40 @@ var ReportCmd = &cobra.Command{
 func init() {
 	ReportCmd.Flags().StringP("results-path", "p", "", "Path to the folder containing JSON test result files (required)")
 	ReportCmd.Flags().StringP("output-path", "o", "./report", "Path to output the generated report files")
-	ReportCmd.Flags().StringP("format", "f", "markdown,json", "Comma-separated list of summary report formats (markdown,json)")
 	ReportCmd.Flags().Float64P("max-pass-ratio", "", 1.0, "The maximum pass ratio threshold for a test to be considered flaky")
 	ReportCmd.Flags().StringP("codeowners-path", "", "", "Path to the CODEOWNERS file")
 	ReportCmd.Flags().StringP("repo-path", "", ".", "The path to the root of the repository/project")
+	ReportCmd.Flags().Bool("generate-pr-comment", false, "Set to true to generate PR comment markdown")
+	ReportCmd.Flags().String("base-branch", "develop", "The base branch to compare against (used in PR comment)")
+	ReportCmd.Flags().String("current-branch", "", "The current branch name (required if generate-pr-comment is set)")
+	ReportCmd.Flags().String("current-commit-sha", "", "The current commit SHA (required if generate-pr-comment is set)")
+	ReportCmd.Flags().String("repo-url", "", "The repository URL (required if generate-pr-comment is set)")
+	ReportCmd.Flags().String("action-run-id", "", "The GitHub Actions run ID (required if generate-pr-comment is set)")
+
 	ReportCmd.MarkFlagRequired("results-path")
 }
 
-func generateReport(report *reports.TestReport, format, outputPath string) error {
+func generateGitHubSummaryMarkdown(report *reports.TestReport, outputPath string) error {
 	fs := reports.OSFileSystem{}
-	format = strings.ToLower(strings.TrimSpace(format))
-
-	switch format {
-	case "markdown":
-		// Adjust the markdown filename to include "-summary"
-		mdFileName := outputPath + "-summary.md"
-		mdFile, err := fs.Create(mdFileName)
-		if err != nil {
-			return fmt.Errorf("error creating markdown file: %w", err)
-		}
-		defer mdFile.Close()
-		reports.GenerateMarkdownSummary(mdFile, report, 1.0)
-	case "json":
-		// Generate summary JSON
-		summaryData := reports.GenerateSummaryData(report.Results, 1.0)
-		summaryFileName := outputPath + "-summary.json"
-		if err := reports.SaveSummaryAsJSON(fs, summaryFileName, summaryData); err != nil {
-			return fmt.Errorf("error saving summary JSON: %w", err)
-		}
-	default:
-		return fmt.Errorf("unsupported summary report format: %s", format)
+	mdFileName := outputPath + "-summary.md"
+	mdFile, err := fs.Create(mdFileName)
+	if err != nil {
+		return fmt.Errorf("error creating GitHub summary markdown file: %w", err)
 	}
+	defer mdFile.Close()
+	reports.GenerateGitHubSummaryMarkdown(mdFile, report, 1.0)
+	return nil
+}
 
+func generatePRCommentMarkdown(report *reports.TestReport, outputPath, baseBranch, currentBranch, currentCommitSHA, repoURL, actionRunID string) error {
+	fs := reports.OSFileSystem{}
+	mdFileName := outputPath + "-pr-comment.md"
+	mdFile, err := fs.Create(mdFileName)
+	if err != nil {
+		return fmt.Errorf("error creating PR comment markdown file: %w", err)
+	}
+	defer mdFile.Close()
+	reports.GeneratePRCommentMarkdown(mdFile, report, 1.0, baseBranch, currentBranch, currentCommitSHA, repoURL, actionRunID)
 	return nil
 }
 
diff --git a/tools/flakeguard/reports/presentation.go b/tools/flakeguard/reports/presentation.go
@@ -83,7 +83,7 @@ func formatPassRatio(passRatio float64) string {
 	return fmt.Sprintf("%.2f%%", passRatio*100)
 }
 
-func GenerateMarkdownSummary(w io.Writer, testReport *TestReport, maxPassRatio float64) {
+func GenerateGitHubSummaryMarkdown(w io.Writer, testReport *TestReport, maxPassRatio float64) {
 	settingsTable := buildSettingsTable(testReport, maxPassRatio)
 	fmt.Fprint(w, "# Flakeguard Summary\n\n")
 	printTable(w, settingsTable)
@@ -104,6 +104,45 @@ func GenerateMarkdownSummary(w io.Writer, testReport *TestReport, maxPassRatio f
 	RenderResults(w, testReport.Results, maxPassRatio, true)
 }
 
+func GeneratePRCommentMarkdown(w io.Writer, testReport *TestReport, maxPassRatio float64, baseBranch, currentBranch, currentCommitSHA, repoURL, actionRunID string) {
+	fmt.Fprint(w, "# Flakeguard Summary\n\n")
+
+	// Construct additionalInfo inside the function
+	additionalInfo := fmt.Sprintf(
+		"Ran new or updated tests between `%s` and %s (`%s`).",
+		baseBranch,
+		currentCommitSHA,
+		currentBranch,
+	)
+
+	// Construct the links
+	viewDetailsLink := fmt.Sprintf("[View Flaky Detector Details](%s/actions/runs/%s)", repoURL, actionRunID)
+	compareChangesLink := fmt.Sprintf("[Compare Changes](%s/compare/%s...%s#files_bucket)", repoURL, baseBranch, currentCommitSHA)
+	linksLine := fmt.Sprintf("%s | %s", viewDetailsLink, compareChangesLink)
+
+	// Include additional information
+	fmt.Fprintln(w, additionalInfo)
+	fmt.Fprintln(w) // Add an extra newline for formatting
+
+	// Include the links
+	fmt.Fprintln(w, linksLine)
+	fmt.Fprintln(w) // Add an extra newline for formatting
+
+	if len(testReport.Results) == 0 {
+		fmt.Fprintln(w, "## No tests ran :warning:")
+		return
+	}
+
+	summary := GenerateSummaryData(testReport.Results, maxPassRatio)
+	if summary.AveragePassRatio < maxPassRatio {
+		fmt.Fprintln(w, "## Found Flaky Tests :x:")
+	} else {
+		fmt.Fprintln(w, "## No Flakes Found :white_check_mark:")
+	}
+
+	RenderResults(w, testReport.Results, maxPassRatio, true)
+}
+
 func buildSettingsTable(testReport *TestReport, maxPassRatio float64) [][]string {
 	rows := [][]string{
 		{"**Setting**", "**Value**"},
diff --git a/tools/flakeguard/reports/presentation_test.go b/tools/flakeguard/reports/presentation_test.go
@@ -2,6 +2,7 @@ package reports
 
 import (
 	"bytes"
+	"fmt"
 	"reflect"
 	"strings"
 	"testing"
@@ -71,8 +72,8 @@ func TestGenerateFlakyTestsTable(t *testing.T) {
 	}
 }
 
-// TestGenerateMarkdownSummary tests the GenerateMarkdownSummary function.
-func TestGenerateMarkdownSummary(t *testing.T) {
+// TestGenerateGitHubSummaryMarkdown tests the GenerateGitHubSummaryMarkdown function.
+func TestGenerateGitHubSummaryMarkdown(t *testing.T) {
 	testReport := &TestReport{
 		GoProject:     "ProjectX",
 		TestRunCount:  3,
@@ -104,7 +105,7 @@ func TestGenerateMarkdownSummary(t *testing.T) {
 	var buffer bytes.Buffer
 	maxPassRatio := 0.9
 
-	GenerateMarkdownSummary(&buffer, testReport, maxPassRatio)
+	GenerateGitHubSummaryMarkdown(&buffer, testReport, maxPassRatio)
 
 	output := buffer.String()
 
@@ -126,6 +127,77 @@ func TestGenerateMarkdownSummary(t *testing.T) {
 	}
 }
 
+// TestGeneratePRCommentMarkdown tests the GeneratePRCommentMarkdown function.
+func TestGeneratePRCommentMarkdown(t *testing.T) {
+	testReport := &TestReport{
+		GoProject:     "ProjectX",
+		TestRunCount:  3,
+		RaceDetection: true,
+		Results: []TestResult{
+			{
+				TestName:    "TestA",
+				PassRatio:   0.8,
+				Runs:        5,
+				Successes:   4,
+				Failures:    1,
+				TestPackage: "pkg1",
+				CodeOwners:  []string{"owner1"},
+				Durations:   []time.Duration{time.Second, time.Second, time.Second, time.Second, time.Second},
+			},
+			{
+				TestName:    "TestB",
+				PassRatio:   1.0,
+				Runs:        3,
+				Successes:   3,
+				Failures:    0,
+				TestPackage: "pkg2",
+				CodeOwners:  []string{"owner2"},
+				Durations:   []time.Duration{2 * time.Second, 2 * time.Second, 2 * time.Second},
+			},
+		},
+	}
+
+	var buffer bytes.Buffer
+	maxPassRatio := 0.9
+	baseBranch := "develop"
+	currentBranch := "feature-branch"
+	currentCommitSHA := "abcdef1234567890"
+	repoURL := "https://github.com/example/repo"
+	actionRunID := "123456789"
+
+	GeneratePRCommentMarkdown(&buffer, testReport, maxPassRatio, baseBranch, currentBranch, currentCommitSHA, repoURL, actionRunID)
+
+	output := buffer.String()
+
+	fmt.Println(output)
+
+	// Check that the output includes the expected headings and links
+	if !strings.Contains(output, "# Flakeguard Summary") {
+		t.Error("Expected markdown summary to contain '# Flakeguard Summary'")
+	}
+	if !strings.Contains(output, fmt.Sprintf("Ran new or updated tests between `%s` and %s (`%s`).", baseBranch, currentCommitSHA, currentBranch)) {
+		t.Error("Expected markdown to contain the additional info line with branches and commit SHA")
+	}
+	if !strings.Contains(output, fmt.Sprintf("[View Flaky Detector Details](%s/actions/runs/%s)", repoURL, actionRunID)) {
+		t.Error("Expected markdown to contain the 'View Flaky Detector Details' link")
+	}
+	if !strings.Contains(output, fmt.Sprintf("[Compare Changes](%s/compare/%s...%s#files_bucket)", repoURL, baseBranch, currentCommitSHA)) {
+		t.Error("Expected markdown to contain the 'Compare Changes' link")
+	}
+	if !strings.Contains(output, "## Found Flaky Tests :x:") {
+		t.Error("Expected markdown summary to contain '## Found Flaky Tests :x:'")
+	}
+	if !strings.Contains(output, "| **Name**") {
+		t.Error("Expected markdown table headers for test results")
+	}
+	if !strings.Contains(output, "| TestA ") {
+		t.Error("Expected markdown table to include TestA")
+	}
+	if strings.Contains(output, "| TestB ") {
+		t.Error("Did not expect markdown table to include TestB since its pass ratio is above the threshold")
+	}
+}
+
 // TestPrintTable tests the printTable function.
 func TestPrintTable(t *testing.T) {
 	table := [][]string{
