TT-2002 Update Flakeguard GitHub summary (#1650)

lukaszcl · web-flow · commit 66ea05943a1c · 2025-02-21T17:41:47.000+01:00
diff --git a/tools/flakeguard/reports/data.go b/tools/flakeguard/reports/data.go
@@ -28,7 +28,10 @@ type TestReport struct {
 
 // GenerateSummaryData generates a summary of a report's test results
 func (testReport *TestReport) GenerateSummaryData() {
-	var runs, testRunCount, passes, fails, skips, panickedTests, racedTests, flakyTests, skippedTests int
+	var runs, testRunCount, passes, fails, skips, panickedTests, racedTests, flakyTests int
+
+	// Map to hold unique test names that were entirely skipped
+	uniqueSkippedTestsMap := make(map[string]struct{})
 
 	for _, result := range testReport.Results {
 		runs += result.Runs
@@ -39,9 +42,8 @@ func (testReport *TestReport) GenerateSummaryData() {
 		fails += result.Failures
 		skips += result.Skips
 
-		// Count tests that were entirely skipped
 		if result.Runs == 0 && result.Skipped {
-			skippedTests++
+			uniqueSkippedTestsMap[result.TestName] = struct{}{}
 		}
 
 		if result.Panic {
@@ -55,6 +57,9 @@ func (testReport *TestReport) GenerateSummaryData() {
 		}
 	}
 
+	// Calculate the unique count of skipped tests
+	uniqueSkippedTestCount := len(uniqueSkippedTestsMap)
+
 	// Calculate the raw pass ratio
 	passRatio := passRatio(passes, runs)
 
@@ -66,12 +71,13 @@ func (testReport *TestReport) GenerateSummaryData() {
 	flakeTestRatioStr := formatRatio(flakeRatio)
 
 	testReport.SummaryData = &SummaryData{
-		UniqueTestsRun:   totalTests,
-		TestRunCount:     testRunCount,
-		PanickedTests:    panickedTests,
-		RacedTests:       racedTests,
-		FlakyTests:       flakyTests,
-		FlakyTestPercent: flakeTestRatioStr,
+		UniqueTestsRun:         totalTests,
+		UniqueSkippedTestCount: uniqueSkippedTestCount,
+		TestRunCount:           testRunCount,
+		PanickedTests:          panickedTests,
+		RacedTests:             racedTests,
+		FlakyTests:             flakyTests,
+		FlakyTestPercent:       flakeTestRatioStr,
 
 		TotalRuns:   runs,
 		PassedRuns:  passes,
@@ -112,6 +118,8 @@ type SummaryData struct {
 	// Overall test run stats
 	// UniqueTestsRun tracks how many unique tests were run
 	UniqueTestsRun int `json:"unique_tests_run"`
+	// UniqueSkippedTestCount tracks how many unique tests were entirely skipped
+	UniqueSkippedTestCount int `json:"unique_skipped_test_count"`
 	// TestRunCount tracks the max amount of times the tests were run, giving an idea of how many times flakeguard was executed
 	// e.g. if TestA was run 5 times, and TestB was run 10 times, UniqueTestsRun == 2 and TestRunCount == 10
 	TestRunCount int `json:"test_run_count"`
diff --git a/tools/flakeguard/reports/data_test.go b/tools/flakeguard/reports/data_test.go
@@ -27,17 +27,18 @@ func TestGenerateSummaryData(t *testing.T) {
 				MaxPassRatio: 1.0,
 			},
 			expected: &SummaryData{
-				UniqueTestsRun:   2,
-				TestRunCount:     10,
-				PanickedTests:    0,
-				RacedTests:       0,
-				FlakyTests:       0,
-				FlakyTestPercent: "0%", // no flaky tests
-				TotalRuns:        15,
-				PassedRuns:       15,
-				FailedRuns:       0,
-				SkippedRuns:      0,
-				PassPercent:      "100%",
+				UniqueTestsRun:         2,
+				TestRunCount:           10,
+				PanickedTests:          0,
+				RacedTests:             0,
+				FlakyTests:             0,
+				FlakyTestPercent:       "0%", // no flaky tests
+				TotalRuns:              15,
+				PassedRuns:             15,
+				FailedRuns:             0,
+				SkippedRuns:            0,
+				PassPercent:            "100%",
+				UniqueSkippedTestCount: 0,
 			},
 		},
 		{
@@ -60,10 +61,11 @@ func TestGenerateSummaryData(t *testing.T) {
 				FlakyTestPercent: "66.6667%",
 				TotalRuns:        19,
 				PassedRuns:       15,
-				FailedRuns:       4, // total failures
+				FailedRuns:       4,
 				SkippedRuns:      0,
 				// 15/19 => ~78.947...
-				PassPercent: "78.9474%",
+				PassPercent:            "78.9474%",
+				UniqueSkippedTestCount: 0,
 			},
 		},
 		{
@@ -89,7 +91,8 @@ func TestGenerateSummaryData(t *testing.T) {
 				FailedRuns:       1,
 				SkippedRuns:      0,
 				// 17/18 => ~94.444...
-				PassPercent: "94.4444%",
+				PassPercent:            "94.4444%",
+				UniqueSkippedTestCount: 0,
 			},
 		},
 		{
@@ -104,70 +107,74 @@ func TestGenerateSummaryData(t *testing.T) {
 				PanickedTests:    0,
 				RacedTests:       0,
 				FlakyTests:       0,
-				FlakyTestPercent: "0%", // no tests => 0%
+				FlakyTestPercent: "0%",
 				TotalRuns:        0,
 				PassedRuns:       0,
 				FailedRuns:       0,
 				SkippedRuns:      0,
 				// With zero runs, we default passRatio to "100%"
-				PassPercent: "100%",
+				PassPercent:            "100%",
+				UniqueSkippedTestCount: 0,
 			},
 		},
 		{
 			name: "Skipped tests included in total but not executed",
 			testReport: &TestReport{
 				Results: []TestResult{
+					// Skipped test with no runs should be counted in UniqueSkippedTestCount.
 					{PassRatio: -1.0, Runs: 0, Successes: 0, Skips: 1, Skipped: true},
 					{PassRatio: 0.7, Runs: 10, Successes: 7, Failures: 3},
 				},
 				MaxPassRatio: 0.8,
 			},
 			expected: &SummaryData{
-				UniqueTestsRun:   2,
-				TestRunCount:     10,
-				PanickedTests:    0,
-				RacedTests:       0,
-				FlakyTests:       1,     // second test has ratio=0.7 => "flaky"
-				FlakyTestPercent: "50%", // 1 out of 2 => 50%
-				TotalRuns:        10,
-				PassedRuns:       7,
-				FailedRuns:       3,
-				SkippedRuns:      1, // from first test
-				PassPercent:      "70%",
+				UniqueTestsRun:         2,
+				TestRunCount:           10,
+				PanickedTests:          0,
+				RacedTests:             0,
+				FlakyTests:             1,
+				FlakyTestPercent:       "50%",
+				TotalRuns:              10,
+				PassedRuns:             7,
+				FailedRuns:             3,
+				SkippedRuns:            1,
+				PassPercent:            "70%",
+				UniqueSkippedTestCount: 1,
 			},
 		},
 		{
 			name: "Mixed skipped and executed tests",
 			testReport: &TestReport{
 				Results: []TestResult{
+					// Skipped test should be counted for UniqueSkippedTestCount.
 					{PassRatio: -1.0, Runs: 0, Successes: 0, Skips: 1, Skipped: true},
 					{PassRatio: 0.9, Runs: 10, Successes: 9, Failures: 1},
 					{PassRatio: 0.5, Runs: 4, Successes: 2, Failures: 2},
 				},
 				MaxPassRatio: 0.85,
 			},
 			expected: &SummaryData{
-				UniqueTestsRun:   3,
-				TestRunCount:     10,
-				PanickedTests:    0,
-				RacedTests:       0,
-				FlakyTests:       1,          // last test has ratio=0.5 => "flaky"
-				FlakyTestPercent: "33.3333%", // 1 out of 3 => 33.333...
-				TotalRuns:        14,         // 10 + 4
-				PassedRuns:       11,         // 9 + 2
-				FailedRuns:       3,          // 1 + 2
-				SkippedRuns:      1,          // from first test
-				// 11/14 => 78.5714...
-				PassPercent: "78.5714%",
+				UniqueTestsRun:         3,
+				TestRunCount:           10,
+				PanickedTests:          0,
+				RacedTests:             0,
+				FlakyTests:             1,
+				FlakyTestPercent:       "33.3333%",
+				TotalRuns:              14,
+				PassedRuns:             11,
+				FailedRuns:             3,
+				SkippedRuns:            1,
+				PassPercent:            "78.5714%",
+				UniqueSkippedTestCount: 1,
 			},
 		},
 		{
 			name: "Tiny flake ratio that is exactly 0.01%",
 			testReport: &TestReport{
 				Results: func() []TestResult {
 					// 9,999 total:
-					//  - 9,998 stable => pass=1.0
-					//  - 1 flaky => pass=0.5
+					//  - 9,998 stable tests => pass ratio = 1.0
+					//  - 1 flaky test => pass ratio = 0.5
 					const total = 9999
 					tests := make([]TestResult, total)
 					for i := 0; i < total-1; i++ {
@@ -178,7 +185,7 @@ func TestGenerateSummaryData(t *testing.T) {
 						}
 					}
 					tests[total-1] = TestResult{
-						PassRatio: 0.5, // 1 success, 1 fail
+						PassRatio: 0.5, // 1 success, 1 failure
 						Runs:      2,
 						Successes: 1,
 						Failures:  1,
@@ -188,17 +195,47 @@ func TestGenerateSummaryData(t *testing.T) {
 				MaxPassRatio: 1.0,
 			},
 			expected: &SummaryData{
-				UniqueTestsRun:   9999,
-				TestRunCount:     10,
-				PanickedTests:    0,
-				RacedTests:       0,
-				FlakyTests:       1,
-				FlakyTestPercent: "0.01%",
-				TotalRuns:        (9998 * 10) + 2,
-				PassedRuns:       (9998 * 10) + 1,
-				FailedRuns:       1,
-				SkippedRuns:      0,
-				PassPercent:      "99.999%",
+				UniqueTestsRun:         9999,
+				TestRunCount:           10,
+				PanickedTests:          0,
+				RacedTests:             0,
+				FlakyTests:             1,
+				FlakyTestPercent:       "0.01%",
+				TotalRuns:              (9998 * 10) + 2,
+				PassedRuns:             (9998 * 10) + 1,
+				FailedRuns:             1,
+				SkippedRuns:            0,
+				PassPercent:            "99.999%",
+				UniqueSkippedTestCount: 0,
+			},
+		},
+		{
+			name: "Duplicate skipped tests",
+			testReport: &TestReport{
+				Results: []TestResult{
+					// Two entries for "TestA" should count as one unique skipped test.
+					{TestName: "TestA", PassRatio: -1.0, Runs: 0, Skips: 1, Skipped: true},
+					{TestName: "TestA", PassRatio: -1.0, Runs: 0, Skips: 1, Skipped: true},
+					// A different test "TestB"
+					{TestName: "TestB", PassRatio: -1.0, Runs: 0, Skips: 1, Skipped: true},
+					// This test was executed so it should not count as skipped.
+					{TestName: "TestC", PassRatio: 1.0, Runs: 5, Successes: 5, Skipped: false},
+				},
+				MaxPassRatio: 1.0,
+			},
+			expected: &SummaryData{
+				UniqueTestsRun:         4,
+				TestRunCount:           5,
+				PanickedTests:          0,
+				RacedTests:             0,
+				FlakyTests:             0,
+				FlakyTestPercent:       "0%",
+				TotalRuns:              5,
+				PassedRuns:             5,
+				FailedRuns:             0,
+				SkippedRuns:            3, // Sum of Skips for all skipped tests.
+				PassPercent:            "100%",
+				UniqueSkippedTestCount: 2, // Only "TestA" and "TestB" count.
 			},
 		},
 	}
diff --git a/tools/flakeguard/reports/presentation.go b/tools/flakeguard/reports/presentation.go
@@ -195,27 +195,27 @@ func RenderResults(
 	collapsible bool,
 ) {
 	resultsTable := GenerateFlakyTestsTable(testReport, markdown)
-	renderSummaryTable(w, testReport.SummaryData, markdown, false) // Don't make the summary collapsible
+	renderSummaryTable(w, testReport.SummaryData, markdown, false, testReport.RaceDetection) // Don't make the summary collapsible
 	renderTestResultsTable(w, resultsTable, collapsible)
 }
 
 // renderSummaryTable renders a summary table with the given data into a console or markdown format.
 // If in markdown mode, the table can also be made collapsible.
-func renderSummaryTable(w io.Writer, summary *SummaryData, markdown bool, collapsible bool) {
+func renderSummaryTable(w io.Writer, summary *SummaryData, markdown bool, collapsible bool, raceDetection bool) {
 	summaryData := [][]string{
 		{"Category", "Total"},
 		{"Unique Tests", fmt.Sprintf("%d", summary.UniqueTestsRun)},
-		{"Test Run Count", fmt.Sprintf("%d", summary.TestRunCount)},
-		{"Panicked Tests", fmt.Sprintf("%d", summary.PanickedTests)},
-		{"Raced Tests", fmt.Sprintf("%d", summary.RacedTests)},
-		{"Flaky Tests", fmt.Sprintf("%d", summary.FlakyTests)},
-		{"Flaky Test Percent", summary.FlakyTestPercent},
+		{"Unique Skipped Tests", fmt.Sprintf("%d", summary.UniqueSkippedTestCount)},
 		{"Total Test Runs", fmt.Sprintf("%d", summary.TotalRuns)},
-		{"Passes", fmt.Sprintf("%d", summary.PassedRuns)},
-		{"Failures", fmt.Sprintf("%d", summary.FailedRuns)},
-		{"Skips", fmt.Sprintf("%d", summary.SkippedRuns)},
-		{"Pass Percent", summary.PassPercent},
+		{"Passed Test Runs", fmt.Sprintf("%d (%s)", summary.PassedRuns, summary.PassPercent)},
+		{"Flaky Test Runs", fmt.Sprintf("%d (%s)", summary.FlakyTests, summary.FlakyTestPercent)},
+		{"Panicked Tests", fmt.Sprintf("%d", summary.PanickedTests)},
 	}
+	// Only include "Raced Tests" row if race detection is enabled.
+	if raceDetection {
+		summaryData = append(summaryData, []string{"Raced Tests", fmt.Sprintf("%d", summary.RacedTests)})
+	}
+
 	if markdown {
 		for i, row := range summaryData {
 			if i == 0 {
diff --git a/tools/flakeguard/runner/runner.go b/tools/flakeguard/runner/runner.go
@@ -84,6 +84,7 @@ func (r *Runner) RunTestPackages(packages []string) (*reports.TestReport, error)
 		MaxPassRatio:  r.MaxPassRatio,
 	}
 	report.GenerateSummaryData()
+
 	return report, nil
 }
 
@@ -119,6 +120,7 @@ func (r *Runner) RunTestCmd(testCmd []string) (*reports.TestReport, error) {
 		MaxPassRatio:  r.MaxPassRatio,
 	}
 	report.GenerateSummaryData()
+
 	return report, nil
 }
 

Original file line number	Diff line number	Diff line change
`@@ -84,6 +84,7 @@ func (r Runner) RunTestPackages(packages []string) (reports.TestReport, error)`
`84`	`84`	`MaxPassRatio: r.MaxPassRatio,`
`85`	`85`	`}`
`86`	`86`	`report.GenerateSummaryData()`
	`87`	`+`
`87`	`88`	`return report, nil`
`88`	`89`	`}`
`89`	`90`
`@@ -119,6 +120,7 @@ func (r Runner) RunTestCmd(testCmd []string) (reports.TestReport, error) {`
`119`	`120`	`MaxPassRatio: r.MaxPassRatio,`
`120`	`121`	`}`
`121`	`122`	`report.GenerateSummaryData()`
	`123`	`+`
`122`	`124`	`return report, nil`
`123`	`125`	`}`
`124`	`126`