Slimmer reporting

Author: kalverra

tools/flakeguard/.gitignore

@@ -1,5 +1,6 @@
-tmp_test_flaky_state
+tmp_test_flaky*
 test_results_*.json
 debug_outputs/
-example_results.json
-example_results.md
+example_results/
+example*.json
+example*.md

tools/flakeguard/Makefile

@@ -10,4 +10,18 @@ test:
 
 .PHONY: example
 example:
-	go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results.json
+	rm -rf example_results
+	mkdir -p example_results
+	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic,TestFlakyPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_1.json
+	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic,TestFlakyPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
+	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic,TestFlakyPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
+	go run . aggregate-results --results-path ./example_results --output-results ./example_results/all_tests_example.json
+
+.PHONY: example_panic
+example_panic:
+	rm -rf example_results
+	mkdir -p example_results
+	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_1.json
+	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
+	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
+	go run . aggregate-results --results-path ./example_results --output-results ./example_results/all_tests_example.json

tools/flakeguard/README.md

@@ -30,3 +30,13 @@ Run with `--help` to see all flags for the commands.
 ### JSON Output
 
 Both `find` and `run` commands support JSON output `--json`, making it easy to integrate Flakeguard with CI/CD pipelines and reporting tools.
+
+### Example Run
+
+You can find example usage and see outputs with:
+
+```sh
+make example       # Run an example flow of running tests and aggregating results
+make example_panic # Run example flow with panicking tests
+ls example_results # See results of each run and aggregation
+```

tools/flakeguard/reports/reports.go

@@ -119,7 +119,7 @@ func AggregateTestResults(folderPath string) (*TestReport, error) {
 			for _, test := range report.ExcludedTests {
 				excludedTests[test] = struct{}{}
 			}
-			// Process each test result
+			// Process each test results
 			for _, result := range report.Results {
 				// Unique key for each test based on TestName and TestPackage
 				key := result.TestName + "|" + result.TestPackage
@@ -180,7 +180,7 @@ func AggregateTestResults(folderPath string) (*TestReport, error) {
 // PrintTests prints tests in a pretty format
 func PrintTests(w io.Writer, tests []TestResult, maxPassRatio float64) (allRuns, passes, fails, panics, skips, races, flakes int) {
 	headers := []string{
-		"Test Name", "Test Package", "Pass Ratio", "Skipped", "Runs", "Successes", "Failures", "Panics", "Races", "Skips", "Avg Duration",
+		"**Test Name**", "**Test Package**", "**Package Panicked**", "**Pass Ratio**", "**Runs**", "**Successes**", "**Failures**", "**Panics**", "**Races**", "**Skips**", "**Avg Duration**",
 	}
 
 	// Build test rows and summary data
@@ -190,8 +190,8 @@ func PrintTests(w io.Writer, tests []TestResult, maxPassRatio float64) (allRuns,
 			rows = append(rows, []string{
 				test.TestName,
 				test.TestPackage,
+				fmt.Sprintf("%t", test.PackagePanicked),
 				fmt.Sprintf("%.2f%%", test.PassRatio*100),
-				fmt.Sprintf("%v", test.Skipped),
 				fmt.Sprintf("%d", test.Runs),
 				fmt.Sprintf("%d", test.Successes),
 				fmt.Sprintf("%d", test.Failures),
@@ -208,19 +208,21 @@ func PrintTests(w io.Writer, tests []TestResult, maxPassRatio float64) (allRuns,
 		skips += test.Skips
 		races += test.Races
 		panics += test.Panics
-		flakes += fails + races + panics
+		if test.PassRatio < maxPassRatio {
+			flakes++
+		}
 	}
 
 	// Print out summary data
 	summaryData := [][]string{
-		{"**Summary**", "**Value**"},
-		{"Runs", fmt.Sprint(allRuns)},
-		{"Passes", fmt.Sprint(passes)},
-		{"Failures", fmt.Sprint(fails)},
-		{"Panics", fmt.Sprint(panics)},
-		{"Skips", fmt.Sprint(skips)},
-		{"Races", fmt.Sprint(races)},
-		{"Flakes", fmt.Sprint(flakes)},
+		{"**Category**", "**Total**"},
+		{"**Runs**", fmt.Sprint(allRuns)},
+		{"**Passes**", fmt.Sprint(passes)},
+		{"**Failures**", fmt.Sprint(fails)},
+		{"**Panics**", fmt.Sprint(panics)},
+		{"**Skips**", fmt.Sprint(skips)},
+		{"**Races**", fmt.Sprint(races)},
+		{"**Flaky Tests**", fmt.Sprint(flakes)},
 	}
 	colWidths := make([]int, len(rows[0]))
 
@@ -238,13 +240,11 @@ func PrintTests(w io.Writer, tests []TestResult, maxPassRatio float64) (allRuns,
 	printSeparator := func() {
 		fmt.Fprintf(w, "|-%s-|-%s-|\n", strings.Repeat("-", colWidths[0]), strings.Repeat("-", colWidths[1]))
 	}
-	printSeparator()
 	printRow(summaryData[0])
 	printSeparator()
 	for _, row := range summaryData[1:] {
 		printRow(row)
 	}
-	printSeparator()
 	fmt.Fprintln(w)
 
 	// Print out test data
@@ -277,17 +277,11 @@ func PrintTests(w io.Writer, tests []TestResult, maxPassRatio float64) (allRuns,
 	}
 
 	// Print table
-	printSeparator()
 	printRow(headers)
 	printSeparator()
 	for _, row := range rows {
 		printRow(row)
 	}
-	printSeparator()
-
-	// Print totals
-	fmt.Fprintf(w, "\nSummary:\n")
-	fmt.Fprintf(w, "Total Runs: %d, Passes: %d, Failures: %d, Panics: %d, Races: %d, Skips: %d\n", allRuns, passes, fails, panics, races, skips)
 	return
 }
 
@@ -326,13 +320,11 @@ func MarkdownSummary(w io.Writer, testReport *TestReport, maxPassRatio float64)
 	}
 	fmt.Fprint(w, "# Flakeguard Summary\n\n")
 	// Print settings data
-	printSeparator()
 	printRow(rows[0])
 	printSeparator()
 	for _, row := range rows[1:] {
 		printRow(row)
 	}
-	printSeparator()
 	fmt.Fprintln(w)
 
 	if len(tests) == 0 {
@@ -345,11 +337,11 @@ func MarkdownSummary(w io.Writer, testReport *TestReport, maxPassRatio float64)
 		avgPassRatio = float64(passes) / float64(allRuns)
 	}
 	if avgPassRatio < maxPassRatio {
-		fmt.Fprintln(w, "## Found Flaky Tests :x:")
+		fmt.Fprint(w, "## Found Flaky Tests :x:\n\n")
 	} else {
-		fmt.Fprintln(w, "## No Flakes Found :white_check_mark:")
+		fmt.Fprint(w, "## No Flakes Found :white_check_mark:\n\n")
 	}
-	fmt.Fprintf(w, "Ran `%d` tests `%d` times with a `%.2f%%` pass ratio and found `%d` flaky tests\n\n", len(tests), allRuns, avgPassRatio*100, flakes)
+	fmt.Fprintf(w, "Ran `%d` tests `%d` times, and found `%d` flaky tests with an overall `%.2f%%` pass ratio\n\n", len(tests), testReport.TestRunCount, flakes, avgPassRatio*100)
 	if avgPassRatio < maxPassRatio {
 		fmt.Fprint(w, "### Flakes\n\n")
 		fmt.Fprint(w, testsData.String())

tools/flakeguard/runner/example_test_package/example_tests_test.go

@@ -51,26 +51,26 @@ func TestPanic(t *testing.T) {
 	panic("This test intentionally panics")
 }
 
-// func TestFlakyPanic(t *testing.T) {
-// 	t.Parallel()
-
-// 	// Track if the test has run before
-// 	stateFile := "tmp_test_flaky_panic_state"
-
-// 	// If the state file does not exist, create it and fail the test
-// 	if _, err := os.Stat(stateFile); os.IsNotExist(err) {
-// 		if err := os.WriteFile(stateFile, []byte("run once"), 0644); err != nil { //nolint:gosec
-// 			t.Fatalf("THIS IS UNEXPECTED: failed to create state file: %v", err)
-// 		}
-// 		panic("This is a designed flaky test panicking as intended")
-// 	}
-// 	t.Cleanup(func() {
-// 		err := os.Remove(stateFile)
-// 		if err != nil {
-// 			t.Fatalf("THIS IS UNEXPECTED: failed to remove state file: %v", err)
-// 		}
-// 	})
-// }
+func TestFlakyPanic(t *testing.T) {
+	t.Parallel()
+
+	// Track if the test has run before
+	stateFile := "tmp_test_flaky_panic_state"
+
+	// If the state file does not exist, create it and fail the test
+	if _, err := os.Stat(stateFile); os.IsNotExist(err) {
+		if err := os.WriteFile(stateFile, []byte("run once"), 0644); err != nil { //nolint:gosec
+			t.Fatalf("THIS IS UNEXPECTED: failed to create state file: %v", err)
+		}
+		panic("This is a designed flaky test panicking as intended")
+	}
+	t.Cleanup(func() {
+		err := os.Remove(stateFile)
+		if err != nil {
+			t.Fatalf("THIS IS UNEXPECTED: failed to remove state file: %v", err)
+		}
+	})
+}
 
 func TestRace(t *testing.T) {
 	t.Parallel()

tools/flakeguard/runner/runner.go

@@ -70,10 +70,14 @@ func (r *Runner) RunTests() (*reports.TestReport, error) {
 		return nil, fmt.Errorf("failed to parse test results: %w", err)
 	}
 	for index, report := range results {
-		if report.Panics >= r.RunCount { // This feels hacky, but there aren't any elegant solutions
-			results[index].Failures = 0 // We can sometimes double-count panics as failures
-			results[index].Panics = r.RunCount
-			results[index].Runs = r.RunCount
+		if report.Runs > r.RunCount { // Panics can introduce double-counting test runs and mask panics as failures, this is a bit of a hacky way around it
+			if report.Panics > 0 {
+				results[index].Failures = 0 // We can sometimes double-count panics as failures
+				results[index].Panics = r.RunCount
+				results[index].Runs = r.RunCount
+			} else {
+				log.Printf("WARN: %s is reporting it ran %d times but it should have only ran %d", report.TestName, report.Runs, r.RunCount)
+			}
 		}
 	}
 	return &reports.TestReport{