Fixes reporting for Flakeguard aggregations

Author: kalverra

tools/flakeguard/Makefile

@@ -24,11 +24,7 @@ bench:
 
 .PHONY: example
 example:
-	rm -rf example_results
-	mkdir -p example_results
-	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic,Timeout --max-pass-ratio=1 --race=false --output-json=example_results/example_run_1.json
-	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic,Timeout --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
-	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=Panic,Timeout --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
+	./run_example.sh 3 TestPanic
 	go run . aggregate-results \
 		--results-path ./example_results \
 		--output-path ./example_results \
@@ -56,11 +52,7 @@ example:
 
 .PHONY: example_flaky_panic
 example_flaky_panic:
-	rm -rf example_results
-	mkdir -p example_results
-	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_1.json
-	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_2.json
-	- go run . run --project-path=./runner --test-packages=./example_test_package --run-count=5 --skip-tests=TestPanic --max-pass-ratio=1 --race=false --output-json=example_results/example_run_3.json
+	./run_example.sh 3 TestPanic
 	go run . aggregate-results \
 		--results-path ./example_results \
 		--output-path ./example_results \

tools/flakeguard/reports/data.go

@@ -28,12 +28,12 @@ type TestReport struct {
 
 // GenerateSummaryData generates a summary of a report's test results
 func (testReport *TestReport) GenerateSummaryData() {
-	var runs, mostRuns, passes, fails, skips, panickedTests, racedTests, flakyTests, skippedTests int
+	var runs, testRunCount, passes, fails, skips, panickedTests, racedTests, flakyTests, skippedTests int
 
 	for _, result := range testReport.Results {
 		runs += result.Runs
-		if result.Runs > mostRuns {
-			mostRuns = result.Runs
+		if result.Runs > testRunCount {
+			testRunCount = result.Runs
 		}
 		passes += result.Successes
 		fails += result.Failures
@@ -67,7 +67,7 @@ func (testReport *TestReport) GenerateSummaryData() {
 
 	testReport.SummaryData = &SummaryData{
 		UniqueTestsRun:   totalTests,
-		TestRunCount:     mostRuns,
+		TestRunCount:     testRunCount,
 		PanickedTests:    panickedTests,
 		RacedTests:       racedTests,
 		FlakyTests:       flakyTests,

tools/flakeguard/reports/io.go

@@ -127,7 +127,9 @@ func LoadAndAggregate(resultsPath string, options ...AggregateOption) (*TestRepo
 	}
 
 	// Apply options
-	opts := aggregateOptions{}
+	opts := aggregateOptions{
+		maxPassRatio: 1.0,
+	}
 	for _, opt := range options {
 		opt(&opts)
 	}

tools/flakeguard/run_example.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+run_flakeguard() {
+  local run_count=$1
+  local skip_tests=$2
+
+  echo "Running Flakeguard $run_count times"
+
+  for ((i=1; i<=$run_count; i++)); do
+    output_file="example_results/example_run_$i.json"
+    go run . run \
+      --project-path=./runner \
+      --test-packages=./example_test_package \
+      --run-count=5 \
+      --skip-tests=$skip_tests \
+      --max-pass-ratio=1 \
+      --race=false \
+      --output-json=$output_file
+    local EXIT_CODE=$?
+    if [ $EXIT_CODE -eq 2 ]; then
+      echo "ERROR: Flakeguard encountered an error while running tests"
+      exit $EXIT_CODE
+    fi
+  done
+}
+
+# Run the commands
+rm -rf example_results
+mkdir -p example_results
+
+run_flakeguard "$1" "$2"