Separates panic tests

Author: kalverra

tools/flakeguard/.gitignore

@@ -1,2 +1,2 @@
 tmp_test_flaky_state
-flaky_test_results.json
+flaky_test_results_*.json

tools/flakeguard/Makefile

@@ -6,4 +6,4 @@ test_unit:
 test:
 	go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
 	set -euo pipefail
-	go list ./... | grep -v 'flaky_test_package' | xargs go test -json -cover -coverprofile unit-test-coverage.out -v 2>&1 | tee /tmp/gotest.log | gotestfmt
+	go list ./... | grep -v 'flaky_test_package' | xargs go test -json -cover -coverprofile unit-test-coverage.out -v 2>&1 | tee /tmp/gotest.log | gotestfmt

tools/flakeguard/git/git_test.go

@@ -8,6 +8,8 @@ import (
 )
 
 func TestGetChangedGoPackagesFromDiff(t *testing.T) {
+	t.Parallel()
+
 	tests := []struct {
 		name        string
 		out         string

tools/flakeguard/runner/flaky_test_package/flaky_test.go

@@ -45,7 +45,7 @@ func TestSkipped(t *testing.T) {
 	t.Skip("This test is intentionally skipped")
 }
 
-// func TestPanic(t *testing.T) {
-// 	t.Parallel()
-// 	panic("This test intentionally panics")
-// }
+func TestPanic(t *testing.T) {
+	t.Parallel()
+	panic("This test intentionally panics")
+}

tools/flakeguard/runner/runner.go

@@ -2,9 +2,11 @@ package runner
 
 import (
 	"bufio"
+	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"log"
 	"os"
 	"os/exec"
@@ -26,6 +28,8 @@ type Runner struct {
 	FailFast             bool     // Stop on first test failure.
 	SkipTests            []string // Test names to exclude.
 	SelectedTestPackages []string // Explicitly selected packages to run.
+	CollectRawOutput     bool     // Collect test output for later inspection.
+	rawOutput            bytes.Buffer
 }
 
 // RunTests executes the tests for each provided package and aggregates all results.
@@ -48,6 +52,11 @@ func (r *Runner) RunTests() ([]reports.TestResult, error) {
 	return parseTestResults(jsonFilePaths)
 }
 
+// RawOutput retrieves the raw output from the test runs, if CollectRawOutput enabled.
+func (r *Runner) RawOutput() bytes.Buffer {
+	return r.rawOutput
+}
+
 type exitCoder interface {
 	ExitCode() int
 }
@@ -77,6 +86,13 @@ func (r *Runner) runTests(packageName string) (string, bool, error) {
 	// Run the command with output directed to the file
 	cmd := exec.Command("go", args...)
 	cmd.Dir = r.ProjectPath
+	if r.CollectRawOutput {
+		cmd.Stdout = io.MultiWriter(tmpFile, &r.rawOutput)
+		cmd.Stderr = io.MultiWriter(tmpFile, &r.rawOutput)
+	} else {
+		cmd.Stdout = tmpFile
+		cmd.Stderr = tmpFile
+	}
 	cmd.Stdout = tmpFile
 	cmd.Stderr = tmpFile
 
@@ -157,7 +173,7 @@ func parseTestResults(filePaths []string) ([]reports.TestResult, error) {
 			if entry.Output != "" {
 				if entry.Test != "" {
 					// Test-level output
-					result.Outputs = append(result.Outputs, entry.Output)
+					result.Outputs = append(result.Outputs, strings.TrimSpace(entry.Output))
 				} else {
 					// Package-level output
 					// Append to PackageOutputs of all TestResults in the same package

tools/flakeguard/runner/runner_test.go

@@ -19,25 +19,27 @@ func (m MockGoRunner) RunCommand(dir string, args []string) (string, bool, error
 	return m.RunFunc(dir, args)
 }
 
-func TestRun(t *testing.T) {
-	t.Parallel()
-
-	runs := 5
+type expectedTestResult struct {
+	TestResult *reports.TestResult
+	seen       bool
+}
 
-	runner := Runner{
+var (
+	defaultRuns       = 5
+	defaultTestRunner = Runner{
 		ProjectPath:          "./",
 		Verbose:              true,
-		RunCount:             runs,
+		RunCount:             defaultRuns,
 		UseRace:              false,
-		SkipTests:            []string{},
+		SkipTests:            []string{"TestPanic"},
 		FailFast:             false,
 		SelectedTestPackages: []string{"./flaky_test_package"},
+		CollectRawOutput:     true,
 	}
+)
 
-	expectedResults := map[string]*struct {
-		*reports.TestResult
-		seen bool
-	}{
+func TestRunDefault(t *testing.T) {
+	expectedResults := map[string]*expectedTestResult{
 		"TestFlaky": {
 			TestResult: &reports.TestResult{
 				TestName: "TestFlaky",
@@ -51,7 +53,7 @@ func TestRun(t *testing.T) {
 				Panicked:  false,
 				Skipped:   false,
 				PassRatio: 0,
-				Failures:  runs,
+				Failures:  defaultRuns,
 			},
 		},
 		"TestPass": {
@@ -60,17 +62,9 @@ func TestRun(t *testing.T) {
 				Panicked:  false,
 				Skipped:   false,
 				PassRatio: 1,
-				Successes: runs,
+				Successes: defaultRuns,
 			},
 		},
-		// "TestPanic": {
-		// 	TestResult: &reports.TestResult{
-		// 		TestName:  "TestPanic",
-		// 		Panicked:  true,
-		// 		Skipped:   false,
-		// 		PassRatio: 0,
-		// 	},
-		// },
 		"TestSkipped": {
 			TestResult: &reports.TestResult{
 				TestName:  "TestSkipped",
@@ -81,32 +75,129 @@ func TestRun(t *testing.T) {
 		},
 	}
 
-	testResults, err := runner.RunTests()
+	testResults, err := defaultTestRunner.RunTests()
 	require.NoError(t, err)
 	t.Cleanup(func() {
 		if t.Failed() {
-			t.Log("Writing test results to flaky_test_results.json")
+			resultsFileName := fmt.Sprintf("flaky_test_results_%s.json", t.Name())
+			t.Logf("Writing test results to %s", resultsFileName)
 			jsonResults, err := json.Marshal(testResults)
 			require.NoError(t, err)
-			err = os.WriteFile("flaky_test_results.json", jsonResults, 0644) //nolint:gosec
+			err = os.WriteFile(resultsFileName, jsonResults, 0644) //nolint:gosec
 			require.NoError(t, err)
 		}
 	})
+
 	for _, result := range testResults {
 		t.Run(fmt.Sprintf("checking results of %s", result.TestName), func(t *testing.T) {
 			expected, ok := expectedResults[result.TestName]
-			// Sanity checks
 			require.True(t, ok, "unexpected test result: %s", result.TestName)
 			require.False(t, expected.seen, "test '%s' was seen multiple times", result.TestName)
 			expected.seen = true
 
-			assert.Equal(t, runs, result.Runs, "test '%s' had an unexpected number of runs", result.TestName)
-			assert.Len(t, result.Durations, runs, "test '%s' had an unexpected number of durations as it was run %d times", result.TestName, runs)
-			if result.TestName == "TestSlow" {
-				for _, duration := range result.Durations {
-					assert.GreaterOrEqual(t, duration, float64(1), "slow test '%s' should have a duration of at least 2s", result.TestName)
-				}
+			assert.Equal(t, defaultRuns, result.Runs, "test '%s' had an unexpected number of runs", result.TestName)
+			assert.Len(t, result.Durations, result.Runs, "test '%s' has a mismatch of runs and duration counts", result.TestName, defaultRuns)
+			resultCounts := result.Successes + result.Failures + result.Panics + result.Skips
+			assert.Equal(t, result.Runs, resultCounts,
+				"test '%s' doesn't match Runs count with results counts\nRuns: %d\nSuccesses: %d\nFailures: %d\nPanics: %d\nSkips: %d\nTotal: %d",
+				result.TestName, result.Runs, result.Successes, result.Failures, result.Panics, result.Skips, resultCounts,
+			)
+			assert.Equal(t, expected.TestResult.Panicked, result.Panicked, "test '%s' had an unexpected panic result", result.TestName)
+			assert.Equal(t, expected.TestResult.Skipped, result.Skipped, "test '%s' had an unexpected skipped result", result.TestName)
+
+			if result.TestName == "TestFlaky" {
+				assert.Greater(t, result.Successes, 0, "flaky test '%s' should have passed some", result.TestName)
+				assert.Greater(t, result.Failures, 0, "flaky test '%s' should have failed some", result.TestName)
+				assert.Greater(t, result.PassRatio, float64(0), "flaky test '%s' should have a flaky pass ratio", result.TestName)
+				assert.Less(t, result.PassRatio, float64(1), "flaky test '%s' should have a flaky pass ratio", result.TestName)
+			} else {
+				assert.Equal(t, expected.TestResult.PassRatio, result.PassRatio, "test '%s' had an unexpected pass ratio", result.TestName)
+				assert.Equal(t, expected.TestResult.Successes, result.Successes, "test '%s' had an unexpected number of successes", result.TestName)
+				assert.Equal(t, expected.TestResult.Failures, result.Failures, "test '%s' had an unexpected number of failures", result.TestName)
 			}
+		})
+	}
+
+	for _, expected := range expectedResults {
+		assert.True(t, expected.seen, "expected test '%s' not found in test runs", expected.TestResult.TestName)
+	}
+}
+
+func TestRunWithPanics(t *testing.T) {
+	panicRunner := defaultTestRunner
+	panicRunner.SkipTests = []string{}
+
+	expectedResults := map[string]*expectedTestResult{
+		"TestFlaky": {
+			TestResult: &reports.TestResult{
+				TestName: "TestFlaky",
+				Panicked: true,
+				Skipped:  false,
+			},
+		},
+		"TestFail": {
+			TestResult: &reports.TestResult{
+				TestName:  "TestFail",
+				Panicked:  false,
+				Skipped:   false,
+				PassRatio: 0,
+				Failures:  defaultRuns,
+			},
+		},
+		"TestPass": {
+			TestResult: &reports.TestResult{
+				TestName:  "TestPass",
+				Panicked:  true,
+				Skipped:   false,
+				PassRatio: 1,
+				Successes: defaultRuns,
+			},
+		},
+		"TestSkipped": {
+			TestResult: &reports.TestResult{
+				TestName:  "TestSkipped",
+				Panicked:  true,
+				Skipped:   true,
+				PassRatio: 0,
+			},
+		},
+		"TestPanic": {
+			TestResult: &reports.TestResult{
+				TestName: "TestPanic",
+				Panicked: true,
+				Skipped:  false,
+			},
+		},
+	}
+
+	testResults, err := panicRunner.RunTests()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		if t.Failed() {
+			resultsFileName := fmt.Sprintf("flaky_test_results_%s.json", t.Name())
+			t.Logf("Writing test results to %s", resultsFileName)
+			jsonResults, err := json.Marshal(testResults)
+			require.NoError(t, err)
+			err = os.WriteFile(resultsFileName, jsonResults, 0644) //nolint:gosec
+			require.NoError(t, err)
+		}
+	})
+
+	for _, result := range testResults {
+		t.Run(fmt.Sprintf("checking results of %s", result.TestName), func(t *testing.T) {
+			expected, ok := expectedResults[result.TestName]
+			// Sanity checks
+			require.True(t, ok, "unexpected test result: %s", result.TestName)
+			require.False(t, expected.seen, "test '%s' was seen multiple times", result.TestName)
+			expected.seen = true
+
+			assert.Equal(t, defaultRuns, result.Runs, "test '%s' had an unexpected number of runs", result.TestName)
+			assert.Len(t, result.Durations, result.Runs, "test '%s' has a mismatch of runs and duration counts", result.TestName, defaultRuns)
+			resultCounts := result.Successes + result.Failures + result.Panics + result.Skips
+			assert.Equal(t, result.Runs, resultCounts,
+				"test '%s' doesn't match Runs count with results counts\nRuns: %d\nSuccesses: %d\nFailures: %d\nPanics: %d\nSkips: %d\nTotal: %d",
+				result.TestName, result.Runs, result.Successes, result.Failures, result.Panics, result.Skips, resultCounts,
+			)
 			assert.Equal(t, expected.TestResult.Panicked, result.Panicked, "test '%s' had an unexpected panic result", result.TestName)
 			assert.Equal(t, expected.TestResult.Skipped, result.Skipped, "test '%s' had an unexpected skipped result", result.TestName)