Add race tests

Author: kalverra

tools/flakeguard/runner/flaky_test_package/example_tests_test.go

@@ -2,6 +2,7 @@ package flakytestpackage
 
 import (
 	"os"
+	"sync"
 	"testing"
 )
 
@@ -49,3 +50,32 @@ func TestPanic(t *testing.T) {
 	t.Parallel()
 	panic("This test intentionally panics")
 }
+
+func TestRace(t *testing.T) {
+	t.Parallel()
+	t.Logf("This test should trigger a failure if run with the -race flag, but otherwise pass")
+
+	var sharedCounter int
+	var wg sync.WaitGroup
+
+	// Define a worker function that accesses sharedCounter without synchronization
+	worker := func() {
+		defer wg.Done()
+		for i := 0; i < 1000; i++ {
+			sharedCounter++
+		}
+	}
+
+	// Start multiple goroutines to introduce a race condition
+	const numGoroutines = 10
+	wg.Add(numGoroutines)
+	for i := 0; i < numGoroutines; i++ {
+		go worker()
+	}
+
+	// Wait for all goroutines to complete
+	wg.Wait()
+
+	// Log the result
+	t.Logf("Final value of sharedCounter: %d", sharedCounter)
+}

tools/flakeguard/runner/runner_test.go

@@ -67,6 +67,15 @@ func TestRunDefault(t *testing.T) {
 				PassRatio: 0,
 			},
 		},
+		"TestRace": {
+			TestResult: &reports.TestResult{
+				TestName:  "TestPass",
+				Panicked:  false,
+				Skipped:   false,
+				PassRatio: 1,
+				Successes: defaultRuns,
+			},
+		},
 	}
 
 	testResults, err := defaultTestRunner.RunTests()
@@ -161,6 +170,15 @@ func TestRunWithPanics(t *testing.T) {
 				Skipped:  false,
 			},
 		},
+		"TestRace": {
+			TestResult: &reports.TestResult{
+				TestName:  "TestPass",
+				Panicked:  false,
+				Skipped:   false,
+				PassRatio: 1,
+				Successes: defaultRuns,
+			},
+		},
 	}
 
 	testResults, err := panicTestRunner.RunTests()
@@ -218,9 +236,6 @@ func cleanup(t *testing.T, testResults []reports.TestResult, runner Runner) {
 			if err != nil {
 				t.Logf("error writing test results: %v", err)
 			}
-			if len(runner.RawOutputs()) == 0 {
-				t.Logf("WARN: no raw output to write")
-			}
 			for packageName, rawOutput := range runner.RawOutputs() {
 				saniPackageName := filepath.Base(packageName)
 				rawOutputFileName := fmt.Sprintf("debug_raw_output_%s_%s.json", t.Name(), saniPackageName)
@@ -233,3 +248,101 @@ func cleanup(t *testing.T, testResults []reports.TestResult, runner Runner) {
 		}
 	})
 }
+
+func TestRunWithRace(t *testing.T) {
+	raceTestRunner := Runner{
+		ProjectPath:          "./",
+		Verbose:              true,
+		RunCount:             defaultRuns,
+		UseRace:              true,
+		SkipTests:            []string{"TestPanic"},
+		FailFast:             false,
+		SelectedTestPackages: []string{"./flaky_test_package"},
+		CollectRawOutput:     true,
+	}
+
+	expectedResults := map[string]*expectedTestResult{
+		"TestFlaky": {
+			TestResult: &reports.TestResult{
+				TestName: "TestFlaky",
+				Panicked: false,
+				Skipped:  false,
+			},
+		},
+		"TestFail": {
+			TestResult: &reports.TestResult{
+				TestName:  "TestFail",
+				Panicked:  false,
+				Skipped:   false,
+				PassRatio: 0,
+				Failures:  defaultRuns,
+			},
+		},
+		"TestPass": {
+			TestResult: &reports.TestResult{
+				TestName:  "TestPass",
+				Panicked:  false,
+				Skipped:   false,
+				PassRatio: 1,
+				Successes: defaultRuns,
+			},
+		},
+		"TestSkipped": {
+			TestResult: &reports.TestResult{
+				TestName:  "TestSkipped",
+				Panicked:  false,
+				Skipped:   true,
+				PassRatio: 0,
+			},
+		},
+		"TestRace": {
+			TestResult: &reports.TestResult{
+				TestName:  "TestPass",
+				Panicked:  false,
+				Skipped:   false,
+				PassRatio: 0,
+				Failures:  defaultRuns,
+			},
+		},
+	}
+
+	testResults, err := raceTestRunner.RunTests()
+	require.NoError(t, err)
+	cleanup(t, testResults, raceTestRunner)
+
+	for _, result := range testResults {
+		t.Run(fmt.Sprintf("checking results of %s", result.TestName), func(t *testing.T) {
+			expected, ok := expectedResults[result.TestName]
+			// Sanity checks
+			require.True(t, ok, "unexpected test result: %s", result.TestName)
+			require.False(t, expected.seen, "test '%s' was seen multiple times", result.TestName)
+			expected.seen = true
+
+			assert.Equal(t, defaultRuns, result.Runs, "test '%s' had an unexpected number of runs", result.TestName)
+			assert.Len(t, result.Durations, result.Runs, "test '%s' has a mismatch of runs and duration counts", result.TestName, defaultRuns)
+			resultCounts := result.Successes + result.Failures + result.Panics + result.Skips
+			assert.Equal(t, result.Runs, resultCounts,
+				"test '%s' doesn't match Runs count with results counts\nRuns: %d\nSuccesses: %d\nFailures: %d\nPanics: %d\nSkips: %d\nTotal: %d",
+				result.TestName, result.Runs, result.Successes, result.Failures, result.Panics, result.Skips, resultCounts,
+			)
+			assert.Equal(t, expected.TestResult.Panicked, result.Panicked, "test '%s' had an unexpected panic result", result.TestName)
+			assert.Equal(t, expected.TestResult.Skipped, result.Skipped, "test '%s' had an unexpected skipped result", result.TestName)
+
+			if result.TestName == "TestFlaky" {
+				assert.Greater(t, result.Successes, 0, "flaky test '%s' should have passed some", result.TestName)
+				assert.Greater(t, result.Failures, 0, "flaky test '%s' should have failed some", result.TestName)
+				assert.Greater(t, result.PassRatio, float64(0), "flaky test '%s' should have a flaky pass ratio", result.TestName)
+				assert.Less(t, result.PassRatio, float64(1), "flaky test '%s' should have a flaky pass ratio", result.TestName)
+			} else {
+				assert.Equal(t, expected.TestResult.PassRatio, result.PassRatio, "test '%s' had an unexpected pass ratio", result.TestName)
+				assert.Equal(t, expected.TestResult.Successes, result.Successes, "test '%s' had an unexpected number of successes", result.TestName)
+				assert.Equal(t, expected.TestResult.Failures, result.Failures, "test '%s' had an unexpected number of failures", result.TestName)
+			}
+		})
+	}
+
+	for _, expected := range expectedResults {
+		assert.True(t, expected.seen, "expected test '%s' not found in test runs", expected.TestResult.TestName)
+	}
+
+}