Adds tests to runner implementation

Author: kalverra

tools/flakeguard/.gitignore

@@ -0,0 +1,2 @@
+tmp_test_flaky_state
+flaky_test_results.json

tools/flakeguard/Makefile

@@ -1,2 +1,9 @@
+.PHONY: test_unit
 test_unit:
-	go test -timeout 5m -json -cover -covermode=count -coverprofile=unit-test-coverage.out ./... 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci
+	go list ./... | grep -v 'flaky_test_package' | xargs go test -timeout 5m -json -cover -covermode=count -coverprofile=unit-test-coverage.out 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci
+
+.PHONY: test
+test:
+	go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
+	set -euo pipefail
+	go list ./... | grep -v 'flaky_test_package' | xargs go test -json -cover -coverprofile unit-test-coverage.out -v 2>&1 | tee /tmp/gotest.log | gotestfmt

tools/flakeguard/README.md

@@ -20,7 +20,8 @@ go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakegu
 
 ## Usage
 
-Flakeguard offers two main commands: 
+Flakeguard offers two main commands:
+
 - `find` identifies test packages affected by recent changes.
 - `run` executes tests multiple times to identify flaky tests
 

tools/flakeguard/cmd/run.go

@@ -80,7 +80,7 @@ var RunTestsCmd = &cobra.Command{
 			if err != nil {
 				log.Fatalf("Error marshaling test results to JSON: %v", err)
 			}
-			if err := os.WriteFile(outputPath, jsonData, 0644); err != nil {
+			if err := os.WriteFile(outputPath, jsonData, 0644); err != nil { //nolint:gosec
 				log.Fatalf("Error writing test results to file: %v", err)
 			}
 			fmt.Printf("All test results saved to %s\n", outputPath)

tools/flakeguard/reports/reports.go

@@ -11,15 +11,20 @@ import (
 	"strings"
 )
 
+// TestResult represents the result of a single test being run through flakeguard.
 type TestResult struct {
 	TestName            string
 	TestPackage         string
-	Panicked            bool    // Indicates a test-level panic
-	PackagePanicked     bool    // Indicates a package-level panic
-	PassRatio           float64 // Pass ratio in decimal format like 0.5
-	PassRatioPercentage string  // Pass ratio in percentage format like "50%"
-	Skipped             bool    // Indicates if the test was skipped
-	Runs                int
+	Panicked            bool      // Indicates a test-level panic
+	PackagePanicked     bool      // Indicates a package-level panic
+	PassRatio           float64   // Pass ratio in decimal format like 0.5
+	PassRatioPercentage string    // Pass ratio in percentage format like "50%"
+	Skipped             bool      // Indicates if the test was skipped
+	Runs                int       // Count of how many times the test was run
+	Failures            int       // Count of how many times the test failed
+	Successes           int       // Count of how many times the test passed
+	Panics              int       // Count of how many times the test panicked
+	Skips               int       // Count of how many times the test was skipped
 	Outputs             []string  // Stores outputs for a test
 	Durations           []float64 // Stores elapsed time in seconds for each run of the test
 	PackageOutputs      []string  // Stores package-level outputs
@@ -205,7 +210,7 @@ func saveResults(filePath string, results []TestResult) error {
 	if err != nil {
 		return fmt.Errorf("error marshaling results: %v", err)
 	}
-	return os.WriteFile(filePath, data, 0644)
+	return os.WriteFile(filePath, data, 0644) //nolint:gosec
 }
 
 // Helper function to save test names, packages, and outputs to JSON file
@@ -233,5 +238,5 @@ func saveTestOutputs(filePath string, results []TestResult) error {
 	if err != nil {
 		return fmt.Errorf("error marshaling outputs: %v", err)
 	}
-	return os.WriteFile(filePath, data, 0644)
+	return os.WriteFile(filePath, data, 0644) //nolint:gosec
 }

tools/flakeguard/reports/reports_test.go

@@ -132,7 +132,7 @@ func writeTempJSONFile(t *testing.T, dir string, filename string, data interface
 	if err != nil {
 		t.Fatalf("Failed to marshal JSON: %v", err)
 	}
-	if err := os.WriteFile(filePath, fileData, 0644); err != nil {
+	if err := os.WriteFile(filePath, fileData, 0644); err != nil { //nolint:gosec
 		t.Fatalf("Failed to write JSON file: %v", err)
 	}
 	return filePath

tools/flakeguard/runner/flaky_test_package/flaky_test.go

@@ -0,0 +1,51 @@
+package flakytestpackage
+
+import (
+	"os"
+	"testing"
+)
+
+func TestPass(t *testing.T) {
+	t.Parallel()
+	t.Log("This test always passes")
+}
+
+func TestFail(t *testing.T) {
+	t.Parallel()
+	t.Fatal("This test always fails")
+}
+
+// This test should have a 50% pass ratio
+func TestFlaky(t *testing.T) {
+	t.Parallel()
+
+	// Track if the test has run before
+	stateFile := "tmp_test_flaky_state"
+
+	// If the state file does not exist, create it and fail the test
+	if _, err := os.Stat(stateFile); os.IsNotExist(err) {
+		if err := os.WriteFile(stateFile, []byte("run once"), 0644); err != nil { //nolint:gosec
+			t.Fatalf("THIS IS UNEXPECTED: failed to create state file: %v", err)
+		}
+		t.Fatalf("This is a designed flaky test working as intended")
+	} else {
+		t.Cleanup(func() {
+			err := os.Remove(stateFile)
+			if err != nil {
+				t.Fatalf("THIS IS UNEXPECTED: failed to remove state file: %v", err)
+			}
+		})
+	}
+
+	t.Log("This test passes after the first run")
+}
+
+func TestSkipped(t *testing.T) {
+	t.Parallel()
+	t.Skip("This test is intentionally skipped")
+}
+
+// func TestPanic(t *testing.T) {
+// 	t.Parallel()
+// 	panic("This test intentionally panics")
+// }

tools/flakeguard/runner/runner.go

@@ -54,16 +54,14 @@ type exitCoder interface {
 
 // runTests runs the tests for a given package and returns the path to the output file.
 func (r *Runner) runTests(packageName string) (string, bool, error) {
-	args := []string{"test", packageName, "-json", "-count=1"} // Enable JSON output
+	args := []string{"test", packageName, "-json", "-count=1"}
 	if r.UseRace {
 		args = append(args, "-race")
 	}
 	if len(r.SkipTests) > 0 {
 		skipPattern := strings.Join(r.SkipTests, "|")
 		args = append(args, fmt.Sprintf("-skip=%s", skipPattern))
 	}
-	// The last arg redirects stderr to null
-	args = append(args, "2>/dev/null")
 
 	if r.Verbose {
 		log.Printf("Running command: go %s\n", strings.Join(args, " "))
@@ -105,8 +103,6 @@ func parseTestResults(filePaths []string) ([]reports.TestResult, error) {
 		if err != nil {
 			return nil, fmt.Errorf("failed to open test output file: %w", err)
 		}
-		defer os.Remove(filePath) // Clean up file after parsing
-		defer file.Close()
 
 		scanner := bufio.NewScanner(file)
 		var precedingLines []string // Store preceding lines for context
@@ -149,7 +145,6 @@ func parseTestResults(filePaths []string) ([]reports.TestResult, error) {
 					testDetails[key] = &reports.TestResult{
 						TestName:       entry.Test,
 						TestPackage:    entry.Package,
-						Runs:           0,
 						PassRatio:      0,
 						Outputs:        []string{},
 						PackageOutputs: []string{},
@@ -181,25 +176,22 @@ func parseTestResults(filePaths []string) ([]reports.TestResult, error) {
 				}
 			case "pass":
 				if entry.Test != "" {
-					result.PassRatio = (result.PassRatio*float64(result.Runs-1) + 1) / float64(result.Runs)
-					result.PassRatioPercentage = fmt.Sprintf("%.0f%%", result.PassRatio*100)
 					result.Durations = append(result.Durations, entry.Elapsed)
+					result.Successes++
 				}
 			case "fail":
 				if entry.Test != "" {
-					result.PassRatio = (result.PassRatio * float64(result.Runs-1)) / float64(result.Runs)
-					result.PassRatioPercentage = fmt.Sprintf("%.0f%%", result.PassRatio*100)
 					result.Durations = append(result.Durations, entry.Elapsed)
+					result.Failures++
 				}
 			case "output":
 				// Output already handled above
 				if panicRe.MatchString(entry.Output) {
 					if entry.Test != "" {
 						// Test-level panic
 						result.Panicked = true
-						result.PassRatio = (result.PassRatio * float64(result.Runs-1)) / float64(result.Runs)
-						result.PassRatioPercentage = fmt.Sprintf("%.0f%%", result.PassRatio*100)
 						result.Durations = append(result.Durations, entry.Elapsed)
+						result.Panics++
 					} else {
 						// Package-level panic
 						// Mark PackagePanicked for all TestResults in the package
@@ -213,15 +205,26 @@ func parseTestResults(filePaths []string) ([]reports.TestResult, error) {
 			case "skip":
 				if entry.Test != "" {
 					result.Skipped = true
-					result.Runs++
+					result.Skips++
 					result.Durations = append(result.Durations, entry.Elapsed)
 				}
 			}
+			if entry.Test != "" {
+				result.PassRatio = float64(result.Successes) / float64(result.Runs)
+				result.PassRatioPercentage = fmt.Sprintf("%.0f%%", result.PassRatio*100)
+			}
 		}
 
 		if err := scanner.Err(); err != nil {
 			return nil, fmt.Errorf("reading test output file: %w", err)
 		}
+		// Clean up file after parsing
+		if err = file.Close(); err != nil {
+			log.Printf("WARN: failed to close file: %v", err)
+		}
+		if err = os.Remove(filePath); err != nil {
+			log.Printf("WARN: failed to delete file: %v", err)
+		}
 	}
 
 	var results []reports.TestResult

tools/flakeguard/runner/runner_test.go

@@ -0,0 +1,129 @@
+package runner
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type MockGoRunner struct {
+	RunFunc func(dir string, args []string) (string, bool, error)
+}
+
+func (m MockGoRunner) RunCommand(dir string, args []string) (string, bool, error) {
+	return m.RunFunc(dir, args)
+}
+
+func TestRun(t *testing.T) {
+	t.Parallel()
+
+	runs := 5
+
+	runner := Runner{
+		ProjectPath:          "./",
+		Verbose:              true,
+		RunCount:             runs,
+		UseRace:              false,
+		SkipTests:            []string{},
+		FailFast:             false,
+		SelectedTestPackages: []string{"./flaky_test_package"},
+	}
+
+	expectedResults := map[string]*struct {
+		*reports.TestResult
+		seen bool
+	}{
+		"TestFlaky": {
+			TestResult: &reports.TestResult{
+				TestName: "TestFlaky",
+				Panicked: false,
+				Skipped:  false,
+			},
+		},
+		"TestFail": {
+			TestResult: &reports.TestResult{
+				TestName:  "TestFail",
+				Panicked:  false,
+				Skipped:   false,
+				PassRatio: 0,
+				Failures:  runs,
+			},
+		},
+		"TestPass": {
+			TestResult: &reports.TestResult{
+				TestName:  "TestPass",
+				Panicked:  false,
+				Skipped:   false,
+				PassRatio: 1,
+				Successes: runs,
+			},
+		},
+		// "TestPanic": {
+		// 	TestResult: &reports.TestResult{
+		// 		TestName:  "TestPanic",
+		// 		Panicked:  true,
+		// 		Skipped:   false,
+		// 		PassRatio: 0,
+		// 	},
+		// },
+		"TestSkipped": {
+			TestResult: &reports.TestResult{
+				TestName:  "TestSkipped",
+				Panicked:  false,
+				Skipped:   true,
+				PassRatio: 0,
+			},
+		},
+	}
+
+	testResults, err := runner.RunTests()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		if t.Failed() {
+			t.Log("Writing test results to flaky_test_results.json")
+			jsonResults, err := json.Marshal(testResults)
+			require.NoError(t, err)
+			err = os.WriteFile("flaky_test_results.json", jsonResults, 0644) //nolint:gosec
+			require.NoError(t, err)
+		}
+	})
+	for _, result := range testResults {
+		t.Run(fmt.Sprintf("checking results of %s", result.TestName), func(t *testing.T) {
+			expected, ok := expectedResults[result.TestName]
+			// Sanity checks
+			require.True(t, ok, "unexpected test result: %s", result.TestName)
+			require.False(t, expected.seen, "test '%s' was seen multiple times", result.TestName)
+			expected.seen = true
+
+			assert.Equal(t, runs, result.Runs, "test '%s' had an unexpected number of runs", result.TestName)
+			assert.Len(t, result.Durations, runs, "test '%s' had an unexpected number of durations as it was run %d times", result.TestName, runs)
+			if result.TestName == "TestSlow" {
+				for _, duration := range result.Durations {
+					assert.GreaterOrEqual(t, duration, float64(1), "slow test '%s' should have a duration of at least 2s", result.TestName)
+				}
+			}
+			assert.Equal(t, expected.TestResult.Panicked, result.Panicked, "test '%s' had an unexpected panic result", result.TestName)
+			assert.Equal(t, expected.TestResult.Skipped, result.Skipped, "test '%s' had an unexpected skipped result", result.TestName)
+
+			if result.TestName == "TestFlaky" {
+				assert.Greater(t, result.Successes, 0, "flaky test '%s' should have passed some", result.TestName)
+				assert.Greater(t, result.Failures, 0, "flaky test '%s' should have failed some", result.TestName)
+				assert.Greater(t, result.PassRatio, float64(0), "flaky test '%s' should have a flaky pass ratio", result.TestName)
+				assert.Less(t, result.PassRatio, float64(1), "flaky test '%s' should have a flaky pass ratio", result.TestName)
+			} else {
+				assert.Equal(t, expected.TestResult.PassRatio, result.PassRatio, "test '%s' had an unexpected pass ratio", result.TestName)
+				assert.Equal(t, expected.TestResult.Successes, result.Successes, "test '%s' had an unexpected number of successes", result.TestName)
+				assert.Equal(t, expected.TestResult.Failures, result.Failures, "test '%s' had an unexpected number of failures", result.TestName)
+			}
+		})
+	}
+
+	for _, expected := range expectedResults {
+		assert.True(t, expected.seen, "expected test '%s' not found in test runs", expected.TestResult.TestName)
+	}
+}