Allow to run all test packages and check dependencies

Author: lukaszcl

tools/flakeguard/cmd/run.go

@@ -1,10 +1,12 @@
 package cmd
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"log"
 	"os"
+	"os/exec"
 
 	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
 	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/runner"
@@ -18,12 +20,18 @@ var RunTestsCmd = &cobra.Command{
 		projectPath, _ := cmd.Flags().GetString("project-path")
 		testPackagesJson, _ := cmd.Flags().GetString("test-packages-json")
 		testPackagesArg, _ := cmd.Flags().GetStringSlice("test-packages")
+		runAllPackages, _ := cmd.Flags().GetBool("run-all-packages")
 		runCount, _ := cmd.Flags().GetInt("run-count")
 		useRace, _ := cmd.Flags().GetBool("race")
 		outputPath, _ := cmd.Flags().GetString("output-json")
 		threshold, _ := cmd.Flags().GetFloat64("threshold")
 		skipTests, _ := cmd.Flags().GetStringSlice("skip-tests")
 
+		// Check if project dependencies are correctly set up
+		if err := checkDependencies(projectPath); err != nil {
+			log.Fatalf("Error: %v", err)
+		}
+
 		var testPackages []string
 		if testPackagesJson != "" {
 			if err := json.Unmarshal([]byte(testPackagesJson), &testPackages); err != nil {
@@ -36,15 +44,17 @@ var RunTestsCmd = &cobra.Command{
 		}
 
 		runner := runner.Runner{
-			ProjectPath: projectPath,
-			Verbose:     true,
-			RunCount:    runCount,
-			UseRace:     useRace,
-			FailFast:    threshold == 1.0, // Fail test on first test run if threshold is 1.0
-			SkipTests:   skipTests,
+			ProjectPath:          projectPath,
+			Verbose:              true,
+			RunCount:             runCount,
+			UseRace:              useRace,
+			FailFast:             threshold == 1.0, // Fail test on first test run if threshold is 1.0
+			SkipTests:            skipTests,
+			RunAllTestPackages:   runAllPackages,
+			SelectedTestPackages: testPackages,
 		}
 
-		testResults, err := runner.RunTests(testPackages)
+		testResults, err := runner.RunTests()
 		if err != nil {
 			fmt.Printf("Error running tests: %v\n", err)
 			os.Exit(1)
@@ -87,10 +97,26 @@ func init() {
 	RunTestsCmd.Flags().StringP("project-path", "r", ".", "The path to the Go project. Default is the current directory. Useful for subprojects")
 	RunTestsCmd.Flags().String("test-packages-json", "", "JSON-encoded string of test packages")
 	RunTestsCmd.Flags().StringSlice("test-packages", nil, "Comma-separated list of test packages to run")
+	RunTestsCmd.Flags().Bool("run-all-packages", false, "Run all test packages in the project. This flag overrides --test-packages and --test-packages-json")
 	RunTestsCmd.Flags().IntP("run-count", "c", 1, "Number of times to run the tests")
 	RunTestsCmd.Flags().Bool("race", false, "Enable the race detector")
 	RunTestsCmd.Flags().Bool("fail-fast", false, "Stop on the first test failure")
 	RunTestsCmd.Flags().String("output-json", "", "Path to output the test results in JSON format")
 	RunTestsCmd.Flags().Float64("threshold", 0.8, "Threshold for considering a test as flaky")
 	RunTestsCmd.Flags().StringSlice("skip-tests", nil, "Comma-separated list of test names to skip from running")
 }
+
+func checkDependencies(projectPath string) error {
+	cmd := exec.Command("go", "mod", "tidy")
+	cmd.Dir = projectPath
+
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = &out
+
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("dependency check failed: %v\n%s\nPlease run 'go mod tidy' to fix missing or unused dependencies", err, out.String())
+	}
+
+	return nil
+}

tools/flakeguard/runner/runner.go

@@ -14,22 +14,28 @@ import (
 )
 
 type Runner struct {
-	ProjectPath string   // Path to the Go project directory.
-	Verbose     bool     // If true, provides detailed logging.
-	RunCount    int      // Number of times to run the tests.
-	UseRace     bool     // Enable race detector.
-	FailFast    bool     // Stop on first test failure.
-	SkipTests   []string // Test names to exclude.
+	ProjectPath          string   // Path to the Go project directory.
+	Verbose              bool     // If true, provides detailed logging.
+	RunCount             int      // Number of times to run the tests.
+	UseRace              bool     // Enable race detector.
+	FailFast             bool     // Stop on first test failure.
+	SkipTests            []string // Test names to exclude.
+	RunAllTestPackages   bool     // Run all test packages if true.
+	SelectedTestPackages []string // Explicitly selected packages to run.
 }
 
 // RunTests executes the tests for each provided package and aggregates all results.
 // It returns all test results and any error encountered during testing.
-func (r *Runner) RunTests(packages []string) ([]reports.TestResult, error) {
+func (r *Runner) RunTests() ([]reports.TestResult, error) {
 	var jsonOutputs [][]byte
+	packages := r.SelectedTestPackages
+	if r.RunAllTestPackages {
+		packages = []string{"./..."}
+	}
 
 	for _, p := range packages {
 		for i := 0; i < r.RunCount; i++ {
-			jsonOutput, passed, err := r.runTestPackage(p)
+			jsonOutput, passed, err := r.runTests(p)
 			if err != nil {
 				return nil, fmt.Errorf("failed to run tests in package %s: %w", p, err)
 			}
@@ -49,20 +55,16 @@ type exitCoder interface {
 
 // runTestPackage executes the test command for a single test package.
 // It returns the command output, a boolean indicating success, and any error encountered.
-func (r *Runner) runTestPackage(testPackage string) ([]byte, bool, error) {
-	args := []string{"test", "-json", "-count=1"} // Enable JSON output
+func (r *Runner) runTests(packageName string) ([]byte, bool, error) {
+	args := []string{"test", packageName, "-json", "-count=1"} // Enable JSON output for parsing
 	if r.UseRace {
 		args = append(args, "-race")
 	}
-
-	// Construct regex pattern from ExcludedTests slice
 	if len(r.SkipTests) > 0 {
 		skipPattern := strings.Join(r.SkipTests, "|")
 		args = append(args, fmt.Sprintf("-skip=%s", skipPattern))
 	}
 
-	args = append(args, testPackage)
-
 	if r.Verbose {
 		log.Printf("Running command: go %s\n", strings.Join(args, " "))
 	}
@@ -79,7 +81,7 @@ func (r *Runner) runTestPackage(testPackage string) ([]byte, bool, error) {
 		var exErr exitCoder
 		// Check if the error is due to a non-zero exit code
 		if errors.As(err, &exErr) && exErr.ExitCode() == 0 {
-			return nil, false, fmt.Errorf("test command failed at %s: %w", testPackage, err)
+			return nil, false, fmt.Errorf("test command failed at %s: %w", packageName, err)
 		}
 		return out.Bytes(), false, nil // Test failed
 	}