flakeguard: Add support for go projects and add test duration to results (#1283)

* flakeguard: Add support for multi go projects

* Add project-path flag to runner

* Add test duration to test results

Author: lukaszcl

tools/flakeguard/cmd/find.go

@@ -30,7 +30,7 @@ var FindTestsCmd = &cobra.Command{
 		// Find all changes in test files and get their package names
 		var changedTestPkgs []string
 		if findByTestFilesDiff {
-			changedTestFiles, err := git.FindChangedFiles(baseRef, "grep '_test\\.go$'", excludes)
+			changedTestFiles, err := git.FindChangedFiles(projectPath, baseRef, "grep '_test\\.go$'")
 			if err != nil {
 				log.Fatalf("Error finding changed test files: %v", err)
 			}

tools/flakeguard/cmd/run.go

@@ -15,6 +15,7 @@ var RunTestsCmd = &cobra.Command{
 	Use:   "run",
 	Short: "Run tests to check if they are flaky",
 	Run: func(cmd *cobra.Command, args []string) {
+		projectPath, _ := cmd.Flags().GetString("project-path")
 		testPackagesJson, _ := cmd.Flags().GetString("test-packages-json")
 		testPackagesArg, _ := cmd.Flags().GetStringSlice("test-packages")
 		runCount, _ := cmd.Flags().GetInt("run-count")
@@ -34,10 +35,11 @@ var RunTestsCmd = &cobra.Command{
 		}
 
 		runner := runner.Runner{
-			Verbose:  true,
-			RunCount: runCount,
-			UseRace:  useRace,
-			FailFast: threshold == 1.0, // Fail test on first test run if threshold is 1.0
+			ProjectPath: projectPath,
+			Verbose:     true,
+			RunCount:    runCount,
+			UseRace:     useRace,
+			FailFast:    threshold == 1.0, // Fail test on first test run if threshold is 1.0
 		}
 
 		testResults, err := runner.RunTests(testPackages)
@@ -82,6 +84,7 @@ var RunTestsCmd = &cobra.Command{
 }
 
 func init() {
+	RunTestsCmd.Flags().StringP("project-path", "r", ".", "The path to the Go project. Default is the current directory. Useful for subprojects.")
 	RunTestsCmd.Flags().String("test-packages-json", "", "JSON-encoded string of test packages")
 	RunTestsCmd.Flags().StringSlice("test-packages", nil, "Comma-separated list of test packages to run")
 	RunTestsCmd.Flags().IntP("run-count", "c", 1, "Number of times to run the tests")

tools/flakeguard/git/git.go

@@ -3,6 +3,7 @@ package git
 import (
 	"bytes"
 	"fmt"
+	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
@@ -14,15 +15,15 @@ import (
 // FindChangedFiles executes a git diff against a specified base reference and pipes the output through a user-defined grep command or sequence.
 // The baseRef parameter specifies the base git reference for comparison (e.g., "main", "develop").
 // The filterCmd parameter should include the full command to be executed after git diff, such as "grep '_test.go$'" or "grep -v '_test.go$' | sort".
-func FindChangedFiles(baseRef, filterCmd string, excludePaths []string) ([]string, error) {
-	// Constructing the exclusion part of the git command
-	excludeStr := ""
-	for _, path := range excludePaths {
-		excludeStr += fmt.Sprintf("':(exclude)%s' ", path)
+func FindChangedFiles(rootGoModPath, baseRef, filterCmd string) ([]string, error) {
+	// Find directories containing a go.mod file and build an exclusion string
+	excludeStr, err := buildExcludeStringForGoModDirs(rootGoModPath)
+	if err != nil {
+		return nil, fmt.Errorf("error finding go.mod directories: %w", err)
 	}
 
 	// First command to list files changed between the baseRef and HEAD, excluding specified paths
-	diffCmdStr := fmt.Sprintf("git diff --name-only --diff-filter=AM %s...HEAD %s", baseRef, excludeStr)
+	diffCmdStr := fmt.Sprintf("git diff --name-only --diff-filter=AM %s...HEAD -- %s %s", baseRef, rootGoModPath, excludeStr)
 	diffCmd := exec.Command("bash", "-c", diffCmdStr)
 
 	// Using a buffer to capture stdout and a separate buffer for stderr
@@ -36,7 +37,7 @@ func FindChangedFiles(baseRef, filterCmd string, excludePaths []string) ([]strin
 		return nil, fmt.Errorf("error executing git diff command: %s; error: %w; stderr: %s", diffCmdStr, err, errBuf.String())
 	}
 
-	// Check if there are any files listed, if not, return an empty slice
+	// Check if there are any files listed; if not, return an empty slice
 	diffOutput := strings.TrimSpace(out.String())
 	if diffOutput == "" {
 		return []string{}, nil
@@ -74,6 +75,32 @@ func FindChangedFiles(baseRef, filterCmd string, excludePaths []string) ([]strin
 	return files, nil
 }
 
+// buildExcludeStringForGoModDirs searches the given root directory for subdirectories
+// containing a go.mod file and returns a formatted string to exclude those directories
+// (except the root directory if it contains a go.mod file) from git diff.
+func buildExcludeStringForGoModDirs(rootGoModPath string) (string, error) {
+	var excludeStr string
+
+	err := filepath.Walk(rootGoModPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.Name() == "go.mod" {
+			dir := filepath.Dir(path)
+			// Skip excluding the root directory if go.mod is found there
+			if dir != rootGoModPath {
+				excludeStr += fmt.Sprintf("':(exclude)%s/**' ", dir)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return "", err
+	}
+
+	return excludeStr, nil
+}
+
 func Diff(baseBranch string) (*utils.CmdOutput, error) {
 	return utils.ExecuteCmd("git", "diff", "--name-only", baseBranch)
 }

tools/flakeguard/reports/reports.go

@@ -4,9 +4,10 @@ type TestResult struct {
 	TestName    string
 	TestPackage string
 	PassRatio   float64
+	Skipped     bool // Indicates if the test was skipped
 	Runs        int
-	Outputs     []string // Stores outputs for a test
-	Skipped     bool     // Indicates if the test was skipped
+	Outputs     []string  // Stores outputs for a test
+	Durations   []float64 // Stores elapsed time in seconds for each run of the test
 }
 
 // FilterFailedTests returns a slice of TestResult where the pass ratio is below the specified threshold.

tools/flakeguard/runner/runner.go

@@ -14,10 +14,11 @@ import (
 )
 
 type Runner struct {
-	Verbose  bool // If true, provides detailed logging.
-	RunCount int  // Number of times to run the tests.
-	UseRace  bool // Enable race detector.
-	FailFast bool // Stop on first test failure.
+	ProjectPath string // Path to the Go project directory.
+	Verbose     bool   // If true, provides detailed logging.
+	RunCount    int    // Number of times to run the tests.
+	UseRace     bool   // Enable race detector.
+	FailFast    bool   // Stop on first test failure.
 }
 
 // RunTests executes the tests for each provided package and aggregates all results.
@@ -58,6 +59,7 @@ func (r *Runner) runTestPackage(testPackage string) ([]byte, bool, error) {
 		log.Printf("Running command: go %s\n", strings.Join(args, " "))
 	}
 	cmd := exec.Command("go", args...)
+	cmd.Dir = r.ProjectPath
 
 	// cmd.Env = append(cmd.Env, "GOFLAGS=-extldflags=-Wl,-ld_classic") // Ensure modules are enabled
 	var out bytes.Buffer
@@ -89,10 +91,11 @@ func parseTestResults(datas [][]byte) ([]reports.TestResult, error) {
 		scanner := bufio.NewScanner(bytes.NewReader(data))
 		for scanner.Scan() {
 			var entry struct {
-				Action  string `json:"Action"`
-				Test    string `json:"Test"`
-				Package string `json:"Package"`
-				Output  string `json:"Output"`
+				Action  string  `json:"Action"`
+				Test    string  `json:"Test"`
+				Package string  `json:"Package"`
+				Output  string  `json:"Output"`
+				Elapsed float64 `json:"Elapsed"`
 			}
 			if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil {
 				return nil, fmt.Errorf("failed to parse json test output: %s, err: %w", scanner.Text(), err)
@@ -120,13 +123,16 @@ func parseTestResults(datas [][]byte) ([]reports.TestResult, error) {
 				result.Runs++
 			case "pass":
 				result.PassRatio = (result.PassRatio*float64(result.Runs-1) + 1) / float64(result.Runs)
+				result.Durations = append(result.Durations, entry.Elapsed)
 			case "output":
 				result.Outputs = append(result.Outputs, entry.Output)
 			case "fail":
 				result.PassRatio = (result.PassRatio * float64(result.Runs-1)) / float64(result.Runs)
+				result.Durations = append(result.Durations, entry.Elapsed)
 			case "skip":
 				result.Skipped = true
 				result.Runs++
+				result.Durations = append(result.Durations, entry.Elapsed)
 			}
 		}