Add flakeguard - a tool for finding flaky tests (#1255)

Author: lukaszcl

tools/flakeguard/cmd/find.go

@@ -0,0 +1,170 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/git"
+	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/golang"
+	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/utils"
+	"github.com/spf13/cobra"
+)
+
+var FindTestsCmd = &cobra.Command{
+	Use:   "find",
+	Long:  "Analyzes Golang project repository for changed files against a specified base reference and determines the test packages that are potentially impacted",
+	Short: "Find test packages that may be affected by changes",
+	Run: func(cmd *cobra.Command, args []string) {
+		projectPath, _ := cmd.Flags().GetString("project-path")
+		verbose, _ := cmd.Flags().GetBool("verbose")
+		jsonOutput, _ := cmd.Flags().GetBool("json")
+		filterEmptyTests, _ := cmd.Flags().GetBool("filter-empty-tests")
+		baseRef, _ := cmd.Flags().GetString("base-ref")
+		excludes, _ := cmd.Flags().GetStringSlice("excludes")
+		levels, _ := cmd.Flags().GetInt("levels")
+		findByTestFilesDiff, _ := cmd.Flags().GetBool("find-by-test-files-diff")
+		findByAffected, _ := cmd.Flags().GetBool("find-by-affected-packages")
+		onlyShowChangedTestFiles, _ := cmd.Flags().GetBool("only-show-changed-test-files")
+
+		// Find all changes in test files and get their package names
+		var changedTestPkgs []string
+		if findByTestFilesDiff {
+			changedTestFiles, err := git.FindChangedFiles(baseRef, "grep '_test\\.go$'", excludes)
+			if err != nil {
+				log.Fatalf("Error finding changed test files: %v", err)
+			}
+			if onlyShowChangedTestFiles {
+				outputResults(changedTestFiles, jsonOutput)
+				return
+			}
+			if verbose {
+				fmt.Println("Changed test files:", changedTestFiles)
+			}
+			changedTestPkgs, err = golang.GetFilePackages(changedTestFiles)
+			if err != nil {
+				log.Fatalf("Error getting package names for test files: %v", err)
+			}
+		}
+
+		// Find all affected test packages
+		var affectedTestPkgs []string
+		if findByAffected {
+			if verbose {
+				fmt.Println("Finding affected packages...")
+			}
+			affectedTestPkgs = findAffectedPackages(baseRef, projectPath, excludes, levels)
+		}
+
+		// Combine and deduplicate test package names
+		testPkgs := append(changedTestPkgs, affectedTestPkgs...)
+		testPkgs = utils.Deduplicate(testPkgs)
+
+		// Filter out packages that do not have tests
+		if filterEmptyTests {
+			if verbose {
+				fmt.Println("Filtering packages without tests...")
+			}
+			testPkgs = golang.FilterPackagesWithTests(testPkgs)
+		}
+
+		outputResults(testPkgs, jsonOutput)
+	},
+}
+
+func init() {
+	FindTestsCmd.Flags().StringP("project-path", "r", ".", "The path to the Go project. Default is the current directory. Useful for subprojects.")
+	FindTestsCmd.Flags().String("base-ref", "", "Git base reference (branch, tag, commit) for comparing changes. Required.")
+	FindTestsCmd.Flags().BoolP("verbose", "v", false, "Enable verbose mode")
+	FindTestsCmd.Flags().Bool("json", false, "Output the results in JSON format")
+	FindTestsCmd.Flags().Bool("filter-empty-tests", false, "Filter out test packages with no actual test functions. Can be very slow for large projects.")
+	FindTestsCmd.Flags().StringSlice("excludes", []string{}, "List of paths to exclude. Useful for repositories with multiple Go projects within.")
+	FindTestsCmd.Flags().IntP("levels", "l", 2, "The number of levels of recursion to search for affected packages. Default is 2. 0 is unlimited.")
+	FindTestsCmd.Flags().Bool("find-by-test-files-diff", true, "Enable the mode to find test packages by changes in test files.")
+	FindTestsCmd.Flags().Bool("find-by-affected-packages", true, "Enable the mode to find test packages that may be affected by changes in any of the project packages.")
+	FindTestsCmd.Flags().Bool("only-show-changed-test-files", false, "Only show the changed test files and exit")
+
+	if err := FindTestsCmd.MarkFlagRequired("base-ref"); err != nil {
+		fmt.Println("Error marking base-ref as required:", err)
+	}
+}
+
+func findAffectedPackages(baseRef, projectPath string, excludes []string, levels int) []string {
+	goList, err := golang.GoList()
+	if err != nil {
+		log.Fatalf("Error getting go list: %v\nStdErr: %s", err, goList.Stderr.String())
+	}
+	gitDiff, err := git.Diff(baseRef)
+	if err != nil {
+		log.Fatalf("Error getting the git diff: %v\nStdErr: %s", err, gitDiff.Stderr.String())
+	}
+	gitModDiff, err := git.ModDiff(baseRef, projectPath)
+	if err != nil {
+		log.Fatalf("Error getting the git mod diff: %v\nStdErr: %s", err, gitModDiff.Stderr.String())
+	}
+
+	packages, err := golang.ParsePackages(goList.Stdout)
+	if err != nil {
+		log.Fatalf("Error parsing packages: %v", err)
+	}
+
+	fileMap := golang.GetGoFileMap(packages, true)
+
+	var changedPackages []string
+	changedPackages, err = git.GetChangedGoPackagesFromDiff(gitDiff.Stdout, projectPath, excludes, fileMap)
+	if err != nil {
+		log.Fatalf("Error getting changed packages: %v", err)
+	}
+
+	changedModPackages, err := git.GetGoModChangesFromDiff(gitModDiff.Stdout)
+	if err != nil {
+		log.Fatalf("Error getting go.mod changes: %v", err)
+	}
+
+	depMap := golang.GetGoDepMap(packages)
+
+	// Find affected packages
+	// use map to make handling duplicates simpler
+	affectedPkgs := map[string]bool{}
+
+	// loop through packages changed via file changes
+	for _, pkg := range changedPackages {
+		p := golang.FindAffectedPackages(pkg, depMap, false, levels)
+		for _, p := range p {
+			affectedPkgs[p] = true
+		}
+	}
+
+	// loop through packages changed via go.mod changes
+	for _, pkg := range changedModPackages {
+		p := golang.FindAffectedPackages(pkg, depMap, true, levels)
+		for _, p := range p {
+			affectedPkgs[p] = true
+		}
+	}
+
+	// convert map to array
+	pkgs := []string{}
+	for k := range affectedPkgs {
+		pkgs = append(pkgs, k)
+	}
+
+	return pkgs
+}
+
+func outputResults(packages []string, jsonOutput bool) {
+	if jsonOutput {
+		if packages == nil {
+			packages = make([]string, 0) // Ensure the slice is initialized to an empty array
+		}
+		data, err := json.Marshal(packages)
+		if err != nil {
+			log.Fatalf("Error marshaling test files to JSON: %v", err)
+		}
+		fmt.Println(string(data))
+	} else {
+		for _, pkg := range packages {
+			fmt.Print(pkg, " ")
+		}
+	}
+}

tools/flakeguard/cmd/run.go

@@ -0,0 +1,90 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/reports"
+	"github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard/runner"
+	"github.com/spf13/cobra"
+)
+
+var RunTestsCmd = &cobra.Command{
+	Use:   "run",
+	Short: "Run tests to check if they are flaky",
+	Run: func(cmd *cobra.Command, args []string) {
+		testPackagesJson, _ := cmd.Flags().GetString("test-packages-json")
+		testPackagesArg, _ := cmd.Flags().GetStringSlice("test-packages")
+		runCount, _ := cmd.Flags().GetInt("run-count")
+		useRace, _ := cmd.Flags().GetBool("race")
+		failFast, _ := cmd.Flags().GetBool("fail-fast")
+		outputPath, _ := cmd.Flags().GetString("output-json")
+		threshold, _ := cmd.Flags().GetFloat64("threshold")
+
+		var testPackages []string
+		if testPackagesJson != "" {
+			if err := json.Unmarshal([]byte(testPackagesJson), &testPackages); err != nil {
+				log.Fatalf("Error decoding test packages JSON: %v", err)
+			}
+		} else if len(testPackagesArg) > 0 {
+			testPackages = testPackagesArg
+		} else {
+			log.Fatalf("Error: must specify either --test-packages-json or --test-packages")
+		}
+
+		runner := runner.Runner{
+			Verbose:  true,
+			RunCount: runCount,
+			UseRace:  useRace,
+			FailFast: failFast,
+		}
+
+		testResults, err := runner.RunTests(testPackages)
+		if err != nil {
+			fmt.Printf("Error running tests: %v\n", err)
+			os.Exit(1)
+		}
+
+		// Filter out failed tests based on the threshold
+		failedTests := reports.FilterFailedTests(testResults, threshold)
+		if len(failedTests) > 0 {
+			jsonData, err := json.MarshalIndent(failedTests, "", "  ")
+			if err != nil {
+				log.Fatalf("Error marshaling test results to JSON: %v", err)
+			}
+			fmt.Printf("Threshold for flaky tests: %.2f\n%d failed tests:\n%s\n", threshold, len(failedTests), string(jsonData))
+		}
+
+		// Save the test results in JSON format
+		if outputPath != "" && len(testResults) > 0 {
+			jsonData, err := json.MarshalIndent(testResults, "", "  ")
+			if err != nil {
+				log.Fatalf("Error marshaling test results to JSON: %v", err)
+			}
+			if err := os.WriteFile(outputPath, jsonData, 0644); err != nil {
+				log.Fatalf("Error writing test results to file: %v", err)
+			}
+			fmt.Printf("All test results saved to %s\n", outputPath)
+		}
+
+		if len(failedTests) > 0 {
+			os.Exit(1)
+		} else if len(testResults) == 0 {
+			fmt.Printf("No tests were run for the specified packages.\n")
+		} else {
+			fmt.Printf("All %d tests passed.\n", len(testResults))
+		}
+	},
+}
+
+func init() {
+	RunTestsCmd.Flags().String("test-packages-json", "", "JSON-encoded string of test packages")
+	RunTestsCmd.Flags().StringSlice("test-packages", nil, "Comma-separated list of test packages to run")
+	RunTestsCmd.Flags().IntP("run-count", "c", 1, "Number of times to run the tests")
+	RunTestsCmd.Flags().Bool("race", false, "Enable the race detector")
+	RunTestsCmd.Flags().Bool("fail-fast", false, "Stop on the first test failure")
+	RunTestsCmd.Flags().String("output-json", "", "Path to output the test results in JSON format")
+	RunTestsCmd.Flags().Float64("threshold", 0.8, "Threshold for considering a test as flaky")
+}