adding tests

Author: blampe

README.md

@@ -17,6 +17,20 @@ jobs:
     - name: Run tests
       run: go test $(cat tests)
 ```
+
+## Why?
+
+Strategies like `go test ./... -list .` compile your code in order to discover test cases.
+This can be especially slow in CI environments depending on the state of your build cache.
+
+We can discover test cases significantly faster by essentially `grep`-ing for patterns like `Test..`, `Fuzz...`, etc.
+
+## Caveats
+
+* A test can potentially be executed more than once if another package shares a test with the same name.
+  Renaming your tests to be globally unique is currently the best workaround if you want to guarantee a single execution per test function.
+  You can discover test with name collisions by running `shard --total 1 --index 0`.
+* Benchmarks aren't currently collected so running with `-bench` will not have any effect.
 
 
 

internal/collect.go

@@ -0,0 +1,144 @@
+package internal
+
+import (
+	"fmt"
+	"go/ast"
+	"go/doc"
+	"go/parser"
+	"go/token"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+type testf struct {
+	Path string
+	Name string
+}
+
+func Collect(root string) ([]testf, error) {
+	tests := []testf{}
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Don't collect testdata directories.
+		if info.IsDir() && filepath.Base(info.Name()) == "testdata" {
+			return filepath.SkipDir
+		}
+
+		if info.IsDir() || !strings.HasSuffix(path, "_test.go") {
+			return nil
+		}
+
+		// Parse the file to find test functions
+		file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.ParseComments)
+		if err != nil {
+			return err
+		}
+		for _, decl := range file.Decls {
+			fn, ok := decl.(*ast.FuncDecl)
+			if !ok {
+				continue
+			}
+			if !isTestFunc(fn) {
+				continue
+			}
+			tests = append(tests, testf{Path: filepath.Dir(path), Name: fn.Name.Name})
+		}
+
+		examples := doc.Examples(file)
+		for _, ex := range examples {
+			if ex.Output == "" && !ex.EmptyOutput {
+				// Don't run tests with empty output.
+				continue
+			}
+			tests = append(tests, testf{Path: filepath.Dir(path), Name: "Example" + ex.Name})
+
+		}
+
+		return nil
+	})
+	return tests, err
+}
+
+func Assign(tests []testf, index int, total int, seed int64) (names, paths []string) {
+	// Shuffle the tests.
+	if seed != 0 {
+		random := rand.New(rand.NewSource(seed))
+		for i := range tests {
+			j := random.Intn(i + 1) //nolint:gosec // Not cryptographic.
+			tests[i], tests[j] = tests[j], tests[i]
+		}
+	}
+
+	// Assign tests to our shard.
+	for idx, test := range tests {
+		if idx%total != index {
+			continue
+		}
+		paths = append(paths, "./"+test.Path)
+		names = append(names, test.Name)
+	}
+
+	// De-dupe.
+	slices.Sort(names)
+	slices.Sort(paths)
+
+	names = slices.CompactFunc(names, func(l, r string) bool {
+		if l == r {
+			fmt.Fprintln(os.Stderr, fmt.Sprintf("warning: %q exists in multiple packages, consider renaming it", l))
+		}
+		return l == r
+	})
+	paths = slices.Compact(paths)
+
+	return names, paths
+}
+
+// isTestFunc tells whether fn has the type of a testing function. arg
+// specifies the parameter type we look for: B, F, M or T.
+func isTestFunc(fn *ast.FuncDecl) bool {
+	if fn.Type.Results != nil && len(fn.Type.Results.List) > 0 ||
+		fn.Type.Params.List == nil ||
+		len(fn.Type.Params.List) != 1 ||
+		len(fn.Type.Params.List[0].Names) > 1 {
+		return false
+	}
+	ptr, ok := fn.Type.Params.List[0].Type.(*ast.StarExpr)
+	if !ok {
+		return false
+	}
+
+	fnName := fn.Name.Name
+
+	if !(strings.HasPrefix(fnName, "Test") || strings.HasPrefix(fnName, "Fuzz")) {
+		return false
+	}
+
+	if len(fnName) > 4 {
+		rune, _ := utf8.DecodeRuneInString(fnName[4:])
+		if unicode.IsLower(rune) {
+			return false
+		}
+	}
+
+	for _, want := range []string{"T", "F"} {
+		// We can't easily check that the type is *testing.M
+		// because we don't know how testing has been imported,
+		// but at least check that it's *M or *something.M.
+		// Same applies for B, F and T.
+		if name, ok := ptr.X.(*ast.Ident); ok && name.Name == want {
+			return true
+		}
+		if sel, ok := ptr.X.(*ast.SelectorExpr); ok && sel.Sel.Name == want {
+			return true
+		}
+	}
+	return false
+}

internal/collect_test.go

@@ -0,0 +1,142 @@
+package internal
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+func TestCollect(t *testing.T) {
+
+	t.Run("ignored", func(t *testing.T) {
+		tests, err := Collect("testdata/ignored")
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if len(tests) == 0 {
+			return
+		}
+		t.Fatalf("unexepected tests collected: %+v", tests)
+	})
+
+	t.Run("collected", func(t *testing.T) {
+		want := []testf{
+			{Path: "testdata/collected", Name: "Test"},
+			{Path: "testdata/collected", Name: "Test_It"},
+			{Path: "testdata/collected", Name: "TestⱯUnicodeName"},
+			{Path: "testdata/collected", Name: "TestAliasedT"},
+			{Path: "testdata/collected", Name: "TestMain"},
+			{Path: "testdata/collected", Name: "FuzzIt"},
+			{Path: "testdata/collected", Name: "TestWithSubtests"},
+			{Path: "testdata/collected", Name: "ExampleWithOutput"},
+		}
+
+		tests, err := Collect("testdata/collected")
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if !reflect.DeepEqual(want, tests) {
+			t.Fatalf("wanted\n\t%+v\nbut got\n\t%+v", want, tests)
+		}
+	})
+
+}
+
+func TestAssign(t *testing.T) {
+	given := []testf{
+		{Path: "package", Name: "Test"},
+		{Path: "package", Name: "Test0"},
+		{Path: "package", Name: "Test1"},
+		{Path: "package", Name: "Test2"},
+		{Path: "other/package", Name: "Test"},
+		{Path: "other/package", Name: "Test3"},
+		{Path: "other/package", Name: "Test4"},
+		{Path: "other/package", Name: "Test5"},
+		{Path: "other/package", Name: "Test6"},
+	}
+
+	tests := []struct {
+		name         string
+		index, total int
+		seed         int64
+		wantPaths    []string
+		wantNames    []string
+	}{
+		{
+			name:      "1/1",
+			index:     0,
+			total:     1,
+			wantPaths: []string{"./other/package", "./package"},
+			wantNames: []string{"Test", "Test0", "Test1", "Test2", "Test3", "Test4", "Test5", "Test6"},
+		},
+		{
+			index:     0,
+			total:     100,
+			wantPaths: []string{"./package"},
+			wantNames: []string{"Test"},
+		},
+		{
+			index:     99,
+			total:     100,
+			wantPaths: nil,
+			wantNames: nil,
+		},
+		{
+			index:     0,
+			total:     4,
+			wantPaths: []string{"./other/package", "./package"},
+			wantNames: []string{"Test", "Test6"},
+		},
+		{
+			index:     1,
+			total:     4,
+			wantPaths: []string{"./other/package", "./package"},
+			wantNames: []string{"Test0", "Test3"},
+		},
+		{
+			index:     2,
+			total:     4,
+			wantPaths: []string{"./other/package", "./package"},
+			wantNames: []string{"Test1", "Test4"},
+		},
+		{
+			index:     3,
+			total:     4,
+			wantPaths: []string{"./other/package", "./package"},
+			wantNames: []string{"Test2", "Test5"},
+		},
+		{
+			index:     0,
+			total:     3,
+			seed:      1,
+			wantPaths: []string{"./other/package", "./package"},
+			wantNames: []string{"Test2", "Test3", "Test4"},
+		},
+		{
+			index:     1,
+			total:     3,
+			seed:      1,
+			wantPaths: []string{"./other/package"},
+			wantNames: []string{"Test", "Test5", "Test6"},
+		},
+		{
+			index:     2,
+			total:     3,
+			seed:      1,
+			wantPaths: []string{"./other/package", "./package"},
+			wantNames: []string{"Test", "Test1"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(fmt.Sprintf("%d/%d", tt.index, tt.total), func(t *testing.T) {
+			names, paths := Assign(given, tt.index, tt.total, tt.seed)
+			if !reflect.DeepEqual(tt.wantPaths, paths) {
+				t.Errorf("wanted\n\t%+v\nbut got\n\t%+v", tt.wantPaths, paths)
+			}
+			if !reflect.DeepEqual(tt.wantNames, names) {
+				t.Errorf("wanted\n\t%+v\nbut got\n\t%+v", tt.wantNames, names)
+			}
+		})
+	}
+}

internal/testdata/collected/collected_test.go

@@ -0,0 +1,29 @@
+package collected
+
+import (
+	"fmt"
+	"testing"
+)
+
+type T = testing.T
+
+func Test(t *testing.T) {}
+
+func Test_It(t *testing.T) {}
+
+func TestⱯUnicodeName(t *testing.T) {}
+
+func TestAliasedT(t *T) {}
+
+func TestMain(t *testing.T) {} // Only testing.M is ignored.
+
+func FuzzIt(f *testing.F) {}
+
+func ExampleWithOutput() {
+	fmt.Println("foo")
+	// Output: foo
+}
+
+func TestWithSubtests(t *testing.T) {
+	t.Run("subtest", func(t *testing.T) {})
+}

internal/testdata/ignored/ignore.go

@@ -0,0 +1,7 @@
+package ignore
+
+import "testing"
+
+func TestHelper(t *testing.T) {}
+
+func ExampleNotTested() {}

internal/testdata/ignored/ignore_test.go

@@ -0,0 +1,22 @@
+package ignore
+
+import "testing"
+
+func TestMain(m *testing.M) {}
+
+func Testament(t *testing.T) {}
+
+func TeståMent(t *testing.T) {}
+
+func testPrivate(t *testing.T) {}
+
+func TestWithExtraArgs(t *testing.T, foo int) {}
+
+func BenchmarkIgnored(b *testing.B) {}
+
+func OtherTest(t *testing.T) {}
+
+func ExampleWithNoOutput() {}
+
+func TestWeirdArg(t *testing.Cover) {}
+func TestWeirdArg2(t testing.T)     {}

internal/testdata/ignored/testdata/testdata_test.go

@@ -0,0 +1,5 @@
+package testdata
+
+import "testing"
+
+func TestInsideTestdata(t *testing.T) {}