Merge pull request #2001 from matthewdale/godriver3445-improve-testpath

GODRIVER-3445 Refactor how spec test paths are built and update test dirs.

Author: matthewdale

.evergreen/config.yml

@@ -44,6 +44,10 @@ functions:
           # Define an alias for the task runner script.
           TASK_RUNNER_ALIAS: &task-runner src/go.mongodb.org/mongo-driver/.evergreen/run-task.sh
         args: [.evergreen/setup-system.sh]
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args: [*task-runner, init-submodule]
     - command: expansions.update
       params:
         file: src/go.mongodb.org/mongo-driver/expansion.yml

Taskfile.yml

@@ -17,6 +17,8 @@ tasks:
 
   check-license: bash etc/check_license.sh
 
+  init-submodule: git submodule update --init
+
   build:
     deps: [install-libmongocrypt]
     cmds:
@@ -149,9 +151,9 @@ tasks:
   evg-test-load-balancers:
     # Load balancer should be tested with all unified tests as well as tests in the following
     # components: retryable reads, retryable writes, change streams, initial DNS seedlist discovery.
-    - go test ${BUILD_TAGS} ./internal/integration -run TestUnifiedSpecs/retryable-reads -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
-    - go test ${BUILD_TAGS} ./internal/integration -run TestRetryableWritesSpec -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
-    - go test ${BUILD_TAGS} ./internal/integration -run TestChangeStreamSpec -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
+    - go test ${BUILD_TAGS} ./internal/integration/unified -run TestUnifiedSpec/retryable-reads -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
+    - go test ${BUILD_TAGS} ./internal/integration/unified -run TestUnifiedSpec/retryable-writes -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
+    - go test ${BUILD_TAGS} ./internal/integration/unified -run TestUnifiedSpec/change-streams -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestInitialDNSSeedlistDiscoverySpec/load_balanced -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestLoadBalancerSupport -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestLoadBalancedConnectionHandshake -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
@@ -174,13 +176,10 @@ tasks:
     - go test ${BUILD_TAGS} ./internal/integration -run TestWriteConcernError -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestErrorsCodeNamePropagated -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestLoadBalancerSupport -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
-    - go test ${BUILD_TAGS} ./internal/integration -run TestUnifiedSpecs/retryable-reads -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestRetryableReadsProse -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestRetryableWritesSpec -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestRetryableWritesProse -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
-    - go test ${BUILD_TAGS} ./internal/integration -run TestUnifiedSpecs/sessions -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestSessionsProse -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
-    - go test ${BUILD_TAGS} ./internal/integration -run TestUnifiedSpecs/transactions/legacy -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestConvenientTransactions -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestCursor -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration/unified -run TestUnifiedSpec -v -timeout {{.TEST_TIMEOUT}}s >> test.suite

internal/integration/retryable_writes_spec_test.go

@@ -98,6 +98,9 @@ func runTestFile(t *testing.T, filepath string, expectValidFail bool, opts ...*O
 	mtOpts := mtest.NewOptions().
 		RunOn(fileReqs...).
 		CreateClient(false)
+	if strings.Contains(filepath, "atlas-data-lake-testing") {
+		mtOpts.AtlasDataLake(true)
+	}
 	mt := mtest.New(t, mtOpts)
 
 	for _, testCase := range testCases {

internal/integration/unified/unified_spec_runner.go

@@ -8,42 +8,41 @@ package unified
 
 import (
 	"context"
-	"path"
 	"testing"
+
+	"go.mongodb.org/mongo-driver/v2/internal/spectest"
 )
 
 var (
 	passDirectories = []string{
-		"unified-test-format/valid-pass",
-		"versioned-api",
-		"crud/unified",
-		"change-streams",
-		"transactions/unified",
-		"load-balancers",
-		"collection-management",
-		"command-monitoring",
-		"command-monitoring/logging",
-		"connection-monitoring-and-pooling/logging",
-		"sessions",
-		"retryable-reads/unified",
-		"retryable-writes/unified",
-		"client-side-encryption/unified",
-		"client-side-operations-timeout",
-		"gridfs",
-		"server-selection/logging",
-		"server-discovery-and-monitoring/unified",
-		"run-command",
-		"index-management",
+		"unified-test-format/tests/valid-pass",
+		"versioned-api/tests",
+		"crud/tests/unified",
+		"change-streams/tests/unified",
+		"transactions/tests/unified",
+		"load-balancers/tests",
+		"collection-management/tests",
+		"command-logging-and-monitoring/tests/monitoring",
+		"command-logging-and-monitoring/tests/logging",
+		"connection-monitoring-and-pooling/tests/logging",
+		"sessions/tests",
+		"retryable-reads/tests/unified",
+		"retryable-writes/tests/unified",
+		"client-side-encryption/tests/unified",
+		"client-side-operations-timeout/tests",
+		"gridfs/tests",
+		"server-selection/tests/logging",
+		"server-discovery-and-monitoring/tests/unified",
+		"run-command/tests/unified",
+		"index-management/tests",
+		"transactions-convenient-api/tests/unified",
+		"atlas-data-lake-testing/tests/unified",
 	}
 	failDirectories = []string{
-		"unified-test-format/valid-fail",
+		"unified-test-format/tests/valid-fail",
 	}
 )
 
-const (
-	dataDirectory = "../../../testdata"
-)
-
 func TestUnifiedSpec(t *testing.T) {
 	// Ensure the cluster is in a clean state before test execution begins.
 	if err := terminateOpenSessions(context.Background()); err != nil {
@@ -52,13 +51,13 @@ func TestUnifiedSpec(t *testing.T) {
 
 	for _, testDir := range passDirectories {
 		t.Run(testDir, func(t *testing.T) {
-			runTestDirectory(t, path.Join(dataDirectory, testDir), false)
+			runTestDirectory(t, spectest.Path(testDir), false)
 		})
 	}
 
 	for _, testDir := range failDirectories {
 		t.Run(testDir, func(t *testing.T) {
-			runTestDirectory(t, path.Join(dataDirectory, testDir), true)
+			runTestDirectory(t, spectest.Path(testDir), true)
 		})
 	}
 }

internal/integration/unified/unified_spec_test.go

@@ -13,7 +13,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"path"
+	"path/filepath"
 	"reflect"
 	"sync"
 	"testing"
@@ -157,14 +157,8 @@ type operationError struct {
 	ErrorLabelsOmit    []string `bson:"errorLabelsOmit"`
 }
 
-const dataPath string = "../../testdata/"
-
 var directories = []string{
-	"transactions/legacy",
-	"convenient-transactions",
-	"retryable-reads/legacy",
-	"read-write-concern/operation",
-	"atlas-data-lake-testing",
+	"read-write-concern/tests/operation",
 }
 
 var checkOutcomeOpts = options.Collection().SetReadPreference(readpref.Primary()).SetReadConcern(readconcern.Local())
@@ -178,9 +172,9 @@ var specTestRegistry = func() *bson.Registry {
 func TestUnifiedSpecs(t *testing.T) {
 	for _, specDir := range directories {
 		t.Run(specDir, func(t *testing.T) {
-			for _, fileName := range jsonFilesInDir(t, path.Join(dataPath, specDir)) {
+			for _, fileName := range jsonFilesInDir(t, spectest.Path(specDir)) {
 				t.Run(fileName, func(t *testing.T) {
-					runSpecTestFile(t, specDir, fileName)
+					runSpecTestFile(t, filepath.Join(specDir, fileName))
 				})
 			}
 		})
@@ -189,8 +183,7 @@ func TestUnifiedSpecs(t *testing.T) {
 
 // specDir: name of directory for a spec in the data/ folder
 // fileName: name of test file in specDir
-func runSpecTestFile(t *testing.T, specDir, fileName string) {
-	filePath := path.Join(dataPath, specDir, fileName)
+func runSpecTestFile(t *testing.T, filePath string) {
 	content, err := ioutil.ReadFile(filePath)
 	assert.Nil(t, err, "unable to read spec test file %v: %v", filePath, err)
 
@@ -206,9 +199,6 @@ func runSpecTestFile(t *testing.T, specDir, fileName string) {
 	mtOpts := mtest.NewOptions().
 		RunOn(testFile.RunOn...).
 		CreateClient(false)
-	if specDir == "atlas-data-lake-testing" {
-		mtOpts.AtlasDataLake(true)
-	}
 	mt := mtest.New(t, mtOpts)
 
 	for _, test := range testFile.Tests {

internal/integration/unified_spec_test.go

@@ -157,7 +157,7 @@ func compareServers(t *testing.T, expected []*serverDesc, actual []description.S
 	}
 }
 
-var maxStalenessTestsDir = spectest.TestPath(2, "max-staleness")
+var maxStalenessTestsDir = spectest.Path("max-staleness/tests")
 
 // Test case for all max staleness spec tests.
 func TestMaxStalenessSpec(t *testing.T) {
@@ -176,7 +176,7 @@ func TestMaxStalenessSpec(t *testing.T) {
 	}
 }
 
-var selectorTestsDir = spectest.TestPath(2, "server-selection")
+var selectorTestsDir = spectest.Path("server-selection/tests")
 
 func selectServers(t *testing.T, test *testCase) error {
 	servers := make([]description.Server, 0, len(test.TopologyDescription.Servers))

internal/serverselector/server_selector_test.go

@@ -10,7 +10,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"strings"
 	"testing"
 
 	"go.mongodb.org/mongo-driver/v2/internal/require"
@@ -33,19 +32,20 @@ func FindJSONFilesInDir(t *testing.T, dir string) []string {
 		files = append(files, entry.Name())
 	}
 
+	if len(files) == 0 {
+		t.Fatalf("no JSON files found in %q", dir)
+	}
+
 	return files
 }
 
-// TestPath will construct a path to a test file in the specifications git
-// submodule. The path will be relative to the current package so a depth should
-// be provided to indicate how many directories to go up.
-func TestPath(depth int, testDir string, subDirs ...string) string {
-	const basePath = "testdata/specifications/source/"
-
-	// Create a string of "../" repeated 'depth' times
-	relativePath := strings.Repeat("../", depth)
-	// Construct the full path
-	fullPath := relativePath + basePath + testDir + "/tests/" + filepath.Join(subDirs...)
+// Path returns the absolute path to the given specifications repo file or
+// subdirectory.
+func Path(subdir string) string {
+	testPath, err := filepath.Abs("../../../testdata/specifications/source/")
+	if err != nil {
+		panic(err)
+	}
 
-	return fullPath
+	return filepath.Join(testPath, subdir)
 }

internal/spectest/spectest.go

@@ -36,7 +36,7 @@ var (
 		reg.RegisterTypeMapEntry(bson.TypeEmbeddedDocument, reflect.TypeOf(bson.Raw{}))
 		return reg
 	}()
-	readWriteConcernTestsDir = spectest.TestPath(1, "read-write-concern")
+	readWriteConcernTestsDir = spectest.Path("read-write-concern/tests")
 )
 
 type connectionStringTestFile struct {

mongo/read_write_concern_spec_test.go

@@ -38,7 +38,7 @@ type testContainer struct {
 }
 
 // Note a test supporting the deprecated gssapiServiceName property was removed from data/auth/auth_tests.json
-var authTestsDir = spectest.TestPath(4, "auth", "legacy")
+var authTestsDir = spectest.Path("auth/tests/legacy")
 
 func runTestsInFile(t *testing.T, dirname string, filename string) {
 	filepath := path.Join(dirname, filename)