GODRIVER-3445 Use Git submodules for spec tests (#2007)

Co-authored-by: Matt Dale <[email protected]>

Author: prestonvasquez

.evergreen/config.yml

@@ -32,6 +32,11 @@ functions:
       type: system
       params:
         directory: src/go.mongodb.org/mongo-driver
+    - command: shell.exec
+      params:
+        working_dir: "src/go.mongodb.org/mongo-driver"
+        script: |
+          git submodule update --init
     # Make an env.sh and evergreen expansion file with dynamic values
     - command: subprocess.exec
       params:

.github/dependabot.yml

@@ -11,4 +11,8 @@ updates:
   - package-ecosystem: gomod
     directory: /
     schedule:
-      interval: "weekly"
+      interval: "weekly"
+  - package-ecosystem: "gitsubmodule"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "specifications"]
-	path = specifications
-	url = git@github.com:mongodb/specifications.git
+	path = testdata/specifications
+	url = https://github.com/mongodb/specifications

Taskfile.yml

@@ -17,6 +17,8 @@ tasks:
 
   check-license: bash etc/check_license.sh
 
+  init-submodule: git submodule update --init
+
   build:
     deps: [install-libmongocrypt]
     cmds:
@@ -125,7 +127,7 @@ tasks:
     - go test -exec "env PKG_CONFIG_PATH=${PKG_CONFIG_PATH} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} DYLD_LIBRARY_PATH=$MACOS_LIBRARY_PATH}" ${BUILD_TAGS} -v -timeout {{.TEST_TIMEOUT}}s -p 1 ./... >> test.suite
 
   evg-test-atlas-data-lake:
-    - ATLAS_DATA_LAKE_INTEGRATION_TEST=true go test -v ./internal/integration -run TestUnifiedSpecs/atlas-data-lake-testing >> spec_test.suite
+    - ATLAS_DATA_LAKE_INTEGRATION_TEST=true go test -v ./internal/integration/unified -run TestUnifiedSpec/atlas-data-lake-testing >> spec_test.suite
     - ATLAS_DATA_LAKE_INTEGRATION_TEST=true go test -v ./internal/integration -run TestAtlasDataLake >> spec_test.suite
 
   evg-test-enterprise-auth: 
@@ -151,9 +153,6 @@ tasks:
   evg-test-load-balancers:
     # Load balancer should be tested with all unified tests as well as tests in the following
     # components: retryable reads, retryable writes, change streams, initial DNS seedlist discovery.
-    - go test ${BUILD_TAGS} ./internal/integration -run TestUnifiedSpecs/retryable-reads -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
-    - go test ${BUILD_TAGS} ./internal/integration -run TestRetryableWritesSpec -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
-    - go test ${BUILD_TAGS} ./internal/integration -run TestChangeStreamSpec -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestInitialDNSSeedlistDiscoverySpec/load_balanced -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestLoadBalancerSupport -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestLoadBalancedConnectionHandshake -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
@@ -169,20 +168,15 @@ tasks:
   evg-test-serverless:
     # Serverless should be tested with all unified tests as well as tests in the following components: CRUD, load balancer,
     # retryable reads, retryable writes, sessions, transactions and cursor behavior.
-    - go test ${BUILD_TAGS} ./internal/integration -run TestCrudSpec -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestWriteErrorsWithLabels -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestWriteErrorsDetails -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestHintErrors -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestWriteConcernError -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestErrorsCodeNamePropagated -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestLoadBalancerSupport -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
-    - go test ${BUILD_TAGS} ./internal/integration -run TestUnifiedSpecs/retryable-reads -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestRetryableReadsProse -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
-    - go test ${BUILD_TAGS} ./internal/integration -run TestRetryableWritesSpec -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestRetryableWritesProse -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
-    - go test ${BUILD_TAGS} ./internal/integration -run TestUnifiedSpecs/sessions -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestSessionsProse -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
-    - go test ${BUILD_TAGS} ./internal/integration -run TestUnifiedSpecs/transactions/legacy -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestConvenientTransactions -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration -run TestCursor -v -timeout {{.TEST_TIMEOUT}}s >> test.suite
     - go test ${BUILD_TAGS} ./internal/integration/unified -run TestUnifiedSpec -v -timeout {{.TEST_TIMEOUT}}s >> test.suite

bson/bson_binary_vector_spec_test.go

@@ -15,9 +15,10 @@ import (
 	"testing"
 
 	"go.mongodb.org/mongo-driver/v2/internal/require"
+	"go.mongodb.org/mongo-driver/v2/internal/spectest"
 )
 
-const bsonBinaryVectorDir = "../testdata/bson-binary-vector/"
+var bsonBinaryVectorDir = spectest.Path("bson-binary-vector")
 
 type bsonBinaryVectorTests struct {
 	Description string                     `json:"description"`
@@ -128,18 +129,7 @@ func runBsonBinaryVectorTest(t *testing.T, testKey string, test bsonBinaryVector
 	require.NoError(t, err, "decoding canonical BSON")
 
 	t.Run("Unmarshaling", func(t *testing.T) {
-		skipCases := map[string]string{
-			"Overflow Vector INT8":                "compile-time restriction",
-			"Underflow Vector INT8":               "compile-time restriction",
-			"INT8 with float inputs":              "compile-time restriction",
-			"Overflow Vector PACKED_BIT":          "compile-time restriction",
-			"Underflow Vector PACKED_BIT":         "compile-time restriction",
-			"Vector with float values PACKED_BIT": "compile-time restriction",
-			"Negative padding PACKED_BIT":         "compile-time restriction",
-		}
-		if reason, ok := skipCases[test.Description]; ok {
-			t.Skipf("skip test case %s: %s", test.Description, reason)
-		}
+		spectest.CheckSkip(t)
 
 		errMap := map[string]string{
 			"FLOAT32 with padding":                             "padding must be 0",
@@ -163,24 +153,7 @@ func runBsonBinaryVectorTest(t *testing.T, testKey string, test bsonBinaryVector
 	})
 
 	t.Run("Marshaling", func(t *testing.T) {
-		skipCases := map[string]string{
-			"FLOAT32 with padding":                             "private padding field",
-			"Insufficient vector data with 3 bytes FLOAT32":    "invalid case",
-			"Insufficient vector data with 5 bytes FLOAT32":    "invalid case",
-			"Overflow Vector INT8":                             "compile-time restriction",
-			"Underflow Vector INT8":                            "compile-time restriction",
-			"INT8 with padding":                                "private padding field",
-			"INT8 with float inputs":                           "compile-time restriction",
-			"Overflow Vector PACKED_BIT":                       "compile-time restriction",
-			"Underflow Vector PACKED_BIT":                      "compile-time restriction",
-			"Vector with float values PACKED_BIT":              "compile-time restriction",
-			"Padding specified with no vector data PACKED_BIT": "run in alternative case",
-			"Exceeding maximum padding PACKED_BIT":             "run in alternative case",
-			"Negative padding PACKED_BIT":                      "compile-time restriction",
-		}
-		if reason, ok := skipCases[test.Description]; ok {
-			t.Skipf("skip test case %s: %s", test.Description, reason)
-		}
+		spectest.CheckSkip(t)
 
 		t.Parallel()
 

bson/bson_corpus_spec_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"go.mongodb.org/mongo-driver/v2/internal/assert"
 	"go.mongodb.org/mongo-driver/v2/internal/require"
+	"go.mongodb.org/mongo-driver/v2/internal/spectest"
 )
 
 type testCase struct {
@@ -57,7 +58,7 @@ type parseErrorTestCase struct {
 	String      string `json:"string"`
 }
 
-const dataDir = "../testdata/bson-corpus/"
+var dataDir = spectest.Path("bson-corpus/tests")
 
 func findJSONFilesInDir(dir string) ([]string, error) {
 	files := make([]string, 0)

internal/integration/client_side_encryption_spec_test.go

@@ -11,15 +11,15 @@ package integration
 
 import (
 	"os"
-	"path"
+	"path/filepath"
 	"regexp"
 	"testing"
-)
 
-const (
-	encryptionSpecName = "client-side-encryption/legacy"
+	"go.mongodb.org/mongo-driver/v2/internal/spectest"
 )
 
+var encryptionSpecDir = spectest.Path("client-side-encryption/tests/legacy")
+
 func verifyClientSideEncryptionVarsSet(t *testing.T) {
 	t.Helper()
 
@@ -53,7 +53,7 @@ func verifyClientSideEncryptionVarsSet(t *testing.T) {
 func TestClientSideEncryptionSpec(t *testing.T) {
 	verifyClientSideEncryptionVarsSet(t)
 
-	for _, fileName := range jsonFilesInDir(t, path.Join(dataPath, encryptionSpecName)) {
+	for _, fileName := range jsonFilesInDir(t, encryptionSpecDir) {
 		t.Run(fileName, func(t *testing.T) {
 			re := regexp.MustCompile(`fle2\-Range\-.*\-Correctness`)
 			if re.Match([]byte(fileName)) {
@@ -62,7 +62,7 @@ func TestClientSideEncryptionSpec(t *testing.T) {
 			if fileName == "kmipKMS.json" && "" == os.Getenv("KMS_MOCK_SERVERS_RUNNING") {
 				t.Skipf("Skipping test as KMS_MOCK_SERVERS_RUNNING is not set")
 			}
-			runSpecTestFile(t, encryptionSpecName, fileName)
+			runSpecTestFile(t, filepath.Join(encryptionSpecDir, fileName))
 		})
 	}
 }

internal/integration/cmd_monitoring_helpers_test.go

@@ -83,7 +83,30 @@ func compareValues(mt *mtest.T, key string, expected, actual bson.RawValue) erro
 		if typeVal, err := e.LookupErr("$$type"); err == nil {
 			// $$type represents a type assertion
 			// for example {field: {$$type: "binData"}} should assert that "field" is an element with a binary value
-			return checkValueType(mt, key, actual.Type, typeVal.StringValue())
+			types := []string{}
+			switch typeVal.Type {
+			case bson.TypeArray:
+				elems, err := typeVal.Array().Values()
+				if err != nil {
+					return fmt.Errorf("error getting expected types: %v", err)
+				}
+
+				for _, elem := range elems {
+					types = append(types, elem.StringValue())
+				}
+			case bson.TypeString:
+				types = append(types, typeVal.StringValue())
+			}
+
+			// If at least one of the types does not return an error, then the test
+			// has passed.
+			for _, t := range types {
+				if err := checkValueType(mt, key, actual.Type, t); err == nil {
+					return nil
+				}
+			}
+
+			return fmt.Errorf("BSON type mismatch for key %s; expected one of %v, got %s", key, types, actual.Type)
 		}
 
 		a := actual.Document()