GODRIVER-3313 Skip CSOT spec tests on Windows and macOS. (mongodb#1818)

Author: matthewdale

.evergreen/config.yml

@@ -377,7 +377,7 @@ functions:
       params:
         shell: "bash"
         working_dir: src/go.mongodb.org/mongo-driver
-        include_expansions_in_env: ["TOPOLOGY", "AUTH", "SSL", "MONGODB_URI", "CRYPT_SHARED_LIB_PATH", "SKIP_CRYPT_SHARED_LIB", "RACE", "MONGO_GO_DRIVER_COMPRESSOR", "REQUIRE_API_VERSION", "LOAD_BALANCER"]
+        include_expansions_in_env: ["TOPOLOGY", "AUTH", "SSL", "SKIP_CSOT_TESTS", "MONGODB_URI", "CRYPT_SHARED_LIB_PATH", "SKIP_CRYPT_SHARED_LIB", "RACE", "MONGO_GO_DRIVER_COMPRESSOR", "REQUIRE_API_VERSION", "LOAD_BALANCER"]
         script: |
           ${PREPARE_SHELL}
           bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
@@ -2090,6 +2090,8 @@ axes:
           GCC_PATH: "/cygdrive/c/ProgramData/chocolatey/lib/mingw/tools/install/mingw64/bin"
           GO_DIST: "C:\\golang\\go1.22"
           VENV_BIN_DIR: "Scripts"
+          # CSOT tests are unreliable on our slow Windows hosts.
+          SKIP_CSOT_TESTS: true
       - id: "rhel87-64"
         display_name: "RHEL 8.7"
         run_on: rhel8.7-large
@@ -2101,6 +2103,8 @@ axes:
         batchtime: 1440 # Run at most once per 24 hours.
         variables:
           GO_DIST: "/opt/golang/go1.22"
+          # CSOT tests are unreliable on our slow macOS hosts.
+          SKIP_CSOT_TESTS: true
 
   # OSes that require >= 4.0 for SSL
   - id: os-ssl-40
@@ -2114,6 +2118,8 @@ axes:
           GCC_PATH: "/cygdrive/c/ProgramData/chocolatey/lib/mingw/tools/install/mingw64/bin"
           GO_DIST: "C:\\golang\\go1.22"
           VENV_BIN_DIR: "Scripts"
+          # CSOT tests are unreliable on our slow Windows hosts.
+          SKIP_CSOT_TESTS: true
       - id: "rhel87-64"
         display_name: "RHEL 8.7"
         run_on: rhel8.7-large
@@ -2125,6 +2131,8 @@ axes:
         batchtime: 1440 # Run at most once per 24 hours.
         variables:
           GO_DIST: "/opt/golang/go1.22"
+          # CSOT tests are unreliable on our slow macOS hosts.
+          SKIP_CSOT_TESTS: true
 
   - id: ocsp-rhel-87
     display_name: OS
@@ -2146,6 +2154,8 @@ axes:
           GCC_PATH: "/cygdrive/c/ProgramData/chocolatey/lib/mingw/tools/install/mingw64/bin"
           GO_DIST: "C:\\golang\\go1.22"
           SKIP_ECS_AUTH_TEST: true
+          # CSOT tests are unreliable on our slow Windows hosts.
+          SKIP_CSOT_TESTS: true
       - id: "ubuntu2004-64"
         display_name: "Ubuntu 20.04"
         run_on: ubuntu2004-test
@@ -2160,6 +2170,8 @@ axes:
           SKIP_ECS_AUTH_TEST: true
           SKIP_EC2_AUTH_TEST: true
           SKIP_WEB_IDENTITY_AUTH_TEST: true
+          # CSOT tests are unreliable on our slow macOS hosts.
+          SKIP_CSOT_TESTS: true
 
   - id: os-faas-80
     display_name: OS

.evergreen/run-tests.sh

@@ -87,6 +87,7 @@ fi
 
 AUTH=${AUTH} \
 SSL=${SSL} \
+SKIP_CSOT_TESTS=${SKIP_CSOT_TESTS} \
 MONGO_GO_DRIVER_CA_FILE=${MONGO_GO_DRIVER_CA_FILE} \
 MONGO_GO_DRIVER_KEY_FILE=${MONGO_GO_DRIVER_KEY_FILE} \
 MONGO_GO_DRIVER_PKCS8_ENCRYPTED_KEY_FILE=${MONGO_GO_DRIVER_PKCS8_ENCRYPTED_KEY_FILE} \

mongo/integration/csot_prose_test.go

@@ -8,6 +8,7 @@ package integration
 
 import (
 	"context"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -22,6 +23,13 @@ import (
 )
 
 func TestCSOTProse(t *testing.T) {
+	// Skip CSOT tests when SKIP_CSOT_TESTS=true. In Evergreen, we typically set
+	// that environment variable on Windows and macOS because the CSOT spec
+	// tests are unreliable on those hosts.
+	if os.Getenv("SKIP_CSOT_TESTS") == "true" {
+		t.Skip("Skipping CSOT test because SKIP_CSOT_TESTS=true")
+	}
+
 	mt := mtest.New(t, mtest.NewOptions().CreateClient(false))
 
 	mt.RunOpts("1. multi-batch writes", mtest.NewOptions().MinServerVersion("4.4").

mongo/integration/csot_test.go

@@ -9,6 +9,7 @@ package integration
 import (
 	"context"
 	"errors"
+	"os"
 	"testing"
 	"time"
 
@@ -27,6 +28,13 @@ import (
 // Test automatic "maxTimeMS" appending and connection closing behavior when
 // CSOT is disabled and enabled.
 func TestCSOT_maxTimeMS(t *testing.T) {
+	// Skip CSOT tests when SKIP_CSOT_TESTS=true. In Evergreen, we typically set
+	// that environment variable on Windows and macOS because the CSOT spec
+	// tests are unreliable on those hosts.
+	if os.Getenv("SKIP_CSOT_TESTS") == "true" {
+		t.Skip("Skipping CSOT test because SKIP_CSOT_TESTS=true")
+	}
+
 	mt := mtest.New(t, mtest.NewOptions().CreateClient(false))
 
 	testCases := []struct {
@@ -586,6 +594,13 @@ func TestCSOT_maxTimeMS(t *testing.T) {
 }
 
 func TestCSOT_errors(t *testing.T) {
+	// Skip CSOT tests when SKIP_CSOT_TESTS=true. In Evergreen, we typically set
+	// that environment variable on Windows and macOS because the CSOT spec
+	// tests are unreliable on those hosts.
+	if os.Getenv("SKIP_CSOT_TESTS") == "true" {
+		t.Skip("Skipping CSOT test because SKIP_CSOT_TESTS=true")
+	}
+
 	mt := mtest.New(t, mtest.NewOptions().
 		CreateClient(false).
 		// Blocking failpoints don't work on pre-4.2 and sharded clusters.

mongo/integration/unified/unified_spec_runner.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	"os"
 	"path"
 	"strings"
 	"testing"
@@ -150,6 +151,13 @@ func runTestFile(t *testing.T, filepath string, expectValidFail bool, opts ...*O
 			CreateClient(false)
 
 		mt.RunOpts(testCase.Description, mtOpts, func(mt *mtest.T) {
+			// Skip CSOT spec tests when SKIP_CSOT_TESTS=true. In Evergreen, we
+			// typically set that environment variable on Windows and macOS
+			// because the CSOT spec tests are unreliable on those hosts.
+			if os.Getenv("SKIP_CSOT_TESTS") == "true" && strings.Contains(filepath, "client-side-operations-timeout") {
+				mt.Skip("Skipping CSOT spec test because SKIP_CSOT_TESTS=true")
+			}
+
 			defer func() {
 				// catch panics from looking up elements and fail if it's unexpected
 				if r := recover(); r != nil {