SWIFT-1337, SWIFT-1629, SWIFT-1557, SWIFT-1566: Various maintenance-related changes (#773)

Author: kmahar

.evergreen/config.yml

@@ -336,15 +336,9 @@ functions:
         script: |
           set -o xtrace
           set -o errexit
-
-          if [ "${SWIFT_MINOR_VERSION}" = "main-snapshot" ]; then
-              export SWIFT_VERSION="${SWIFT_MINOR_VERSION}"
-          else
-              # otherwise, find the latest patch release for the specified version
-              ${PYTHON} -m virtualenv ./requests-env
-              ./requests-env/${VENV_BIN_DIR}/python3 -m pip install requests
-              export SWIFT_VERSION="$(./requests-env/${VENV_BIN_DIR}/python3 .evergreen/get_latest_swift_patch.py ${SWIFT_MINOR_VERSION})"
-          fi
+          ${PYTHON} -m virtualenv ./requests-env
+          ./requests-env/${VENV_BIN_DIR}/python3 -m pip install requests
+          export SWIFT_VERSION="$(./requests-env/${VENV_BIN_DIR}/python3 .evergreen/get_latest_swift_patch.py ${SWIFT_MINOR_VERSION})"
           echo "SWIFT_VERSION: $SWIFT_VERSION" > swift-version.yml
           cat swift-version.yml
     - command: expansions.update
@@ -620,6 +614,72 @@ tasks:
             MONGODB_VERSION: "5.0"
             TOPOLOGY: "sharded_cluster"
         - func: "run tests"
+  
+    - name: "test-5.0-load_balancer"
+      tags: ["5.0", "load-balancer"]
+      commands:
+        - func: "prepare resources"
+        - func: "fix absolute paths"
+        - func: "bootstrap mongo-orchestration"
+          vars:
+            MONGODB_VERSION: "5.0"
+            TOPOLOGY: "sharded_cluster"
+            LOAD_BALANCER: "true"
+        - func: start-load-balancer
+        - func: "run-lb-tests"
+          vars:
+            TOPOLOGY: "load_balanced"
+        - func: stop-load-balancer
+
+    - name: "test-6.0-standalone"
+      tags: ["6.0", "standalone"]
+      commands:
+        - func: "prepare resources"
+        - func: "fix absolute paths"
+        - func: "bootstrap mongo-orchestration"
+          vars:
+            MONGODB_VERSION: "6.0"
+            TOPOLOGY: "server"
+        - func: "run tests"
+
+    - name: "test-6.0-replica_set"
+      tags: ["6.0", "replica_set"]
+      commands:
+        - func: "prepare resources"
+        - func: "fix absolute paths"
+        - func: "bootstrap mongo-orchestration"
+          vars:
+            MONGODB_VERSION: "6.0"
+            TOPOLOGY: "replica_set"
+        - func: "run tests"
+
+    - name: "test-6.0-sharded_cluster"
+      tags: ["6.0", "sharded_cluster"]
+      exec_timeout_secs: 3600 # TODO SWIFT-1410: remove
+      commands:
+        - func: "prepare resources"
+        - func: "fix absolute paths"
+        - func: "bootstrap mongo-orchestration"
+          vars:
+            MONGODB_VERSION: "6.0"
+            TOPOLOGY: "sharded_cluster"
+        - func: "run tests"
+
+    - name: "test-6.0-load_balancer"
+      tags: ["6.0", "load-balancer"]
+      commands:
+        - func: "prepare resources"
+        - func: "fix absolute paths"
+        - func: "bootstrap mongo-orchestration"
+          vars:
+            MONGODB_VERSION: "6.0"
+            TOPOLOGY: "sharded_cluster"
+            LOAD_BALANCER: "true"
+        - func: start-load-balancer
+        - func: "run-lb-tests"
+          vars:
+            TOPOLOGY: "load_balanced"
+        - func: stop-load-balancer
 
     - name: "test-latest-standalone"
       tags: ["latest", "standalone"]
@@ -671,6 +731,56 @@ tasks:
             TOPOLOGY: "load_balanced"
         - func: stop-load-balancer
 
+    - name: "test-rapid-standalone"
+      tags: ["rapid", "standalone"]
+      commands:
+        - func: "prepare resources"
+        - func: "fix absolute paths"
+        - func: "bootstrap mongo-orchestration"
+          vars:
+            MONGODB_VERSION: "rapid"
+            TOPOLOGY: "server"
+        - func: "run tests"
+
+    - name: "test-rapid-replica_set"
+      tags: ["rapid", "replica_set"]
+      commands:
+        - func: "prepare resources"
+        - func: "fix absolute paths"
+        - func: "bootstrap mongo-orchestration"
+          vars:
+            MONGODB_VERSION: "rapid"
+            TOPOLOGY: "replica_set"
+        - func: "run tests"
+
+    - name: "test-rapid-sharded_cluster"
+      tags: ["rapid", "sharded_cluster"]
+      exec_timeout_secs: 3600 # TODO SWIFT-1410: remove
+      commands:
+        - func: "prepare resources"
+        - func: "fix absolute paths"
+        - func: "bootstrap mongo-orchestration"
+          vars:
+            MONGODB_VERSION: "rapid"
+            TOPOLOGY: "sharded_cluster"
+        - func: "run tests"
+  
+    - name: "test-rapid-load_balancer"
+      tags: ["rapid", "load-balancer"]
+      commands:
+        - func: "prepare resources"
+        - func: "fix absolute paths"
+        - func: "bootstrap mongo-orchestration"
+          vars:
+            MONGODB_VERSION: "rapid"
+            TOPOLOGY: "sharded_cluster"
+            LOAD_BALANCER: "true"
+        - func: start-load-balancer
+        - func: "run-lb-tests"
+          vars:
+            TOPOLOGY: "load_balanced"
+        - func: stop-load-balancer
+
     - name: "test-atlas-connectivity"
       tags: ["atlas-connect"]
       commands:
@@ -1049,6 +1159,14 @@ axes:
         display_name: "latest"
         variables:
            MONGODB_VERSION: "latest"
+      - id: "rapid"
+        display_name: "rapid"
+        variables:
+           MONGODB_VERSION: "rapid"
+      - id: "6.0"
+        display_name: "6.0"
+        variables:
+          MONGODB_VERSION: "6.0"
       - id: "5.0"
         display_name: "5.0"
         variables:
@@ -1087,6 +1205,13 @@ axes:
           PYTHON: "/opt/mongodbtoolchain/v3/bin/python3"
           VENV_BIN_DIR: "bin"
 
+      - id: macos-10.14
+        display_name: "macOS 10.14"
+        run_on: macos-1014
+        variables:
+          PYTHON: "/opt/mongodbtoolchain/v3/bin/python3"
+          VENV_BIN_DIR: "bin"
+
       - id: macos-11
         display_name: "macOS 11"
         run_on: macos-1100
@@ -1112,10 +1237,6 @@ axes:
   - id: swift-version
     display_name: "Swift"
     values:
-      - id: "5.1"
-        display_name: "Swift 5.1"
-        variables:
-           SWIFT_MINOR_VERSION: "5.1"
       - id: "5.2"
         display_name: "Swift 5.2"
         variables:
@@ -1250,27 +1371,32 @@ buildvariants:
 
 - matrix_name: "min-version-compile"
   matrix_spec:
-  # Ubuntu 20.04 does not have Swift 5.1 toolchains.
     os-fully-featured:
-      - "macos-11"
-      - "ubuntu-18.04"
-    swift-version: "5.1"
+      - macos-10.14
+      - ubuntu-18.04
+      - ubuntu-20.04
+    swift-version: "5.2"
   display_name: "Compile ${swift-version} ${os-fully-featured}"
   tasks:
     - "compile"
 
 - matrix_name: "tests-all"
   matrix_spec:
-    os-fully-featured: "*"
+    os-fully-featured:
+      - macos-11
+      - ubuntu-18.04
+      - ubuntu-20.04
     swift-version:
       - "5.5"
       - "5.6"
       - "5.7-dev"
     ssl-auth: "*"
   display_name: "${swift-version} ${os-fully-featured} ${ssl-auth}"
   tasks:
+     - ".rapid !.load-balancer"
      - ".latest !.load-balancer"
-     - ".5.0"
+     - ".6.0 !.load-balancer"
+     - ".5.0 !.load-balancer"
      - ".4.4"
      - ".4.2"
      - ".4.0"
@@ -1302,7 +1428,10 @@ buildvariants:
 
 - matrix_name: "atlas-connect"
   matrix_spec:
-    os-fully-featured: "*"
+    os-fully-featured:
+      - macos-11
+      - ubuntu-18.04
+      - ubuntu-20.04
     swift-version:
       - "5.5"
       - "5.6"
@@ -1340,6 +1469,8 @@ buildvariants:
       - "ubuntu-20.04"
     versions:
       - latest
+      - rapid
+      - 6.0
       - 5.0
       - 4.4
     swift-version: "5.6"
@@ -1363,6 +1494,8 @@ buildvariants:
     swift-version: "5.6"
     versions:
       - latest
+      - rapid
+      - 6.0
       - 5.0
       - 4.4
   display_name: "OCSP ${swift-version} ${os-fully-featured} ${versions}"
@@ -1382,6 +1515,9 @@ buildvariants:
   tasks:
     # Versioned API was introduced in MongoDB 4.7
     - "test-latest-standalone"
+    - "test-rapid-standalone"
+    - "test-6.0-standalone"
+    - "test-5.0-standalone"
 
 - matrix_name: "format-lint"
   display_name: "Format and Lint"
@@ -1418,4 +1554,4 @@ buildvariants:
     ssl-auth: "*"
     check-leaks: "leaks"
   tasks:
-     - ".5.0"
+     - ".6.0 !.load-balancer"

.evergreen/configure-swift.sh

@@ -16,20 +16,16 @@ export SWIFTENV_ROOT="${INSTALL_DIR}/swiftenv"
 export PATH="${SWIFTENV_ROOT}/bin:$PATH"
 eval "$(swiftenv init -)"
 
-# dynamically determine latest available snapshot if needed
-if [ "$SWIFT_VERSION" = "main-snapshot" ]; then
-    SWIFT_VERSION=$(swiftenv install --list-snapshots | tail -1)
-fi
-
 if [ "$OS" == "darwin" ]; then
-    # latest snapshots require a newer version of Xcode/Command Line Tools
-    if [[ "$SWIFT_VERSION" == DEVELOPMENT-SNAPSHOT* ]]; then
-        sudo xcode-select -s /Applications/Xcode13.1.app
+    # 5.2 requires an older version of Xcode/Command Line Tools
+    if [[ "$SWIFT_VERSION" == 5.2.* ]]; then
+        sudo xcode-select -s /Applications/Xcode11.3.app
     else
-        sudo xcode-select -s /Applications/Xcode12.app
+        sudo xcode-select -s /Applications/Xcode13.2.1.app
     fi
 
-    # TODO SWIFT-1421: remove this once we have new Xcode on Evergreen to test with
+    # required to use the toolchain's concurrency library rather than the one which ships with Xcode
+    # and may not necessarily match the Swift version being used.
     export DYLD_LIBRARY_PATH=${SWIFTENV_ROOT}/versions/${SWIFT_VERSION}/usr/lib/swift/macosx/
 fi
 

.evergreen/get_latest_swift_patch.py

@@ -2,33 +2,43 @@
 import requests
 import sys
 
-# This script accepts a version number in the form "x.y" as an argument. It will query the Swift Github
-# repo for releases and find the latest x.y.z patch release if available, and print out the matching tag.
+# This script accepts a single argument which can be either:
+# 1) a version number in the form "x.y", or 
+# 2) the string "snapshot"
+# If a version number is provided the script will query the Swift GitHub repo for releases and find the latest x.y.z patch
+# release if available, and print out the matching tag.
+# If "main-snapshot" is provided the script will find the latest development snapshot tagged on the Swift GitHub repo.
 
 if len(sys.argv) != 2:
     print("Expected 1 argument, but got: {}".format(sys.argv[1:]))
     exit(1)
 
 version = sys.argv[1]
-components = version.split('.')
-if len(components) != 2:
-    print("Expected version number in form x.y, got {}".format(version))
-    exit(1)
 
-major = components[0]
-minor = components[1]
+if version == "main-snapshot":
+    tag_data = requests.get('https://api.github.com/repos/apple/swift/tags').json()
+    latest_snapshot = next(filter(lambda tag: 'swift-DEVELOPMENT-SNAPSHOT' in tag['name'], tag_data))
+    print(latest_snapshot['name'][6:]) # drop the preceding 'swift-' from the tag, because swiftenv does the same when installing it.
+else:
+    components = version.split('.')
+    if len(components) != 2:
+        print("Expected version number in form x.y, got {}".format(version))
+        exit(1)
+
+    major = components[0]
+    minor = components[1]
 
-version_regex = '^swift-({}\.{}(\.(\d+))?)-RELEASE$'.format(major, minor)
+    version_regex = '^swift-({}\.{}(\.(\d+))?)-RELEASE$'.format(major, minor)
 
-release_data = requests.get('https://api.github.com/repos/apple/swift/releases').json()
-tag_names = map(lambda release: release['tag_name'], release_data)
+    release_data = requests.get('https://api.github.com/repos/apple/swift/releases').json()
+    tag_names = map(lambda release: release['tag_name'], release_data)
 
-# find tags matching the specified regexes
-matches = filter(lambda match: match is not None, map(lambda tag: re.match(version_regex, tag), tag_names))
+    # find tags matching the specified regexes
+    matches = filter(lambda match: match is not None, map(lambda tag: re.match(version_regex, tag), tag_names))
 
-# sort matches by their patch versions. patch versions of 0 are omitted so substitute 0 when the group is None.
-sorted_matches = sorted(matches, key=lambda match: int(match.group(2)[1:]) if match.group(2) is not None else 0, reverse=True)
+    # sort matches by their patch versions. patch versions of 0 are omitted so substitute 0 when the group is None.
+    sorted_matches = sorted(matches, key=lambda match: int(match.group(2)[1:]) if match.group(2) is not None else 0, reverse=True)
 
-# map to the first match group which contains the full version number.
-sorted_version_numbers = map(lambda match: match.group(1), sorted_matches)
-print(next(sorted_version_numbers))
+    # map to the first match group which contains the full version number.
+    sorted_version_numbers = map(lambda match: match.group(1), sorted_matches)
+    print(next(sorted_version_numbers))

.evergreen/install-swift.sh

@@ -20,8 +20,4 @@ git clone --depth 1 -b "osx-install-path" https://github.com/kmahar/swiftenv.git
 # install swift
 eval "$(swiftenv init -)"
 
-if [ "$SWIFT_VERSION" = "main-snapshot" ]; then
-    SWIFT_VERSION=$(swiftenv install --list-snapshots | tail -1)
-fi
-
 swiftenv install --install-local $SWIFT_VERSION

.evergreen/install-tools.sh

@@ -35,10 +35,10 @@ install_from_gh () {
 
 if [ $1 == "swiftlint" ]
 then
-    build_from_gh swiftlint https://github.com/realm/SwiftLint "0.46.3"
+    build_from_gh swiftlint https://github.com/realm/SwiftLint "0.48.0"
 elif [ $1 == "swiftformat" ]
 then
-    build_from_gh swiftformat https://github.com/nicklockwood/SwiftFormat "0.49.4"
+    build_from_gh swiftformat https://github.com/nicklockwood/SwiftFormat "0.49.17"
 elif [ $1 == "sourcery" ]
 then
     install_from_gh sourcery https://github.com/krzysztofzablocki/Sourcery/releases/download/1.6.1/Sourcery-1.6.1.zip

.evergreen/run-atlas-tests.sh

@@ -9,6 +9,13 @@ INSTALL_DIR="${PROJECT_DIRECTORY}/opt"
 # configure Swift
 . ${PROJECT_DIRECTORY}/.evergreen/configure-swift.sh
 
+
+# work around https://github.com/mattgallagher/CwlPreconditionTesting/issues/22 (bug still exists in version 1.x
+# when using Xcode 13.2)
+if [ "$OS" == "darwin" ]; then
+    EXTRA_FLAGS="-Xswiftc -Xfrontend -Xswiftc -validate-tbd-against-ir=none"
+fi
+
 # run the tests
 ATLAS_REPL="$ATLAS_REPL" ATLAS_SHRD="$ATLAS_SHRD" ATLAS_FREE="$ATLAS_FREE" ATLAS_TLS11="$ATLAS_TLS11" ATLAS_TLS12="$ATLAS_TLS12" \
 ATLAS_REPL_SRV="$ATLAS_REPL_SRV" ATLAS_SHRD_SRV="$ATLAS_SHRD_SRV" ATLAS_FREE_SRV="$ATLAS_FREE_SRV" ATLAS_TLS11_SRV="$ATLAS_TLS11_SRV" ATLAS_TLS12_SRV="$ATLAS_TLS12_SRV" \