PYTHON-4483 Use hatch as task runner (mongodb#1728)

blink1073 · NoahStapp · web-flow · commit 875688cecc88 · 2024-07-11T11:56:18.000-05:00
Co-authored-by: Noah Stapp &lt;noah@noahstapp.com&gt;
diff --git a/.evergreen/config.yml b/.evergreen/config.yml
@@ -365,7 +365,7 @@ functions:
           ${PREPARE_SHELL}
           set -o xtrace
           export PYTHON_BINARY=${PYTHON_BINARY}
-          bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-mockupdb
+          bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-mockupdb
 
   "run doctests":
      - command: shell.exec
@@ -375,18 +375,19 @@ functions:
          script: |
            ${PREPARE_SHELL}
            set -o xtrace
-           PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m doc-test
+           PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh doctest:test
 
   "run tests":
     - command: shell.exec
       params:
         working_dir: "src"
         shell: bash
-        include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
+        background: true
+        include_expansions_in_env:  ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
         script: |
           ${PREPARE_SHELL}
           if [ -n "${test_encryption}" ]; then
-            . .evergreen/tox.sh -m setup-encryption
+            ./.evergreen/hatch.sh encryption:setup
           fi
     - command: shell.exec
       type: test
@@ -407,6 +408,7 @@ functions:
           if [ -n "${test_encryption}" ]; then
             # Disable xtrace (just in case it was accidentally set).
             set +x
+            bash ${DRIVERS_TOOLS}/.evergreen/csfle/await-servers.sh
             export TEST_ENCRYPTION=1
             if [ -n "${test_encryption_pyopenssl}" ]; then
               export TEST_ENCRYPTION_PYOPENSSL=1
@@ -446,7 +448,7 @@ functions:
             SSL=${SSL} \
             TEST_DATA_LAKE=${TEST_DATA_LAKE} \
             MONGODB_API_VERSION=${MONGODB_API_VERSION} \
-            bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg
+            bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg
 
   "run enterprise auth tests":
     - command: shell.exec
@@ -462,7 +464,7 @@ functions:
             PYTHON_BINARY="${PYTHON_BINARY}" \
             TEST_ENTERPRISE_AUTH=1 \
             AUTH=auth \
-            bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg
+            bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg
 
   "run atlas tests":
     - command: shell.exec
@@ -478,7 +480,7 @@ functions:
           PROJECT_DIRECTORY="${PROJECT_DIRECTORY}" \
             PYTHON_BINARY="${PYTHON_BINARY}" \
             TEST_ATLAS=1 \
-            bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg
+            bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg
 
   "get aws auth secrets":
     - command: subprocess.exec
@@ -597,7 +599,7 @@ functions:
         working_dir: "src"
         script: |
           ${PREPARE_SHELL}
-          . .evergreen/tox.sh -m teardown-encryption
+          . .evergreen/hatch.sh encryption:teardown
           rm -rf $DRIVERS_TOOLS || true
           rm -f ./secrets-export.sh || true
 
@@ -683,7 +685,7 @@ functions:
           PYTHON_BINARY=${PYTHON_BINARY} \
           CA_FILE="$DRIVERS_TOOLS/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \
           OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \
-          bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg
+          bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg
 
   run-valid-ocsp-server:
     - command: shell.exec
@@ -2128,7 +2130,7 @@ tasks:
               ${PREPARE_SHELL}
               export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
               export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/${bucket_name}/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz
-              SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/tox.sh -m test-eg
+              SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/hatch.sh test:test-eg
 
     - name: testazurekms-task
       commands:
diff --git a/.evergreen/hatch.sh b/.evergreen/hatch.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+set -o errexit  # Exit the script with error if any of the commands fail
+set -x
+
+. .evergreen/utils.sh
+
+if [ -z "$PYTHON_BINARY" ]; then
+    PYTHON_BINARY=$(find_python3)
+fi
+
+if $PYTHON_BINARY -m hatch --version; then
+    run_hatch() {
+      $PYTHON_BINARY hatch run "$@"
+    }
+else # No toolchain hatch present, set up virtualenv before installing hatch
+    createvirtualenv "$PYTHON_BINARY" hatchenv
+    trap "deactivate; rm -rf hatchenv" EXIT HUP
+    python -m pip install -q hatch
+    run_hatch() {
+      python -m hatch run "$@"
+    }
+fi
+
+run_hatch "${@:1}"
diff --git a/.evergreen/run-azurekms-fail-test.sh b/.evergreen/run-azurekms-fail-test.sh
@@ -7,4 +7,4 @@ PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 \
     KEY_VAULT_ENDPOINT="${AZUREKMS_KEYVAULTENDPOINT}" \
     LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz \
     SUCCESS=false TEST_FLE_AZURE_AUTO=1 \
-    ./.evergreen/tox.sh -m test-eg
+    ./.evergreen/hatch.sh test:test-eg
diff --git a/.evergreen/run-azurekms-test.sh b/.evergreen/run-azurekms-test.sh
@@ -16,6 +16,6 @@ AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \
     $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
 echo "Untarring file ... end"
 echo "Running test ... begin"
-AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/tox.sh -m test-eg" \
+AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/hatch.sh test:test-eg" \
     $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
 echo "Running test ... end"
diff --git a/.evergreen/run-gcpkms-test.sh b/.evergreen/run-gcpkms-test.sh
@@ -14,5 +14,5 @@ echo "Untarring file ... begin"
 GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
 echo "Untarring file ... end"
 echo "Running test ... begin"
-GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz ./.evergreen/tox.sh -m test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
+GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz ./.evergreen/hatch.sh test:test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
 echo "Running test ... end"
diff --git a/.evergreen/run-mongodb-aws-ecs-test.sh b/.evergreen/run-mongodb-aws-ecs-test.sh
@@ -30,5 +30,5 @@ export TEST_AUTH_AWS=1
 export AUTH="auth"
 export SET_XTRACE_ON=1
 cd src
-$PYTHON_BINARY -m pip install -q --user tox
-bash .evergreen/tox.sh -m test-eg
+$PYTHON_BINARY -m pip install -q --user hatch
+bash .evergreen/hatch.sh test:test-eg
diff --git a/.evergreen/run-mongodb-aws-test.sh b/.evergreen/run-mongodb-aws-test.sh
@@ -24,4 +24,4 @@ set -x
 export TEST_AUTH_AWS=1
 export AUTH="auth"
 export SET_XTRACE_ON=1
-bash ./.evergreen/tox.sh -m test-eg
+bash ./.evergreen/hatch.sh test:test-eg
diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh
@@ -29,4 +29,4 @@ fi
 export TEST_AUTH_OIDC=1
 export COVERAGE=1
 export AUTH="auth"
-bash ./.evergreen/tox.sh -m test-eg -- "${@:1}"
+bash ./.evergreen/hatch.sh test:test-eg -- "${@:1}"
diff --git a/.evergreen/run-perf-tests.sh b/.evergreen/run-perf-tests.sh
@@ -16,4 +16,4 @@ export OUTPUT_FILE="${PROJECT_DIRECTORY}/results.json"
 export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
 export PERF_TEST=1
 
-bash ./.evergreen/tox.sh -m test-eg
+bash ./.evergreen/hatch.sh test:test-eg
diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh
@@ -123,9 +123,9 @@ if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE
 
     python -m pip install '.[encryption]'
 
-    # Install libmongocrypt if necessary.
+    # Setup encryption if necessary.
     if [ ! -d "libmongocrypt" ]; then
-        bash ./.evergreen/setup-libmongocrypt.sh
+        bash ./.evergreen/setup-encryption.sh
     fi
 
     # Use the nocrypto build to avoid dependency issues with older windows/python versions.
diff --git a/.evergreen/setup-encryption.sh b/.evergreen/setup-encryption.sh
@@ -2,6 +2,10 @@
 set -o errexit  # Exit the script with error if any of the commands fail
 set -o xtrace
 
+if [ -z "${DRIVERS_TOOLS}" ]; then
+    echo "Missing environment variable DRIVERS_TOOLS"
+fi
+
 TARGET=""
 
 if [ "Windows_NT" = "${OS:-''}" ]; then # Magic variable in cygwin
@@ -45,3 +49,6 @@ mkdir libmongocrypt
 tar xzf libmongocrypt.tar.gz -C ./libmongocrypt
 ls -la libmongocrypt
 ls -la libmongocrypt/nocrypto
+
+bash ${DRIVERS_TOOLS}/.evergreen/csfle/setup-secrets.sh
+bash ${DRIVERS_TOOLS}/.evergreen/csfle/start-servers.sh
diff --git a/.evergreen/teardown-encryption.sh b/.evergreen/teardown-encryption.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+set -o errexit  # Exit the script with error if any of the commands fail
+set -o xtrace
+
+if [ -z "${DRIVERS_TOOLS}" ]; then
+    echo "Missing environment variable DRIVERS_TOOLS"
+fi
+
+bash ${DRIVERS_TOOLS}/.evergreen/csfle/stop-servers.sh
+rm -rf libmongocrypt/ libmongocrypt_git/ libmongocrypt.tar.gz mongocryptd.pid
diff --git a/.evergreen/tox.sh b/.evergreen/tox.sh
diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh
@@ -66,7 +66,7 @@ createvirtualenv () {
 
     export PIP_QUIET=1
     python -m pip install --upgrade pip
-    python -m pip install --upgrade tox
+    python -m pip install --upgrade hatch
 }
 
 # Usage:
diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml
@@ -27,18 +27,18 @@ jobs:
           cache-dependency-path: 'pyproject.toml'
       - name: Install Python dependencies
         run: |
-          python -m pip install -U pip tox
+          python -m pip install -U pip hatch
       - name: Run linters
         run: |
-          tox -m lint-manual
+          hatch run lint:run-manual
       - name: Run compilation
         run: |
           export PYMONGO_C_EXT_MUST_BUILD=1
           pip install -v -e .
           python tools/fail_if_no_c.py
       - name: Run typecheck
         run: |
-          tox -m typecheck
+          hatch run typing:check
       - run: |
           sudo apt-get install -y cppcheck
       - run: |
@@ -64,17 +64,23 @@ jobs:
           allow-prereleases: true
       - name: Install dependencies
         run: |
-          pip install -q tox
+          pip install -U pip
+          if [ "${{ matrix.python-version }}" == "3.13" ]; then
+            pip install --pre cffi setuptools
+            pip install --no-build-isolation hatch
+          else
+            pip install hatch
+          fi
       - name: Start MongoDB
         uses: supercharge/mongodb-github-action@1.10.0
         with:
           mongodb-version: 6.0
       - name: Run tests
         run: |
-          tox -m test
+          hatch run test:test
       - name: Run async tests
         run: |
-          tox -m test-async
+          hatch run test:test-async
 
   doctest:
     runs-on: ubuntu-latest
@@ -89,14 +95,14 @@ jobs:
           cache-dependency-path: 'pyproject.toml'
       - name: Install dependencies
         run: |
-          pip install -q tox
+          pip install -U hatch pip
       - name: Start MongoDB
         uses: supercharge/mongodb-github-action@1.10.0
         with:
           mongodb-version: 4.4
       - name: Run tests
         run: |
-          tox -m doc-test
+          hatch run doctest:test
 
   docs:
     name: Docs Checks
@@ -111,10 +117,10 @@ jobs:
           python-version: '3.8'
       - name: Install dependencies
         run: |
-          pip install -q tox
+          pip install -U pip hatch
       - name: Build docs
         run: |
-          tox -m doc
+          hatch run doc:build
 
   linkcheck:
     name: Link Check
@@ -129,10 +135,10 @@ jobs:
           python-version: '3.8'
       - name: Install dependencies
         run: |
-          pip install -q tox
+          pip install -U pip hatch
       - name: Build docs
         run: |
-          tox -m linkcheck
+          hatch run doc:linkcheck
 
   typing:
     name: Typing Tests
@@ -149,10 +155,10 @@ jobs:
           cache-dependency-path: 'pyproject.toml'
       - name: Install dependencies
         run: |
-          pip install -q tox
+          pip install -U pip hatch
       - name: Run typecheck
         run: |
-          tox -m typecheck
+          hatch run typing:check
 
   make_sdist:
     runs-on: ubuntu-latest
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
diff --git a/README.md b/README.md
diff --git a/doc/index.rst b/doc/index.rst
diff --git a/hatch.toml b/hatch.toml
diff --git a/tox.ini b/tox.ini
