Simplify justfile usage

Author: blink1073

.evergreen/hatch.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 set -eu
 
-. .evergreen/scripts/ensure-hatch.sh
+. .evergreen/scripts/setup-dev-env.sh
 hatch run "$@"

.evergreen/scripts/configure-env.sh

@@ -14,12 +14,14 @@ fi
 PROJECT_DIRECTORY="$(pwd)"
 DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools"
 CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo}
+HATCH_CONFIG=$PROJECT_DIRECTORY/hatch_config.toml
 
 # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
 if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
     DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
     PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY)
     CARGO_HOME=$(cygpath -m $CARGO_HOME)
+    HATCH_CONFIG=$(cygpath -m "$HATCH_CONFIG")
 fi
 
 SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts"
@@ -34,7 +36,6 @@ export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration"
 export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
 
 cat <<EOT > "$SCRIPT_DIR"/env.sh
-set -o errexit
 export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
 export CURRENT_VERSION="$CURRENT_VERSION"
 export SKIP_LEGACY_SHELL=1
@@ -59,6 +60,7 @@ export skip_ECS_auth_test="${skip_ECS_auth_test:-}"
 
 export CARGO_HOME="$CARGO_HOME"
 export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
+export HATCH_CONFIG="$HATCH_CONFIG"
 export PATH="$MONGODB_BINARIES:$PATH"
 # shellcheck disable=SC2154
 export PROJECT="${project:-mongo-python-driver}"

.evergreen/scripts/ensure-hatch.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+set -eu
+
+HERE=$(dirname ${BASH_SOURCE:-$0})
+pushd "$(dirname "$(dirname $HERE)")" > /dev/null
+
+# Source the env file to pick up common variables.
+if [ -f $HERE/scripts/env.sh ]; then
+  source $HERE/scripts/env.sh
+fi
+
+# Set the location of the python bin dir.
+if [ "Windows_NT" = "${OS:-}" ]; then
+  BIN_DIR=.venv/Scripts
+else
+  BIN_DIR=.venv/bin
+fi
+
+# Ensure there is a python venv.
+if [ ! -d $BIN_DIR ]; then
+  . .evergreen/utils.sh
+
+  if [ -z "${PYTHON_BINARY:-}" ]; then
+      PYTHON_BINARY=$(find_python3)
+  fi
+
+  echo "Creating virtual environment..."
+  createvirtualenv "$PYTHON_BINARY" .venv
+  echo "Creating virtual environment... done."
+fi
+
+# Activate the virtual env.
+. $BIN_DIR/activate
+
+# Ensure there is a local hatch.
+if [ ! -f $BIN_DIR/hatch ]; then
+  echo "Installing hatch..."
+  python -m pip install hatch || {
+    # CARGO_HOME is defined in configure-env.sh
+    export CARGO_HOME=${CARGO_HOME:-$HOME/.cargo/}
+    export RUSTUP_HOME="${CARGO_HOME}/.rustup"
+    ${DRIVERS_TOOLS}/.evergreen/install-rust.sh
+    source "${CARGO_HOME}/env"
+    python -m pip install hatch
+  }
+  echo "Installing hatch... done."
+fi
+
+# Ensure hatch does not write to user or global locations.
+HATCH_CONFIG=${HATCH_CONFIG:-hatch_config.toml}
+if [ ! -f ${HATCH_CONFIG} ]; then
+  touch hatch_config.toml
+  hatch config restore
+  hatch config set dirs.data "$(pwd)/.hatch/data"
+  hatch config set dirs.cache "$(pwd)/.hatch/cache"
+fi
+
+# Ensure there is a local pre-commit.
+if [ ! -f $BIN_DIR/pre-commit ]; then
+  python -m pip install pre-commit
+fi
+
+# Ensure the pre-commit hook is installed.
+if [ ! -f .git/hooks/pre-commit ]; then
+  pre-commit install
+fi
+
+# Install pymongo and its test deps.
+python -m pip install ".[test]"

.evergreen/scripts/setup-dev-env.sh

@@ -16,4 +16,4 @@ rsync -az -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:$remote_
 echo "Copying files to $target... done"
 
 ssh $target $remote_dir/.evergreen/scripts/setup-system.sh
-ssh $target "PYTHON_BINARY=${PYTHON_BINARY:-} $remote_dir/.evergreen/scripts/ensure-hatch.sh"
+ssh $target "PYTHON_BINARY=${PYTHON_BINARY:-} $remote_dir/.evergreen/scripts/setup-dev-env.sh"

.evergreen/setup-spawn-host.sh

@@ -1,6 +1,6 @@
-#!/bin/bash -ex
+#!/bin/bash
 
-set -o xtrace
+set -eu
 
 find_python3() {
     PYTHON=""

.evergreen/utils.sh

@@ -27,9 +27,11 @@ jobs:
           python-version: "3.9"
           cache: 'pip'
           cache-dependency-path: 'pyproject.toml'
+      - name: Install just
+        uses: extractions/setup-just@v2
       - name: Install Python dependencies
         run: |
-          python -m pip install -U pip hatch rust-just pre-commit
+          just install
       - name: Run linters
         run: |
           just lint-manual
@@ -73,18 +75,16 @@ jobs:
           cache: 'pip'
           cache-dependency-path: 'pyproject.toml'
           allow-prereleases: true
+      - name: Install just
+        uses: extractions/setup-just@v2
       - name: Install dependencies
         run: |
-          pip install -U pip rust-just
-          if [[ "${{ matrix.python-version }}" == "3.13" ]]; then
-            pip install --pre cffi setuptools
-            pip install --no-build-isolation hatch
-          elif [[ "${{ matrix.python-version }}" == "3.13t" ]]; then
+         if [[ "${{ matrix.python-version }}" == "3.13t" ]]; then
             # Hatch can't be installed on 3.13t, use pytest directly.
             pip install .
             pip install -r requirements/test.txt
           else
-            pip install hatch
+            just install
           fi
       - name: Start MongoDB
         uses: supercharge/[email protected]
@@ -111,9 +111,8 @@ jobs:
           python-version: "3.9"
           cache: 'pip'
           cache-dependency-path: 'pyproject.toml'
-      - name: Install dependencies
-        run: |
-          pip install -U hatch pip rust-just
+      - name: Install just
+        uses: extractions/setup-just@v2
       - name: Start MongoDB
         uses: supercharge/[email protected]
         with:
@@ -135,9 +134,11 @@ jobs:
           cache-dependency-path: 'pyproject.toml'
           # Build docs on lowest supported Python for furo
           python-version: '3.9'
+      - name: Install just
+        uses: extractions/setup-just@v2
       - name: Install dependencies
         run: |
-          pip install -U pip hatch rust-just
+          just install
       - name: Build docs
         run: |
           just docs
@@ -155,9 +156,11 @@ jobs:
           cache-dependency-path: 'pyproject.toml'
           # Build docs on lowest supported Python for furo
           python-version: '3.9'
+      - name: Install just
+        uses: extractions/setup-just@v2
       - name: Install dependencies
         run: |
-          pip install -U pip hatch rust-just
+          just install
       - name: Build docs
         run: |
           just docs-linkcheck
@@ -177,9 +180,11 @@ jobs:
           python-version: "${{matrix.python}}"
           cache: 'pip'
           cache-dependency-path: 'pyproject.toml'
+      - name: Install just
+        uses: extractions/setup-just@v2
       - name: Install dependencies
         run: |
-          pip install -U pip hatch rust-just
+          just install
       - name: Run typecheck
         run: |
           just typing

.github/workflows/test-python.yml

@@ -25,6 +25,8 @@ libmongocrypt/
 libmongocrypt_git/
 hatch_config.toml
 .venv
+expansion.yml
+.evergreen/scripts/env.sh
 
 # Lambda temp files
 test/lambda/.aws-sam

.gitignore

@@ -158,6 +158,7 @@ pre-commit run --all-files --hook-stage manual ruff
 Typically we use `just` to run the linters, e.g.
 
 ```bash
+just install  # this will install a venv with pre-commit installed, and install the pre-commit hook.
 just typing-mypy
 just run lint-manual
 ```
@@ -194,7 +195,8 @@ the pages will re-render and the browser will automatically refresh.
 
 -   Ensure you have started the appropriate Mongo Server(s).
 -   Run `just install` to set up `hatch` in a local virtual environment, or you can manually
-    create a virtual environment and run `pytest` directly.
+    create a virtual environment and run `pytest` directly.  If you want to use a specific
+    version of Python, remove the `.venv` folder and set `PYTHON_BINARY` before running `just install`.
 -   Run `just test` or `pytest` to run all of the tests.
 -   Append `test/<mod_name>.py::<class_name>::<test_name>` to run
     specific tests. You can omit the `<test_name>` to test a full class

CONTRIBUTING.md

@@ -3,57 +3,62 @@ set shell := ["bash", "-c"]
 set dotenv-load
 set dotenv-filename := "./.evergreen/scripts/env.sh"
 
+# Handle cross-platform paths to local python cli tools.
+python_bin_dir := if os_family() == "windows" { "./.venv/Scripts" } else { "./.venv/bin" }
+hatch_bin := python_bin_dir + "/hatch"
+pre_commit_bin := python_bin_dir + "/pre-commit"
+
 # Make the default recipe private so it doesn't show up in the list.
 [private]
 default:
   @just --list
 
 install:
-    bash .evergreen/scripts/ensure-hatch.sh
+   bash .evergreen/scripts/setup-dev-env.sh
 
 [group('docs')]
 docs:
-    source .evergreen/scripts/ensure-hatch.sh && hatch run doc:build
+    {{hatch_bin}} run doc:build
 
 [group('docs')]
 docs-serve:
-    source .evergreen/scripts/ensure-hatch.sh && hatch run doc:serve
+    {{hatch_bin}} run doc:serve
 
 [group('docs')]
 docs-linkcheck:
-    hatch run doc:linkcheck
+    {{hatch_bin}} run doc:linkcheck
 
 [group('docs')]
 docs-test:
-    source .evergreen/scripts/ensure-hatch.sh && hatch run doctest:test
+    {{hatch_bin}} run doctest:test
 
 [group('typing')]
 typing:
-    source .evergreen/scripts/ensure-hatch.sh && hatch run typing:check
+    {{hatch_bin}} run typing:check
 
 [group('typing')]
 typing-mypy:
-    source .evergreen/scripts/ensure-hatch.sh && hatch run typing:mypy
+    {{hatch_bin}} run typing:mypy
 
 [group('lint')]
 lint:
-    pre-commit run --all-files
+    {{pre_commit_bin}} run --all-files
 
 [group('lint')]
 lint-manual:
-    pre-commit run --all-files --hook-stage manual
+    {{pre_commit_bin}} run --all-files --hook-stage manual
 
 [group('test')]
 test *args:
-    source .evergreen/scripts/ensure-hatch.sh && hatch run test:test {{args}}
+    {{hatch_bin}} run test:test {{args}}
 
 [group('test')]
 test-mockupdb:
-    source .evergreen/scripts/ensure-hatch.sh && hatch run test:test-mockupdb
+    {{hatch_bin}} run test:test-mockupdb
 
 [group('test')]
 test-eg *args:
-    source .evergreen/scripts/ensure-hatch.sh && hatch run test:test-eg {{args}}
+    {{hatch_bin}} run test:test-eg {{args}}
 
 [group('encryption')]
 setup-encryption: