PYTHON-4953 Add contributing guide and improve helper scripts (#980)

blink1073 · web-flow · commit caf71767d206 · 2025-03-05T13:38:11.000-06:00
diff --git a/.evergreen/config.yml b/.evergreen/config.yml
@@ -306,7 +306,7 @@ functions:
         script: |
           set -ex
           cd ./libmongocrypt/bindings/python
-          PYTHON=${PYTHON} ./release.sh
+          PYTHON=${PYTHON} ./scripts/release.sh
 
   "upload python release":
     - command: archive.targz_pack
diff --git a/.github/workflows/codeql-python.yml b/.github/workflows/codeql-python.yml
@@ -55,9 +55,9 @@ jobs:
     - name: Install package
       run: |
         cd bindings/python
-        export LIBMONGOCRYPT_VERSION=$(cat ./libmongocrypt-version.txt)
+        export LIBMONGOCRYPT_VERSION=$(cat ./scripts/libmongocrypt-version.txt)
         git fetch origin $LIBMONGOCRYPT_VERSION
-        bash release.sh
+        bash ./scripts/release.sh
         pip install dist/*.whl
 
     - name: Perform CodeQL Analysis
diff --git a/.github/workflows/dist-python.yml b/.github/workflows/dist-python.yml
@@ -54,9 +54,9 @@ jobs:
 
       - name: Build and test dist files
         run: |
-          export LIBMONGOCRYPT_VERSION=$(cat ./libmongocrypt-version.txt)
+          export LIBMONGOCRYPT_VERSION=$(cat ./scripts/libmongocrypt-version.txt)
           git fetch origin $LIBMONGOCRYPT_VERSION
-          bash ./release.sh
+          bash ./scripts/release.sh
 
       - uses: actions/upload-artifact@v4
         with:
diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml
@@ -40,6 +40,7 @@ jobs:
     if: github.repository_owner == 'mongodb'
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
         python-version: ["3.8", "3.13"]
@@ -59,6 +60,6 @@ jobs:
           if [ "${{ matrix.python-version }}" == "3.13" ]; then 
             export PIP_PRE=1
           fi
-          export LIBMONGOCRYPT_VERSION=$(cat ./libmongocrypt-version.txt)
+          export LIBMONGOCRYPT_VERSION=$(cat ./scripts/libmongocrypt-version.txt)
           git fetch origin $LIBMONGOCRYPT_VERSION
-          bash ./release.sh
+          bash ./scripts/release.sh
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -53,6 +53,18 @@ repos:
     language: system
     types: [shell]
 
+- repo: local
+  hooks:
+    - id: synchro
+      name: synchro
+      entry: bash ./bindings/python/scripts/synchro.sh
+      language: python
+      require_serial: true
+      fail_fast: true
+      additional_dependencies:
+        - ruff==0.1.3
+        - unasync
+
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
   rev: v0.1.3
diff --git a/bindings/python/.evergreen/test.sh b/bindings/python/.evergreen/test.sh
@@ -86,7 +86,7 @@ for PYTHON_BINARY in "${PYTHONS[@]}"; do
 done
 
 # Verify the sbom file
-LIBMONGOCRYPT_VERSION=$(cat ./libmongocrypt-version.txt)
+LIBMONGOCRYPT_VERSION=$(cat ./scripts/libmongocrypt-version.txt)
 EXPECTED="pkg:github/mongodb/libmongocrypt@$LIBMONGOCRYPT_VERSION"
 if grep -q $EXPECTED sbom.json; then
   echo "SBOM is up to date!"
diff --git a/bindings/python/CONTRIBUTING.md b/bindings/python/CONTRIBUTING.md
@@ -0,0 +1,34 @@
+# Contributing to PyMongoCrypt
+
+## Asyncio considerations
+PyMongoCrypt adds asyncio capability by modifying the source files in */asynchronous to */synchronous using [unasync](https://github.com/python-trio/unasync/) and some custom transforms.
+
+Where possible, edit the code in `*/asynchronous/*.py` and not the synchronous files. You can run `pre-commit run --all-files synchro` before running tests if you are testing synchronous code.
+
+To prevent the synchro hook from accidentally overwriting code, it first checks to see whether a sync version of a file is changing and not its async counterpart, and will fail. In the unlikely scenario that you want to override this behavior, first export `OVERRIDE_SYNCHRO_CHECK=1`.
+
+Sometimes, the synchro hook will fail and introduce changes many previously unmodified files. This is due to static Python errors, such as missing imports, incorrect syntax, or other fatal typos. To resolve these issues, run `pre-commit run --all-files --hook-stage manual ruff` and fix all reported errors before running the synchro hook again.
+
+## Updating the libmongocrypt bindings
+
+To update the libmongocrypt bindings in `pymongocrypt/binding.py`, run the following script:
+
+```bash
+python scripts/update_binding.py
+```
+
+## Update the bundled version of libmongocrypt
+
+To update the bundled version of libmongocrypt, run the following script:
+
+```bash
+bash script/update-version.sh <new-version>
+```
+
+This will set the version in `scripts/libmongocrypt-version.sh` and update `sbom.json` to reflect
+the new vendored version of `libmongocrypt`.
+
+## Building wheels
+
+To build wheels, run `scripts/release.sh`.  It will build the appropriate wheel for the current system
+on Windows and MacOS.  If docker is available on Linux or MacOS, it will build the manylinux wheels.
diff --git a/bindings/python/pymongocrypt/binding.py b/bindings/python/pymongocrypt/binding.py
@@ -29,7 +29,7 @@ def _parse_version(version):
 
 ffi = cffi.FFI()
 
-# Generated with strip_header.py
+# Start embedding from update_binding.py
 ffi.cdef(
     """/*
  * Copyright 2019-present MongoDB, Inc.
@@ -1468,6 +1468,7 @@ def _parse_version(version):
 // DEPRECATED: Support "rangePreview" has been removed in favor of "range".
 """
 )
+# End embedding from update_binding.py
 
 
 def _to_string(cdata):
diff --git a/bindings/python/scripts/build-manylinux-wheel.sh b/bindings/python/scripts/build-manylinux-wheel.sh
diff --git a/bindings/python/scripts/libmongocrypt-version.txt b/bindings/python/scripts/libmongocrypt-version.txt
diff --git a/bindings/python/scripts/release.sh b/bindings/python/scripts/release.sh
@@ -15,14 +15,19 @@
 set -o xtrace   # Write all commands first to stderr
 set -o errexit  # Exit the script with error if any of the commands fail
 
+SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0})
+
 # The libmongocrypt git revision release to embed in our wheels.
-LIBMONGOCRYPT_VERSION=$(cat ./libmongocrypt-version.txt)
+LIBMONGOCRYPT_VERSION=$(cat $SCRIPT_DIR/libmongocrypt-version.txt)
 REVISION=$(git rev-list -n 1 $LIBMONGOCRYPT_VERSION)
 # The libmongocrypt release branch.
-BRANCH="r1.12"
+MINOR_VERSION=$(echo $LIBMONGOCRYPT_VERSION | cut -d. -f1,2)
+BRANCH="r${MINOR_VERSION}"
 # The python executable to use.
 PYTHON=${PYTHON:-python}
 
+pushd $SCRIPT_DIR/..
+
 # Clean slate.
 rm -rf dist .venv build libmongocrypt pymongocrypt/*.so pymongocrypt/*.dll pymongocrypt/*.dylib
 
@@ -48,7 +53,7 @@ function build_wheel() {
 function build_manylinux_wheel() {
     python -m pip install unasync
     docker pull $1
-    docker run --rm -v `pwd`:/python $1 /python/build-manylinux-wheel.sh
+    docker run --rm -v `pwd`:/python $1 /python/scripts/build-manylinux-wheel.sh
     # Sudo is needed to remove the files created by docker.
     sudo rm -rf build libmongocrypt pymongocrypt/*.so pymongocrypt/*.dll pymongocrypt/*.dylib
 }
@@ -126,3 +131,4 @@ if [ $(command -v docker) ]; then
 fi
 
 ls -ltr dist
+popd
diff --git a/bindings/python/scripts/synchro.py b/bindings/python/scripts/synchro.py
@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+import sys
 from os import listdir
 from pathlib import Path
 
@@ -27,33 +29,38 @@
     "aclose": "close",
 }
 
-_base = "pymongocrypt"
+ROOT = Path(__file__).absolute().parent.parent
+
+_base = ROOT / "pymongocrypt"
 
 async_files = [
-    f"./{_base}/asynchronous/{f}"
-    for f in listdir("pymongocrypt/asynchronous")
-    if (Path(_base) / "asynchronous" / f).is_file()
+    f"{_base}/asynchronous/{f}"
+    for f in listdir(f"{_base}/asynchronous")
+    if (_base / "asynchronous" / f).is_file()
 ]
 
 
 unasync_files(
     async_files,
     [
         Rule(
-            fromdir="/pymongocrypt/asynchronous/",
-            todir="/pymongocrypt/synchronous/",
+            fromdir=f"{_base}/asynchronous/",
+            todir=f"{_base}/synchronous/",
             additional_replacements=replacements,
         )
     ],
 )
 
 sync_files = [
-    f"./{_base}/synchronous/{f}"
-    for f in listdir("pymongocrypt/synchronous")
-    if (Path(_base) / "synchronous" / f).is_file()
+    f"{_base}/synchronous/{f}"
+    for f in listdir(f"{_base}/synchronous")
+    if (_base / "synchronous" / f).is_file()
 ]
 
+modified_files = [f"./{f}" for f in sys.argv[1:]]
 for file in sync_files:
+    if file in modified_files and "OVERRIDE_SYNCHRO_CHECK" not in os.environ:
+        raise ValueError(f"Refusing to overwrite {file}")
     with open(file, "r+") as f:
         lines = f.readlines()
         for i in range(len(lines)):
diff --git a/bindings/python/scripts/synchro.sh b/bindings/python/scripts/synchro.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -eu
+
+SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0})
+
+python $SCRIPT_DIR/synchro.py "$@"
+python -m ruff check $SCRIPT_DIR/../pymongocrypt/synchronous --fix --silent
diff --git a/bindings/python/scripts/update-version.sh b/bindings/python/scripts/update-version.sh
@@ -1,14 +1,27 @@
 #!/bin/bash
 
-set -eux
+set -eu
 
-LIBMONGOCRYPT_VERSION=$(cat ./libmongocrypt-version.txt)
+SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0})
+
+if [ -z "${1:-}" ]; then
+    echo "Provide the new version of libmongocrypt!"
+    exit 1
+fi
+
+LIBMONGOCRYPT_VERSION=$1
+
+echo $LIBMONGOCRYPT_VERSION > libmongocrypt-version.txt
+
+pushd $SCRIPT_DIR/..
 if [ $(command -v podman) ]; then
     DOCKER=podman
 else
     DOCKER=docker
 fi
 
 echo "pkg:github/mongodb/libmongocrypt@$LIBMONGOCRYPT_VERSION" > purls.txt
-$DOCKER run --platform="linux/amd64" -it --rm -v $(pwd):$(pwd) artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:1.0 update --purls=$(pwd)/purls.txt -o $(pwd)/sbom.json
+$DOCKER run --platform="linux/amd64" -it --rm -v $(pwd):$(pwd) artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:2.0 update --purls=$(pwd)/purls.txt -o $(pwd)/sbom.json
 rm purls.txt
+
+popd
diff --git a/bindings/python/scripts/update_binding.py b/bindings/python/scripts/update_binding.py
@@ -0,0 +1,76 @@
+# Copyright 2019-present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Update pymongocrypt/bindings.py using mongocrypt.h.
+"""
+
+import re
+from pathlib import Path
+
+DROP_RE = re.compile(r"^\s*(#|MONGOCRYPT_EXPORT)")
+HERE = Path(__file__).absolute().parent
+
+
+# itertools.pairwise backport for Python 3.9 support.
+def pairwise(iterable):
+    # pairwise('ABCDEFG') → AB BC CD DE EF FG
+
+    iterator = iter(iterable)
+    a = next(iterator, None)
+
+    for b in iterator:
+        yield a, b
+        a = b
+
+
+def strip_file(content):
+    fold = content.replace("\\\n", " ")
+    all_lines = [*fold.split("\n"), ""]
+    keep_lines = (line for line in all_lines if not DROP_RE.match(line))
+    fin = ""
+    for line, peek in pairwise(keep_lines):
+        if peek == "" and line == "":
+            # Drop adjacent empty lines
+            continue
+        yield line
+        fin = peek
+    yield fin
+
+
+def update_bindings():
+    header_file = HERE.parent.parent.parent / "src/mongocrypt.h"
+    with header_file.open(encoding="utf-8") as fp:
+        header_lines = strip_file(fp.read())
+
+    target = HERE.parent / "pymongocrypt/binding.py"
+    source_lines = target.read_text().splitlines()
+    new_lines = []
+    skip = False
+    for line in source_lines:
+        if not skip:
+            new_lines.append(line)
+        if line.strip() == "# Start embedding from update_binding.py":
+            skip = True
+            new_lines.append("ffi.cdef(")
+            new_lines.append('"""')
+            new_lines.extend(header_lines)
+        if line.strip() == "# End embedding from update_binding.py":
+            new_lines.append('"""')
+            new_lines.append(")")
+            new_lines.append(line)
+            skip = False
+
+
+if __name__ == "__main__":
+    update_bindings()
diff --git a/bindings/python/strip_header.py b/bindings/python/strip_header.py
