diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 5e2d97bcd..1e1f711c5 100755 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -306,7 +306,7 @@ functions: script: | set -ex cd ./libmongocrypt/bindings/python - PYTHON=${PYTHON} ./release.sh + PYTHON=${PYTHON} ./scripts/release.sh "upload python release": - command: archive.targz_pack diff --git a/.github/workflows/codeql-python.yml b/.github/workflows/codeql-python.yml index b8964e2a0..2453b94f5 100644 --- a/.github/workflows/codeql-python.yml +++ b/.github/workflows/codeql-python.yml @@ -55,9 +55,9 @@ jobs: - name: Install package run: | cd bindings/python - export LIBMONGOCRYPT_VERSION=$(cat ./libmongocrypt-version.txt) + export LIBMONGOCRYPT_VERSION=$(cat ./scripts/libmongocrypt-version.txt) git fetch origin $LIBMONGOCRYPT_VERSION - bash release.sh + bash ./scripts/release.sh pip install dist/*.whl - name: Perform CodeQL Analysis diff --git a/.github/workflows/dist-python.yml b/.github/workflows/dist-python.yml index dd278d4ea..b25b84c18 100644 --- a/.github/workflows/dist-python.yml +++ b/.github/workflows/dist-python.yml @@ -54,9 +54,9 @@ jobs: - name: Build and test dist files run: | - export LIBMONGOCRYPT_VERSION=$(cat ./libmongocrypt-version.txt) + export LIBMONGOCRYPT_VERSION=$(cat ./scripts/libmongocrypt-version.txt) git fetch origin $LIBMONGOCRYPT_VERSION - bash ./release.sh + bash ./scripts/release.sh - uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 961127312..b818692be 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -40,6 +40,7 @@ jobs: if: github.repository_owner == 'mongodb' runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] python-version: ["3.8", "3.13"] @@ -59,6 +60,6 @@ jobs: if [ "${{ matrix.python-version }}" == "3.13" ]; then export PIP_PRE=1 fi - export LIBMONGOCRYPT_VERSION=$(cat ./libmongocrypt-version.txt) + export LIBMONGOCRYPT_VERSION=$(cat ./scripts/libmongocrypt-version.txt) git fetch origin $LIBMONGOCRYPT_VERSION - bash ./release.sh + bash ./scripts/release.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bbd2bbbd1..9d3a41a30 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,6 +53,18 @@ repos: language: system types: [shell] +- repo: local + hooks: + - id: synchro + name: synchro + entry: bash ./bindings/python/scripts/synchro.sh + language: python + require_serial: true + fail_fast: true + additional_dependencies: + - ruff==0.1.3 + - unasync + - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. rev: v0.1.3 diff --git a/bindings/python/.evergreen/test.sh b/bindings/python/.evergreen/test.sh index 0e5b68222..e49577c6f 100755 --- a/bindings/python/.evergreen/test.sh +++ b/bindings/python/.evergreen/test.sh @@ -86,7 +86,7 @@ for PYTHON_BINARY in "${PYTHONS[@]}"; do done # Verify the sbom file -LIBMONGOCRYPT_VERSION=$(cat ./libmongocrypt-version.txt) +LIBMONGOCRYPT_VERSION=$(cat ./scripts/libmongocrypt-version.txt) EXPECTED="pkg:github/mongodb/libmongocrypt@$LIBMONGOCRYPT_VERSION" if grep -q $EXPECTED sbom.json; then echo "SBOM is up to date!" diff --git a/bindings/python/CONTRIBUTING.md b/bindings/python/CONTRIBUTING.md new file mode 100644 index 000000000..a1a598368 --- /dev/null +++ b/bindings/python/CONTRIBUTING.md @@ -0,0 +1,34 @@ +# Contributing to PyMongoCrypt + +## Asyncio considerations +PyMongoCrypt adds asyncio capability by modifying the source files in */asynchronous to */synchronous using [unasync](https://github.com/python-trio/unasync/) and some custom transforms. + +Where possible, edit the code in `*/asynchronous/*.py` and not the synchronous files. You can run `pre-commit run --all-files synchro` before running tests if you are testing synchronous code. + +To prevent the synchro hook from accidentally overwriting code, it first checks to see whether a sync version of a file is changing and not its async counterpart, and will fail. In the unlikely scenario that you want to override this behavior, first export `OVERRIDE_SYNCHRO_CHECK=1`. + +Sometimes, the synchro hook will fail and introduce changes many previously unmodified files. This is due to static Python errors, such as missing imports, incorrect syntax, or other fatal typos. To resolve these issues, run `pre-commit run --all-files --hook-stage manual ruff` and fix all reported errors before running the synchro hook again. + +## Updating the libmongocrypt bindings + +To update the libmongocrypt bindings in `pymongocrypt/binding.py`, run the following script: + +```bash +python scripts/update_binding.py +``` + +## Update the bundled version of libmongocrypt + +To update the bundled version of libmongocrypt, run the following script: + +```bash +bash script/update-version.sh +``` + +This will set the version in `scripts/libmongocrypt-version.sh` and update `sbom.json` to reflect +the new vendored version of `libmongocrypt`. + +## Building wheels + +To build wheels, run `scripts/release.sh`. It will build the appropriate wheel for the current system +on Windows and MacOS. If docker is available on Linux or MacOS, it will build the manylinux wheels. diff --git a/bindings/python/pymongocrypt/binding.py b/bindings/python/pymongocrypt/binding.py index fe371a52a..c1bd543cf 100644 --- a/bindings/python/pymongocrypt/binding.py +++ b/bindings/python/pymongocrypt/binding.py @@ -29,7 +29,7 @@ def _parse_version(version): ffi = cffi.FFI() -# Generated with strip_header.py +# Start embedding from update_binding.py ffi.cdef( """/* * Copyright 2019-present MongoDB, Inc. @@ -1468,6 +1468,7 @@ def _parse_version(version): // DEPRECATED: Support "rangePreview" has been removed in favor of "range". """ ) +# End embedding from update_binding.py def _to_string(cdata): diff --git a/bindings/python/build-manylinux-wheel.sh b/bindings/python/scripts/build-manylinux-wheel.sh similarity index 100% rename from bindings/python/build-manylinux-wheel.sh rename to bindings/python/scripts/build-manylinux-wheel.sh diff --git a/bindings/python/libmongocrypt-version.txt b/bindings/python/scripts/libmongocrypt-version.txt similarity index 100% rename from bindings/python/libmongocrypt-version.txt rename to bindings/python/scripts/libmongocrypt-version.txt diff --git a/bindings/python/release.sh b/bindings/python/scripts/release.sh similarity index 93% rename from bindings/python/release.sh rename to bindings/python/scripts/release.sh index 684b172b0..37bdced4e 100755 --- a/bindings/python/release.sh +++ b/bindings/python/scripts/release.sh @@ -15,14 +15,19 @@ set -o xtrace # Write all commands first to stderr set -o errexit # Exit the script with error if any of the commands fail +SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) + # The libmongocrypt git revision release to embed in our wheels. -LIBMONGOCRYPT_VERSION=$(cat ./libmongocrypt-version.txt) +LIBMONGOCRYPT_VERSION=$(cat $SCRIPT_DIR/libmongocrypt-version.txt) REVISION=$(git rev-list -n 1 $LIBMONGOCRYPT_VERSION) # The libmongocrypt release branch. -BRANCH="r1.12" +MINOR_VERSION=$(echo $LIBMONGOCRYPT_VERSION | cut -d. -f1,2) +BRANCH="r${MINOR_VERSION}" # The python executable to use. PYTHON=${PYTHON:-python} +pushd $SCRIPT_DIR/.. + # Clean slate. rm -rf dist .venv build libmongocrypt pymongocrypt/*.so pymongocrypt/*.dll pymongocrypt/*.dylib @@ -48,7 +53,7 @@ function build_wheel() { function build_manylinux_wheel() { python -m pip install unasync docker pull $1 - docker run --rm -v `pwd`:/python $1 /python/build-manylinux-wheel.sh + docker run --rm -v `pwd`:/python $1 /python/scripts/build-manylinux-wheel.sh # Sudo is needed to remove the files created by docker. sudo rm -rf build libmongocrypt pymongocrypt/*.so pymongocrypt/*.dll pymongocrypt/*.dylib } @@ -126,3 +131,4 @@ if [ $(command -v docker) ]; then fi ls -ltr dist +popd diff --git a/bindings/python/synchro.py b/bindings/python/scripts/synchro.py similarity index 69% rename from bindings/python/synchro.py rename to bindings/python/scripts/synchro.py index a9a6ba686..26737ccbc 100644 --- a/bindings/python/synchro.py +++ b/bindings/python/scripts/synchro.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +import sys from os import listdir from pathlib import Path @@ -27,12 +29,14 @@ "aclose": "close", } -_base = "pymongocrypt" +ROOT = Path(__file__).absolute().parent.parent + +_base = ROOT / "pymongocrypt" async_files = [ - f"./{_base}/asynchronous/{f}" - for f in listdir("pymongocrypt/asynchronous") - if (Path(_base) / "asynchronous" / f).is_file() + f"{_base}/asynchronous/{f}" + for f in listdir(f"{_base}/asynchronous") + if (_base / "asynchronous" / f).is_file() ] @@ -40,20 +44,23 @@ async_files, [ Rule( - fromdir="/pymongocrypt/asynchronous/", - todir="/pymongocrypt/synchronous/", + fromdir=f"{_base}/asynchronous/", + todir=f"{_base}/synchronous/", additional_replacements=replacements, ) ], ) sync_files = [ - f"./{_base}/synchronous/{f}" - for f in listdir("pymongocrypt/synchronous") - if (Path(_base) / "synchronous" / f).is_file() + f"{_base}/synchronous/{f}" + for f in listdir(f"{_base}/synchronous") + if (_base / "synchronous" / f).is_file() ] +modified_files = [f"./{f}" for f in sys.argv[1:]] for file in sync_files: + if file in modified_files and "OVERRIDE_SYNCHRO_CHECK" not in os.environ: + raise ValueError(f"Refusing to overwrite {file}") with open(file, "r+") as f: lines = f.readlines() for i in range(len(lines)): diff --git a/bindings/python/scripts/synchro.sh b/bindings/python/scripts/synchro.sh new file mode 100755 index 000000000..70efcadac --- /dev/null +++ b/bindings/python/scripts/synchro.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -eu + +SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) + +python $SCRIPT_DIR/synchro.py "$@" +python -m ruff check $SCRIPT_DIR/../pymongocrypt/synchronous --fix --silent diff --git a/bindings/python/update-sbom.sh b/bindings/python/scripts/update-version.sh similarity index 50% rename from bindings/python/update-sbom.sh rename to bindings/python/scripts/update-version.sh index e0c903149..3bdb7b05e 100755 --- a/bindings/python/update-sbom.sh +++ b/bindings/python/scripts/update-version.sh @@ -1,8 +1,19 @@ #!/bin/bash -set -eux +set -eu -LIBMONGOCRYPT_VERSION=$(cat ./libmongocrypt-version.txt) +SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) + +if [ -z "${1:-}" ]; then + echo "Provide the new version of libmongocrypt!" + exit 1 +fi + +LIBMONGOCRYPT_VERSION=$1 + +echo $LIBMONGOCRYPT_VERSION > libmongocrypt-version.txt + +pushd $SCRIPT_DIR/.. if [ $(command -v podman) ]; then DOCKER=podman else @@ -10,5 +21,7 @@ else fi echo "pkg:github/mongodb/libmongocrypt@$LIBMONGOCRYPT_VERSION" > purls.txt -$DOCKER run --platform="linux/amd64" -it --rm -v $(pwd):$(pwd) artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:1.0 update --purls=$(pwd)/purls.txt -o $(pwd)/sbom.json +$DOCKER run --platform="linux/amd64" -it --rm -v $(pwd):$(pwd) artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:2.0 update --purls=$(pwd)/purls.txt -o $(pwd)/sbom.json rm purls.txt + +popd diff --git a/bindings/python/scripts/update_binding.py b/bindings/python/scripts/update_binding.py new file mode 100644 index 000000000..0f1c7c89c --- /dev/null +++ b/bindings/python/scripts/update_binding.py @@ -0,0 +1,76 @@ +# Copyright 2019-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Update pymongocrypt/bindings.py using mongocrypt.h. +""" + +import re +from pathlib import Path + +DROP_RE = re.compile(r"^\s*(#|MONGOCRYPT_EXPORT)") +HERE = Path(__file__).absolute().parent + + +# itertools.pairwise backport for Python 3.9 support. +def pairwise(iterable): + # pairwise('ABCDEFG') → AB BC CD DE EF FG + + iterator = iter(iterable) + a = next(iterator, None) + + for b in iterator: + yield a, b + a = b + + +def strip_file(content): + fold = content.replace("\\\n", " ") + all_lines = [*fold.split("\n"), ""] + keep_lines = (line for line in all_lines if not DROP_RE.match(line)) + fin = "" + for line, peek in pairwise(keep_lines): + if peek == "" and line == "": + # Drop adjacent empty lines + continue + yield line + fin = peek + yield fin + + +def update_bindings(): + header_file = HERE.parent.parent.parent / "src/mongocrypt.h" + with header_file.open(encoding="utf-8") as fp: + header_lines = strip_file(fp.read()) + + target = HERE.parent / "pymongocrypt/binding.py" + source_lines = target.read_text().splitlines() + new_lines = [] + skip = False + for line in source_lines: + if not skip: + new_lines.append(line) + if line.strip() == "# Start embedding from update_binding.py": + skip = True + new_lines.append("ffi.cdef(") + new_lines.append('"""') + new_lines.extend(header_lines) + if line.strip() == "# End embedding from update_binding.py": + new_lines.append('"""') + new_lines.append(")") + new_lines.append(line) + skip = False + + +if __name__ == "__main__": + update_bindings() diff --git a/bindings/python/strip_header.py b/bindings/python/strip_header.py deleted file mode 100644 index fcc5426fc..000000000 --- a/bindings/python/strip_header.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2019-present MongoDB, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Generate a CFFI.cdef() string from a C header file - -Usage (on macOS):: python strip_header.py ../../src/mongocrypt.h | pbcopy -""" - -import itertools -import re -import sys - -DROP_RE = re.compile(r"^\s*(#|MONGOCRYPT_EXPORT)") - - -def strip_file(content): - fold = content.replace("\\\n", " ") - all_lines = [*fold.split("\n"), ""] - keep_lines = (line for line in all_lines if not DROP_RE.match(line)) - fin = "" - for line, peek in itertools.pairwise(keep_lines): - if peek == "" and line == "": - # Drop adjacent empty lines - continue - yield line - fin = peek - yield fin - - -def strip(hdr): - with open(hdr) as fp: - out = strip_file(fp.read()) - print("\n".join(out)) # noqa: T201 - - -if __name__ == "__main__": - if len(sys.argv) != 2: - raise Exception("Usage: strip_header.py header.h") - strip(sys.argv[1])