ARROW-56 Improve Wheel Support (#67)

blink1073 · web-flow · commit 13805ff931d2 · 2022-04-07T21:48:59.000-05:00
diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml
@@ -9,101 +9,103 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build-non-linux-wheels:
-    runs-on: ${{ matrix.os }}
+  build_wheels:
+    name: Build wheel for ${{ matrix.python }}-${{ matrix.buildplat[1] }}
+    runs-on: ${{ matrix.buildplat[0] }}
     strategy:
+      # Ensure that a wheel builder finishes even if another fails
+      fail-fast: false
       matrix:
-        os: [macos-10.15, windows-latest]
-        python-version: [3.7, 3.8, 3.9, "3.10"]
-    name: Build CPython ${{ matrix.python-version }}-${{ matrix.os }}
+        # Github Actions doesn't support pairing matrix values together, let's improvise
+        # https://github.com/github/feedback/discussions/7835#discussioncomment-1769026
+        buildplat:
+        - [ubuntu-20.04, manylinux_x86_64]
+        - [macos-10.15, macosx_*]
+        - [windows-2019, win_amd64]
+        python: ["cp37", "cp38", "cp39", "cp310"]
+
     steps:
-      - uses: actions/checkout@v2
-      - name: Setup Python
-        uses: actions/setup-python@v2
+      - name: Checkout pymongoarrow
+        uses: actions/checkout@v2
         with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/setup.cfg'
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v2
+
+      - name: Install cibuildwheel
+        run: python -m pip install "cibuildwheel>=2.4,<3"
+
+      - name: Build MacOS Py38 Wheel
+        if: ${{ matrix.python == 'cp38' && matrix.buildplat[0] == 'macos-10.15' }}
+        working-directory: ./bindings/python
+        shell: bash
+        env:
+          CIBW_BUILD: cp38-macosx_x86_64
+          MACOSX_DEPLOYMENT_TARGET: "10.13"
+        run: python -m cibuildwheel --output-dir wheelhouse
+
       - name: Build wheels
+        if: ${{ matrix.python != 'cp38' || matrix.buildplat[0] != 'macos-10.15' }}
         working-directory: ./bindings/python
         shell: bash
-        run: |
-          ./release.sh
+        env:
+          CIBW_BUILD: ${{ matrix.python }}-${{ matrix.buildplat[1] }}
+          MACOSX_DEPLOYMENT_TARGET: "10.13"
+        run: python -m cibuildwheel --output-dir wheelhouse
+
       - uses: actions/upload-artifact@v2
         with:
-          name: pymongoarrow-${{ matrix.python-version }}-${{ matrix.os }}-wheel
-          path: ./bindings/python/dist/*.whl
+          name: ${{ matrix.python }}-${{ startsWith(matrix.buildplat[1], 'macosx') && 'macosx' || matrix.buildplat[1] }}
+          path: ./bindings/python/wheelhouse/*.whl
           if-no-files-found: error
-  test-non-linux-wheels:
-    needs: build-non-linux-wheels
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [macos-10.15, windows-latest]
-        python-version: [3.7, 3.8, 3.9, "3.10"]
-    name: Test CPython ${{ matrix.python-version }}-${{ matrix.os }}
+
+  make_sdist:
+    name: Make SDist
+    runs-on: ubuntu-latest
     steps:
-      - name: Setup Python
-        uses: actions/setup-python@v2
+      - uses: actions/checkout@v2
         with:
-          python-version: ${{ matrix.python-version }}
-      - name: Download a previously created wheel
-        uses: actions/download-artifact@v2
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v2
         with:
-          name: pymongoarrow-${{ matrix.python-version }}-${{ matrix.os }}-wheel
-      - name: Test wheel
-        shell: bash
+          # Build sdist on lowest supported Python
+          python-version: '3.7'
+
+      - name: Build SDist
+        working-directory: ./bindings/python
         run: |
-          python -m pip install -U pip
-          python -m pip install *.whl
-          python -c "from pymongoarrow.lib import process_bson_stream"
-  # Linux
-  build-manylinux-wheels:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        # Cannot have '/' in artifact names, so hardcode 'quay.io/pypa/'
-        container: ['manylinux2010_x86_64', 'manylinux2014_x86_64']
-        python-version: ['cp37-cp37m', 'cp38-cp38',
-                         'cp39-cp39', 'cp310-cp310']
-    name: Build CPython ${{ matrix.python-version }}-${{ matrix.container }}
-    steps:
-      - uses: actions/checkout@v2
-      - name: Build wheels
+          set -ex
+          python -m pip install -U pip build
+          export LIBBSON_INSTALL_DIR="$(pwd)/libbson"
+          ./build-libbson.sh
+          python -m build --sdist .
+
+      - name: Test Sdist
         working-directory: ./bindings/python
         run: |
-          docker run --rm --volume `pwd`:/python --workdir /python --env PLAT=${{ matrix.container }} --env PYTHON_BINARY=/opt/python/${{ matrix.python-version }}/bin/python quay.io/pypa/${{ matrix.container }} ./release.sh
+          export LIBBSON_INSTALL_DIR="$(pwd)/libbson"
+          python -m pip install dist/*.gz
+          cd ..
+          python -c "from pymongoarrow.lib import process_bson_stream"
+
       - uses: actions/upload-artifact@v2
         with:
-          name: pymongoarrow-${{ matrix.python-version }}-${{ matrix.container }}-wheel
-          path: ./bindings/python/wheelhouse/*.whl
-          if-no-files-found: error
-  test-manylinux-wheels:
-    runs-on: ubuntu-latest
-    needs: build-manylinux-wheels
-    strategy:
-      matrix:
-        container: ['manylinux2010_x86_64', 'manylinux2014_x86_64']
-        python-version: ['cp37-cp37m', 'cp38-cp38',
-                         'cp39-cp39', 'cp310-cp310']
-    name: Test CPython ${{ matrix.python-version }}-${{ matrix.container }}
-    steps:
-      - name: Download a previously created wheel
-        uses: actions/download-artifact@v2
-        with:
-          name: pymongoarrow-${{ matrix.python-version }}-${{ matrix.container }}-wheel
-      - name: Test wheel
-        run: |
-          docker run --rm --volume `pwd`:/python quay.io/pypa/${{ matrix.container }} /bin/bash -c "/opt/python/${{ matrix.python-version }}/bin/python -m pip install -U pip && /opt/python/${{ matrix.python-version }}/bin/python -m pip install /python/*.whl && /opt/python/${{ matrix.python-version }}/bin/python -c 'from pymongoarrow.lib import process_bson_stream'"
-  # Collect all built wheels
-  collect-wheels:
+          name: "sdist"
+          path: ./bindings/python/dist/*.tar.gz
+
+  collect-dist:
     runs-on: ubuntu-latest
-    needs: [build-non-linux-wheels, build-manylinux-wheels]
+    needs: [build_wheels, make_sdist]
     name: Download Wheels
     steps:
       - name: Download all workflow run artifacts
         uses: actions/download-artifact@v2
+      - name: Flatten directory
+        run: |
+          find . -mindepth 2 -type f -exec mv {} . \;
+          find . -type d -empty -delete
       - uses: actions/upload-artifact@v2
         with:
-          name: nix-wheels
-          path: "*-wheel"
+          name: all-dist
+          path: "./*"
diff --git a/bindings/python/MANIFEST.in b/bindings/python/MANIFEST.in
@@ -1,6 +1,6 @@
 include README.rst
 include LICENSE
-include build-libbson.sh
+include *.sh
 include pyproject.toml
 
 exclude THIRD-PARTY-NOTICES
diff --git a/bindings/python/build-libbson.sh b/bindings/python/build-libbson.sh
@@ -19,26 +19,39 @@ then
   git clone --depth 1 -b "$LIBBSON_VERSION" https://github.com/mongodb/mongo-c-driver.git "$WORKDIR"
 fi
 
+echo "Installing libbson..."
+
+MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET:-"10.15"}
+CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES:-"x86_64"}
+
 # Directory where build artifacts will be placed
 LIBBSON_INSTALL_DIR=${LIBBSON_INSTALL_DIR:-""}
 
+# Replace a relative path with an absolute one for cmake
+LIBBSON_INSTALL_DIR="$(cd "$(dirname "$LIBBSON_INSTALL_DIR")"; pwd)/$(basename "$LIBBSON_INSTALL_DIR")"
+
+echo "MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET}"
+echo "CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}"
+echo "LIBBSON_INSTALL_DIR=${LIBBSON_INSTALL_DIR}"
+
 # Build libbson
 pushd "$WORKDIR"
   git checkout "$LIBBSON_VERSION"
   mkdir -p cmake-build
   pushd cmake-build
     if [ -n "$LIBBSON_INSTALL_DIR" ]
     then
-      echo "Installing libbson in $LIBBSON_INSTALL_DIR"
       cmake -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF \
             -DENABLE_MONGOC=OFF \
-            -DCMAKE_OSX_DEPLOYMENT_TARGET="10.9" \
+            -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \
+            -DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET} \
             -DCMAKE_INSTALL_PREFIX:PATH="$LIBBSON_INSTALL_DIR" \
             ..
     else
       cmake -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF \
+            -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \
+            -DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET} \
             -DENABLE_MONGOC=OFF \
-            -DCMAKE_OSX_DEPLOYMENT_TARGET="10.9" \
             ..
     fi
     cmake --build . --target clean
diff --git a/bindings/python/cibw_before_build.sh b/bindings/python/cibw_before_build.sh
@@ -0,0 +1,27 @@
+#!/bin/bash -ex
+
+set -o xtrace
+set -o errexit
+
+# Handle architectures from cibuildwheel.
+# Set CMAKE_OSX_ARCHITECTURES for libbson.
+# Get the appropriate version of pyarrow for macos.
+
+if [[ "$CIBW_BUILD" == *"macosx_"* ]]
+then
+  mac_version="${MACOSX_DEPLOYMENT_TARGET/\./_}"
+  if [[ "$ARCHFLAGS" == *"arm64"* ]]
+  then
+    platform="macosx_${mac_version}_universal2"
+    export CMAKE_OSX_ARCHITECTURES="arm64;x86_64"
+  else
+    platform="macosx_${mac_version}_x86_64"
+  fi
+
+  # Install pyarrow with the appropriate platform.
+  pip install --platform $platform --target $HOME/wheels --no-deps --only-binary=:all: pyarrow
+fi
+
+
+# Build libbson with the appropriate arch.
+./build-libbson.sh
diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml
@@ -6,3 +6,19 @@ requires = [
     # Must be kept in sync with the `install_requires` in `setup.cfg`
     "pyarrow>=7.0.0,<7.1.0",
 ]
+
+[tool.cibuildwheel]
+skip = "pp* *-manylinux_i686 *_ppc64le *_s390x *-musllinux*"
+before-build = "bash ./cibw_before_build.sh"
+build-frontend = "build"
+test-command = "python -c \"from pymongoarrow.lib import process_bson_stream\""
+
+[tool.cibuildwheel.environment]
+LIBBSON_INSTALL_DIR = "./libbson"
+
+[tool.cibuildwheel.linux]
+manylinux-x86_64-image = "manylinux2014"
+
+[tool.cibuildwheel.macos]
+archs = "x86_64 universal2"
+test-skip = "*universal2:arm64"
diff --git a/bindings/python/setup.py b/bindings/python/setup.py
@@ -37,6 +37,7 @@ def append_libbson_flags(module):
     pc_path = "libbson-1.0"
     install_dir = os.environ.get("LIBBSON_INSTALL_DIR")
     if install_dir:
+        install_dir = os.path.abspath(install_dir)
         # Handle the copy-able library file if applicable.
         if COPY_LIBBSON:
             if platform == "darwin":
@@ -56,6 +57,10 @@ def append_libbson_flags(module):
         lib_dirs = glob.glob(os.path.join(install_dir, "lib*"))
         if len(lib_dirs) != 1:
             warnings.warn(f"Unable to locate libbson in {install_dir}")
+            if IS_WIN:
+                raise ValueError(
+                    "We require a LIBBSON_INSTALL_DIR with a compiled library on Windows"
+                )
         else:
             lib_dir = lib_dirs[0]
             if IS_WIN:
@@ -84,6 +89,7 @@ def append_libbson_flags(module):
         raise ValueError(f'Could not find "{pc_path}" library')
 
     cflags = query_pkgconfig("pkg-config --cflags {}".format(pc_path))
+
     if cflags:
         orig_cflags = os.environ.get("CFLAGS", "")
         os.environ["CFLAGS"] = cflags + " " + orig_cflags
@@ -174,6 +180,7 @@ def get_extension_modules():
             module.extra_link_args += ["-rpath", "@loader_path"]
         elif platform == "linux":
             module.extra_link_args += ["-Wl,-rpath,$ORIGIN"]
+
     return modules
 
 
