Add cibuildwheel with Linux AArch64 wheel build support (#2672)

Co-authored-by: wjsi <[email protected]>

Author: odidev

.github/workflows/build-wheels.sh

@@ -7,67 +7,72 @@ on:
 
 jobs:
   build:
+    name: Build wheels on ${{ matrix.os }} for ${{ matrix.arch }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macOS-latest, windows-latest]
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
-        node-version: ["14.x"]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        arch: [auto]
         include:
-          - { os: ubuntu-latest, python-version: "3.7", python-abis: "cp37-cp37m" }
-          - { os: ubuntu-latest, python-version: "3.8", python-abis: "cp38-cp38" }
-          - { os: ubuntu-latest, python-version: "3.9", python-abis: "cp39-cp39" }
-          - { os: ubuntu-latest, python-version: "3.10", python-abis: "cp310-cp310" }
-          - { os: windows-latest, python-version: "3.10", build-static: 1 }
+          - os: ubuntu-latest
+            arch: aarch64
+          - os: macos-latest
+            arch: universal2
 
     steps:
       - name: Check out code
         uses: actions/checkout@v2
 
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v2
-        with:
-          node-version: ${{ matrix.node-version }}
+      - name: Set up QEMU
+        if: ${{ matrix.arch == 'aarch64' }}
+        uses: docker/setup-qemu-action@v1
 
-      - name: Set up conda ${{ matrix.python-version }}
+      - name: Build wheels
+        uses: pypa/[email protected]
         env:
-          PYTHON: ${{ matrix.python-version }}
-        shell: bash
-        run: |
-          source ./ci/install-conda.sh
-          python -m pip install --upgrade pip setuptools wheel coverage;
+          CIBW_BEFORE_BUILD: pip install -r ci/requirements-wheel.txt
+          CIBW_ARCHS: ${{ matrix.arch }}
+          CIBW_SKIP: pp* *-musllinux* cp36-* cp310-win32 *i686 cp310-manylinux_x86_64
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux1
+          CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014
 
-      - name: Install dependencies
+      - name: Build wheels (Python 3.10 x86_64)
+        if: ${{ matrix.os == 'ubuntu-latest' && matrix.arch == 'auto'}}
+        uses: pypa/[email protected]
         env:
-          WITH_HADOOP: ${{ matrix.with-hadoop }}
-          WITH_KUBERNETES: ${{ matrix.with-kubernetes }}
-          NO_COMMON_TESTS: ${{ matrix.no-common-tests }}
-        shell: bash
-        run: |
-          source ./ci/reload-env.sh
-          export DEFAULT_VENV=$VIRTUAL_ENV
+          CIBW_BEFORE_BUILD: pip install -r ci/requirements-wheel.txt
+          CIBW_BUILD: cp310-manylinux_x86_64
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
 
-          if [[ "$PYTHON" =~ "3.9" ]]; then
-            conda install -n test --quiet --yes -c conda-forge python=$PYTHON pyarrow
-          fi
-
-          pip install numpy scipy cython
-          pip install -e ".[dev,extra]"
-          conda list -n test
+      - name: Build source
+        if: ${{ matrix.os == 'ubuntu-latest' && matrix.arch == 'auto'}}
+        run: |
+          pip install -r ci/requirements-wheel.txt
+          python setup.py sdist --formats=gztar --dist-dir=./wheelhouse
 
-      - name: Deploy packages
-        if: startsWith(github.ref, 'refs/tags/') && matrix.no-deploy != '1'
+      - name: Release to pypi
         shell: bash
         env:
-          PYABI: ${{ matrix.python-abis }}
-          BUILD_STATIC: ${{ matrix.build-static }}
           PYPI_PWD: ${{ secrets.PYPI_PASSWORD }}
         run: |
-          source ./ci/reload-env.sh
-          if [[ "$PYTHON" =~ "3.10" ]]; then
-            export DOCKER_IMAGE="quay.io/pypa/manylinux2014_x86_64"
+          if [[ "$GITHUB_REPOSITORY" == "mars-project/mars" ]]; then
+            PYPI_REPO="https://upload.pypi.org/legacy/"
           else
-            export DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64"
+            PYPI_REPO="https://test.pypi.org/legacy/"
           fi
-          source ./.github/workflows/upload-packages.sh
+          echo "[distutils]"             > ~/.pypirc
+          echo "index-servers ="        >> ~/.pypirc
+          echo "    pypi"               >> ~/.pypirc
+          echo "[pypi]"                 >> ~/.pypirc
+          echo "repository=$PYPI_REPO"  >> ~/.pypirc
+          echo "username=pyodps"        >> ~/.pypirc
+          echo "password=$PYPI_PWD"     >> ~/.pypirc
+          python -m pip install twine
+          python -m twine upload -r pypi --skip-existing wheelhouse/*
+
+      - name: Upload artifacts to github
+        uses: actions/upload-artifact@v1
+        with:
+          name: wheels
+          path: ./wheelhouse

.github/workflows/pypi-cd.yml

@@ -1,15 +1,15 @@
-numpy==1.14.5; python_version<'3.9'
-numpy==1.19.3; python_version>='3.9' and python_version<'3.10'
-numpy==1.21.4; python_version>='3.10'
+oldest-supported-numpy
 
-pandas==1.0.4; python_version<'3.9'
-pandas==1.1.3; python_version>='3.9' and python_version<'3.10'
+pandas==1.0.4; python_version<'3.9' and platform_machine!='aarch64'
+pandas==1.1.3; python_version<'3.9' and platform_machine=='aarch64'
+pandas==1.2.2; python_version>='3.9' and python_version<'3.10'
 pandas==1.3.4; python_version>='3.10'
 
-scipy==1.4.1; python_version<'3.9'
+scipy==1.4.1; python_version<'3.9' and platform_machine!='aarch64'
+scipy==1.5.3; python_version<'3.9' and platform_machine=='aarch64'
 scipy==1.5.4; python_version>='3.9' and python_version<'3.10'
 scipy==1.7.2; python_version>='3.10'
 
-cython==0.29.21
+cython==0.29.26
 requests>=2.4.0
 cloudpickle>=1.5.0

.github/workflows/upload-packages.sh

@@ -61,7 +61,12 @@ def deserialize(self, header: Dict, buffers: List, context: Dict):
         if header["pickle"]:
             return unpickle_buffers(buffers)
 
-        dtype = np.lib.format.descr_to_dtype(header["descr"])
+        try:
+            dtype = np.lib.format.descr_to_dtype(header["descr"])
+        except AttributeError:  # pragma: no cover
+            # for older numpy versions, descr_to_dtype is not implemented
+            dtype = np.dtype(header["descr"])
+
         dtype_new_order = header["dtype_new_order"]
         if dtype_new_order:
             dtype = dtype[dtype_new_order]