Skip to content

Commit aaaeeb1

Browse files
charlesblucacharlesbluca
authored
First pass at getting architectured builds working (#350)
* First pass at getting architectured builds working * Fix RAT failures * Build add builds on windows/macOS * Refactor artifacts upload * Remove windows/macOS builds, include all py versions * Rename conda package to avoid collision * Handle licensing * Update recipes to reflect recent changes * Build for linux-aarch64 * Add licenses to files * Add zlib to host deps * Fixes to build/overlinking errors * Always include c compiler in build deps
1 parent fc3c24b commit aaaeeb1

File tree

4 files changed

+202
-33
lines changed

4 files changed

+202
-33
lines changed

‎.github/workflows/conda.yml‎

Lines changed: 69 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,20 @@
11
name: Build conda nightly
2-
on: [push, pull_request]
2+
on:
3+
push:
4+
branches:
5+
- main
6+
pull_request:
7+
paths:
8+
- Cargo.toml
9+
- Cargo.lock
10+
- pyproject.toml
11+
- conda/recipes/**
12+
- .github/workflows/conda.yml
13+
schedule:
14+
- cron: '0 0 * * 0'
315

4-
# Cancel any already running instances of this build
16+
# When this workflow is queued, automatically cancel any previous running
17+
# or pending jobs from the same branch
518
concurrency:
619
group: conda-${{ github.head_ref }}
720
cancel-in-progress: true
@@ -13,9 +26,44 @@ defaults:
1326

1427
jobs:
1528
conda:
16-
name: Build (and optionally upload) the conda nightly
29+
name: "Build conda nightlies (python: ${{ matrix.python }}, arch: ${{ matrix.arch }})"
1730
runs-on: ubuntu-latest
31+
strategy:
32+
fail-fast: false
33+
matrix:
34+
python: ["3.8", "3.9", "3.10"]
35+
arch: ["linux-64", "linux-aarch64"]
1836
steps:
37+
- name: Manage disk space
38+
if: matrix.arch == 'linux-aarch64'
39+
run: |
40+
sudo mkdir -p /opt/empty_dir || true
41+
for d in \
42+
/opt/ghc \
43+
/opt/hostedtoolcache \
44+
/usr/lib/jvm \
45+
/usr/local/.ghcup \
46+
/usr/local/lib/android \
47+
/usr/local/share/powershell \
48+
/usr/share/dotnet \
49+
/usr/share/swift \
50+
; do
51+
sudo rsync --stats -a --delete /opt/empty_dir/ $d || true
52+
done
53+
sudo apt-get purge -y -f firefox \
54+
google-chrome-stable \
55+
microsoft-edge-stable
56+
sudo apt-get autoremove -y >& /dev/null
57+
sudo apt-get autoclean -y >& /dev/null
58+
sudo docker image prune --all --force
59+
df -h
60+
- name: Create swapfile
61+
if: matrix.arch == 'linux-aarch64'
62+
run: |
63+
sudo fallocate -l 10GiB /swapfile || true
64+
sudo chmod 600 /swapfile || true
65+
sudo mkswap /swapfile || true
66+
sudo swapon /swapfile || true
1967
- uses: actions/checkout@v3
2068
with:
2169
fetch-depth: 0
@@ -24,38 +72,34 @@ jobs:
2472
with:
2573
miniforge-variant: Mambaforge
2674
use-mamba: true
27-
python-version: "3.10"
75+
python-version: "3.8"
2876
channel-priority: strict
2977
- name: Install dependencies
3078
run: |
31-
mamba install boa conda-verify
79+
mamba install -c conda-forge boa conda-verify
3280
3381
which python
3482
pip list
3583
mamba list
36-
- uses: actions-rs/toolchain@v1
37-
with:
38-
profile: minimal
39-
toolchain: stable
40-
override: true
41-
components: clippy, rustfmt
42-
- name: Build conda package
84+
- name: Build conda packages
4385
run: |
4486
# suffix for nightly package versions
4587
export VERSION_SUFFIX=a`date +%y%m%d`
4688
4789
conda mambabuild conda/recipes \
90+
--python ${{ matrix.python }} \
91+
--variants "{target_platform: [${{ matrix.arch }}]}" \
92+
--error-overlinking \
93+
--no-test \
4894
--no-anaconda-upload \
49-
--output-folder .
50-
# - name: Upload conda package
51-
# if: |
52-
# github.event_name == 'push'
53-
# && github.repository == 'apache/arrow-datafusion-python'
54-
# env:
55-
# ANACONDA_API_TOKEN: ${{ secrets.DASK_CONDA_TOKEN }}
56-
# LABEL: ${{ github.ref == 'refs/heads/datafusion-sql-planner' && 'dev_datafusion' || 'dev' }}
57-
# run: |
58-
# # install anaconda for upload
59-
# mamba install anaconda-client
60-
61-
# anaconda upload --label $LABEL linux-64/*.tar.bz2
95+
--output-folder packages
96+
- name: Test conda packages
97+
if: matrix.arch == 'linux-64' # can only test native platform packages
98+
run: |
99+
conda mambabuild --test packages/${{ matrix.arch }}/*.tar.bz2
100+
- name: Upload conda packages as artifacts
101+
uses: actions/upload-artifact@v3
102+
with:
103+
name: "conda nightlies (python - ${{ matrix.python }}, arch - ${{ matrix.arch }})"
104+
# need to install all conda channel metadata to properly install locally
105+
path: packages/

‎conda/recipes/bld.bat‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
maturin build -vv -j %CPU_COUNT% --release --strip --manylinux off --interpreter=%PYTHON%
21+
22+
FOR /F "delims=" %%i IN ('dir /s /b target\wheels\*.whl') DO set datafusion_wheel=%%i
23+
24+
%PYTHON% -m pip install --no-deps %datafusion_wheel% -vv
25+
26+
cargo-bundle-licenses --format yaml --output THIRDPARTY.yml

‎conda/recipes/build.sh‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
#!/bin/bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
#
20+
21+
set -ex
22+
23+
# See https://github.com/conda-forge/rust-feedstock/blob/master/recipe/build.sh for cc env explanation
24+
if [ "$c_compiler" = gcc ] ; then
25+
case "$target_platform" in
26+
linux-64) rust_env_arch=X86_64_UNKNOWN_LINUX_GNU ;;
27+
linux-aarch64) rust_env_arch=AARCH64_UNKNOWN_LINUX_GNU ;;
28+
linux-ppc64le) rust_env_arch=POWERPC64LE_UNKNOWN_LINUX_GNU ;;
29+
*) echo "unknown target_platform $target_platform" ; exit 1 ;;
30+
esac
31+
32+
export CARGO_TARGET_${rust_env_arch}_LINKER=$CC
33+
fi
34+
35+
declare -a _xtra_maturin_args
36+
37+
mkdir -p $SRC_DIR/.cargo
38+
39+
if [ "$target_platform" = "osx-64" ] ; then
40+
cat <<EOF >> $SRC_DIR/.cargo/config
41+
[target.x86_64-apple-darwin]
42+
linker = "$CC"
43+
rustflags = [
44+
"-C", "link-arg=-undefined",
45+
"-C", "link-arg=dynamic_lookup",
46+
]
47+
48+
EOF
49+
50+
_xtra_maturin_args+=(--target=x86_64-apple-darwin)
51+
52+
elif [ "$target_platform" = "osx-arm64" ] ; then
53+
cat <<EOF >> $SRC_DIR/.cargo/config
54+
# Required for intermediate codegen stuff
55+
[target.x86_64-apple-darwin]
56+
linker = "$CC_FOR_BUILD"
57+
58+
# Required for final binary artifacts for target
59+
[target.aarch64-apple-darwin]
60+
linker = "$CC"
61+
rustflags = [
62+
"-C", "link-arg=-undefined",
63+
"-C", "link-arg=dynamic_lookup",
64+
]
65+
66+
EOF
67+
_xtra_maturin_args+=(--target=aarch64-apple-darwin)
68+
69+
# This variable must be set to the directory containing the target's libpython DSO
70+
export PYO3_CROSS_LIB_DIR=$PREFIX/lib
71+
72+
# xref: https://github.com/PyO3/pyo3/commit/7beb2720
73+
export PYO3_PYTHON_VERSION=${PY_VER}
74+
75+
# xref: https://github.com/conda-forge/python-feedstock/issues/621
76+
sed -i.bak 's,aarch64,arm64,g' $BUILD_PREFIX/venv/lib/os-patch.py
77+
sed -i.bak 's,aarch64,arm64,g' $BUILD_PREFIX/venv/lib/platform-patch.py
78+
fi
79+
80+
maturin build -vv -j "${CPU_COUNT}" --release --strip --manylinux off --interpreter="${PYTHON}" "${_xtra_maturin_args[@]}"
81+
82+
"${PYTHON}" -m pip install $SRC_DIR/target/wheels/datafusion*.whl --no-deps -vv
83+
84+
cargo-bundle-licenses --format yaml --output THIRDPARTY.yml

‎conda/recipes/meta.yaml‎

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,18 +29,27 @@ source:
2929
git_url: ../..
3030

3131
build:
32-
noarch: python
33-
script: {{ PYTHON }} -m pip install . -vv
34-
number: 0
32+
number: {{ GIT_DESCRIBE_NUMBER }}
33+
string: py{{ python | replace(".", "") }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
3534

3635
requirements:
37-
host:
38-
- python >=3.7
36+
build:
37+
- python # [build_platform != target_platform]
38+
- cross-python_{{ target_platform }} # [build_platform != target_platform]
39+
- zlib # [build_platform != target_platform]
40+
- {{ compiler('c') }}
41+
- {{ compiler('rust') }}
42+
- cargo-bundle-licenses
3943
- maturin >=0.15,<0.16
4044
- libprotobuf =3
45+
host:
46+
- python
47+
- maturin >=0.15,<0.16
4148
- pip
49+
- zlib
50+
- xz # [linux64]
4251
run:
43-
- python >=3.7
52+
- python
4453
- pyarrow >=11.0.0
4554

4655
test:
@@ -55,5 +64,11 @@ about:
5564
home: https://arrow.apache.org/datafusion
5665
license: Apache-2.0
5766
license_family: APACHE
58-
license_file: LICENSE.txt
59-
summary: Apache Arrow DataFusion Python Bindings
67+
license_file:
68+
- LICENSE.txt
69+
- THIRDPARTY.yml
70+
description: |
71+
DataFusion is an extensible query execution framework, written in Rust,
72+
that uses Apache Arrow as its in-memory format.
73+
doc_url: https://arrow.apache.org/datafusion
74+
dev_url: https://github.com/apache/arrow-datafusion

0 commit comments

Comments
 (0)