Skip to content

Commit 325d524

Browse files
WillAydWillAyd
committed
GH-36411: [C++][Python] Use meson-python for PyArrow build system
1 parent 398fc5a commit 325d524

32 files changed

+757
-1479
lines changed

‎.gitattributes‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,8 @@ r/src/RcppExports.cpp linguist-generated=true
1313
r/src/arrowExports.cpp linguist-generated=true
1414
r/man/*.Rd linguist-generated=true
1515
r/NEWS.md merge=union
16+
python/asv* export-ignore
17+
python/benchmarks/** export-ignore
18+
python/examples/** export-ignore
19+
python/requirements* export-ignore
20+
python/scripts/** export-ignore

‎.github/workflows/dev.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ jobs:
107107
shell: bash
108108
run: |
109109
gem install test-unit openssl
110-
pip install "cython>=3.1" setuptools pytest requests setuptools-scm
110+
pip install "cython>=3.1" meson-python pytest requests setuptools-scm
111111
- name: Run Release Test
112112
shell: bash
113113
run: |

‎.github/workflows/python.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ jobs:
118118
with:
119119
python-version: 3.12
120120
- name: Setup Archery
121-
run: pip install -e dev/archery[docker]
121+
run: pip install -e dev/archery[docker] meson-python
122122
- name: Execute Docker Build
123123
env:
124124
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}

‎ci/conda_env_python.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,9 @@ cython>=3.1
2424
cloudpickle
2525
fsspec
2626
hypothesis
27+
meson-python
2728
numpy>=1.16.6
2829
pytest
2930
pytest-faulthandler
3031
s3fs>=2023.10.0
31-
setuptools>=64
3232
setuptools_scm>=8

‎ci/conda_env_sphinx.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ cython>3.1.1
2121
doxygen
2222
ipython
2323
linkify-it-py
24+
meson-python
2425
# We can't install linuxdoc by conda. We install linuxdoc by pip in
2526
# ci/dockerfiles/conda-python-pandas.dockerfile.
2627
# linuxdoc

‎ci/docker/conda-python-pandas.dockerfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,5 @@ RUN mamba install -q -y --file arrow/ci/conda_env_sphinx.txt && \
3434
COPY ci/scripts/install_pandas.sh /arrow/ci/scripts/
3535
RUN mamba uninstall -q -y numpy && \
3636
/arrow/ci/scripts/install_pandas.sh ${pandas} ${numpy}
37+
38+
RUN apt-get update && apt-get install -y patchelf

‎ci/docker/conda-python.dockerfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,3 +46,5 @@ ENV ARROW_ACERO=ON \
4646
ARROW_SUBSTRAIT=OFF \
4747
ARROW_TENSORFLOW=ON \
4848
ARROW_USE_GLOG=OFF
49+
50+
RUN apt-get update && apt-get install -y patchelf

‎ci/scripts/python_build.sh‎

Lines changed: 99 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -59,24 +59,71 @@ if [ -n "${CONDA_PREFIX}" ]; then
5959
conda list
6060
fi
6161

62-
export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja}
63-
export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug}
64-
65-
export PYARROW_WITH_ACERO=${ARROW_ACERO:-OFF}
66-
export PYARROW_WITH_AZURE=${ARROW_AZURE:-OFF}
67-
export PYARROW_WITH_CUDA=${ARROW_CUDA:-OFF}
68-
export PYARROW_WITH_DATASET=${ARROW_DATASET:-ON}
69-
export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT:-OFF}
70-
export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA:-OFF}
71-
export PYARROW_WITH_GCS=${ARROW_GCS:-OFF}
72-
export PYARROW_WITH_HDFS=${ARROW_HDFS:-ON}
73-
export PYARROW_WITH_ORC=${ARROW_ORC:-OFF}
74-
export PYARROW_WITH_PARQUET=${ARROW_PARQUET:-OFF}
75-
export PYARROW_WITH_PARQUET_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION:-ON}
76-
export PYARROW_WITH_S3=${ARROW_S3:-OFF}
77-
export PYARROW_WITH_SUBSTRAIT=${ARROW_SUBSTRAIT:-OFF}
78-
79-
export PYARROW_PARALLEL=${n_jobs}
62+
PYARROW_WITH_ACERO=$(case "$ARROW_ACERO" in
63+
ON) echo "enabled" ;;
64+
OFF) echo "disabled" ;;
65+
*) echo "auto" ;;
66+
esac)
67+
PYARROW_WITH_AZURE=$(case "$ARROW_AZURE" in
68+
ON) echo "enabled" ;;
69+
OFF) echo "disabled" ;;
70+
*) echo "auto" ;;
71+
esac)
72+
PYARROW_WITH_CUDA=$(case "$ARROW_CUDA" in
73+
ON) echo "enabled" ;;
74+
OFF) echo "disabled" ;;
75+
*) echo "auto" ;;
76+
esac)
77+
PYARROW_WITH_DATASET=$(case "$ARROW_DATASET" in
78+
ON) echo "enabled" ;;
79+
OFF) echo "disabled" ;;
80+
*) echo "enabled" ;;
81+
esac)
82+
PYARROW_WITH_FLIGHT=$(case "$ARROW_FLIGHT" in
83+
ON) echo "enabled" ;;
84+
OFF) echo "disabled" ;;
85+
*) echo "auto" ;;
86+
esac)
87+
PYARROW_WITH_GANDIVA=$(case "$ARROW_GANDIVA" in
88+
ON) echo "enabled" ;;
89+
OFF) echo "disabled" ;;
90+
*) echo "auto" ;;
91+
esac)
92+
PYARROW_WITH_GCS=$(case "$ARROW_GCS" in
93+
ON) echo "enabled" ;;
94+
OFF) echo "disabled" ;;
95+
*) echo "auto" ;;
96+
esac)
97+
PYARROW_WITH_HDFS=$(case "$ARROW_HDFS" in
98+
ON) echo "enabled" ;;
99+
OFF) echo "disabled" ;;
100+
*) echo "enabled" ;;
101+
esac)
102+
PYARROW_WITH_ORC=$(case "$ARROW_ORC" in
103+
ON) echo "enabled" ;;
104+
OFF) echo "disabled" ;;
105+
*) echo "auto" ;;
106+
esac)
107+
PYARROW_WITH_PARQUET=$(case "$ARROW_PARQUET" in
108+
ON) echo "enabled" ;;
109+
OFF) echo "disabled" ;;
110+
*) echo "auto" ;;
111+
esac)
112+
PYARROW_WITH_PARQUET_ENCRYPTION=$(case "$PARQUET_REQUIRE_ENCRYPTION" in
113+
ON) echo "enabled" ;;
114+
OFF) echo "disabled" ;;
115+
*) echo "enabled" ;;
116+
esac)
117+
PYARROW_WITH_S3=$(case "$ARROW_S3" in
118+
ON) echo "enabled" ;;
119+
OFF) echo "disabled" ;;
120+
*) echo "auto" ;;
121+
esac)
122+
PYARROW_WITH_SUBSTRAIT=$(case "$ARROW_SUBSTRAIT" in
123+
ON) echo "enabled" ;;
124+
OFF) echo "disabled" ;;
125+
*) echo "auto" ;;
126+
esac)
80127

81128
: "${CMAKE_PREFIX_PATH:=${ARROW_HOME}}"
82129
export CMAKE_PREFIX_PATH
@@ -93,7 +140,40 @@ pushd "${python_build_dir}"
93140
# on Debian/Ubuntu (ARROW-15243).
94141
# - Cannot use build isolation as we want to use specific dependency versions
95142
# (e.g. Numpy, Pandas) on some CI jobs.
96-
${PYTHON:-python} -m pip install --no-deps --no-build-isolation -vv .
143+
144+
# The conda compilers package may mess with C{XX}_FLAGS in a way that interferes
145+
# with the compiler
146+
OLD_CFLAGS=$CFLAGS
147+
OLD_CPPFLAGS=$CPPFLAGS
148+
OLD_CXXFLAGS=$CXXFLAGS
149+
export CFLAGS=
150+
export CPPFLAGS=
151+
export CXXFLAGS=
152+
153+
BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug}
154+
BUILD_TYPE=${BUILD_TYPE,,} # Meson requires lowercase values
155+
156+
${PYTHON:-python} -m pip install --no-deps --no-build-isolation -vv . \
157+
-Csetup-args="-Dbuildtype=${BUILD_TYPE}" \
158+
-Csetup-args="-Dacero=${PYARROW_WITH_ACERO}" \
159+
-Csetup-args="-Dazure=${PYARROW_WITH_AZURE}" \
160+
-Csetup-args="-Dcuda=${PYARROW_WITH_CUDA}" \
161+
-Csetup-args="-Ddataset=${PYARROW_WITH_DATASET}" \
162+
-Csetup-args="-Dflight=${PYARROW_WITH_FLIGHT}" \
163+
-Csetup-args="-Dgandiva=${PYARROW_WITH_GANDIVA}" \
164+
-Csetup-args="-Dgcs=${PYARROW_WITH_GCS}" \
165+
-Csetup-args="-Dhdfs=${PYARROW_WITH_HDFS}" \
166+
-Csetup-args="-Dorc=${PYARROW_WITH_ORC}" \
167+
-Csetup-args="-Dparquet=${PYARROW_WITH_PARQUET}" \
168+
-Csetup-args="-Dparquet_require_encryption=${PYARROW_WITH_PARQUET_ENCRYPTION}" \
169+
-Csetup-args="-Ds3=${PYARROW_WITH_S3}" \
170+
-Csetup-args="-Dsubstrait=${PYARROW_WITH_SUBSTRAIT}" \
171+
-Ccompile-args="-v" \
172+
-Csetup-args="--pkg-config-path=${ARROW_HOME}/lib/pkgconfig"
173+
174+
export CFLAGS=$OLD_CFLAGS
175+
export CPPFLAGS=$OLD_CPPFLAGS
176+
export CXXFLAGS=$OLD_CXXFLAGS
97177
popd
98178

99179
if [ "${BUILD_DOCS_PYTHON}" == "ON" ]; then

‎ci/scripts/python_sdist_build.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,5 +23,9 @@ source_dir=${1}/python
2323

2424
pushd "${source_dir}"
2525
export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION:-}
26-
${PYTHON:-python} setup.py sdist
26+
# Meson dist must be run from a VCS, so initiate a dummy repo
27+
git init .
28+
git add --all .
29+
git commit -m "dummy commit for meson dist"
30+
${PYTHON:-python} -m build --sdist .
2731
popd

‎cpp/meson.build‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,9 @@ project(
2222
version: '23.0.0-SNAPSHOT',
2323
license: 'Apache-2.0',
2424
meson_version: '>=1.3.0',
25-
default_options: ['c_std=gnu11,c11', 'warning_level=2', 'cpp_std=c++17'],
25+
# TODO: revert after https://github.com/mesonbuild/meson/issues/15202
26+
# OR figure out if we even need gnu11
27+
default_options: ['c_std=c11', 'warning_level=2', 'cpp_std=c++17'],
2628
)
2729

2830
project_args = [

0 commit comments

Comments
 (0)