Skip to content

Commit aa8c6ce

Browse files
WillAydWillAyd
committed
GH-36411: [C++][Python] Use meson-python for PyArrow build system
1 parent 2fb2f79 commit aa8c6ce

33 files changed

+870
-1502
lines changed

‎.github/workflows/dev.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ jobs:
107107
shell: bash
108108
run: |
109109
gem install test-unit openssl
110-
pip install "cython>=3.1" setuptools pytest requests setuptools-scm
110+
pip install "cython>=3.1" meson-python pytest requests setuptools-scm
111111
- name: Run Release Test
112112
shell: bash
113113
run: |

‎.github/workflows/python.yml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ jobs:
118118
with:
119119
python-version: 3.12
120120
- name: Setup Archery
121-
run: pip install -e dev/archery[docker]
121+
run: pip install -e dev/archery[docker] meson-python
122122
- name: Execute Docker Build
123123
env:
124124
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
@@ -174,8 +174,6 @@ jobs:
174174
ARROW_WITH_BROTLI: ON
175175
ARROW_BUILD_TESTS: OFF
176176
PYARROW_TEST_LARGE_MEMORY: ON
177-
# Current oldest supported version according to https://endoflife.date/macos
178-
MACOSX_DEPLOYMENT_TARGET: 12.0
179177
steps:
180178
- name: Checkout Arrow
181179
uses: actions/checkout@v5
@@ -292,6 +290,12 @@ jobs:
292290
shell: cmd
293291
run: |
294292
call "ci\scripts\python_build.bat" %cd% "${{ steps.path-info.outputs.usr-windows-dir }}"
293+
- name: Upload wheel
294+
if: always() # ensures this runs even on failure
295+
uses: actions/upload-artifact@v4
296+
with:
297+
name: built-wheel
298+
path: python/wheelhouse/*.whl
295299
- name: Test PyArrow
296300
shell: cmd
297301
run: |

‎ci/conda_env_python.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,9 @@ cython>=3.1
2424
cloudpickle
2525
fsspec
2626
hypothesis
27+
meson-python
2728
numpy>=1.16.6
2829
pytest
2930
pytest-faulthandler
3031
s3fs>=2023.10.0
31-
setuptools>=64
3232
setuptools_scm>=8

‎ci/conda_env_sphinx.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ cython>3.1.1
2121
doxygen
2222
ipython
2323
linkify-it-py
24+
meson-python
2425
# We can't install linuxdoc by conda. We install linuxdoc by pip in
2526
# ci/dockerfiles/conda-python-pandas.dockerfile.
2627
# linuxdoc

‎ci/scripts/python_build.bat‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,8 +132,16 @@ pushd %SOURCE_DIR%\python
132132
@REM Install Python build dependencies
133133
%PYTHON_CMD% -m pip install --upgrade pip || exit /B 1
134134
%PYTHON_CMD% -m pip install -r requirements-build.txt || exit /B 1
135+
%PYTHON_CMD% -m pip install build delvewheel || exit /B 1
136+
137+
@REM by default, CMake installs .lib import libs to lib and .dll libs to bin
138+
@REM delvewheel requires these to be side-by-side to properly vendor
139+
copy %CMAKE_INSTALL_PREFIX%\lib\*.lib %CMAKE_INSTALL_PREFIX%\bin\
140+
dir %CMAKE_INSTALL_PREFIX%\bin\
135141

136142
@REM Build PyArrow
137-
%PYTHON_CMD% -m pip install --no-deps --no-build-isolation -vv . || exit /B 1
143+
%PYTHON_CMD% -m build --wheel . --no-isolation || exit /B 1
144+
%PYTHON_CMD% -m delvewheel repair --ignore-existing --no-mangle-all --include-imports dist\* || exit /B 1
145+
%PYTHON_CMD% -m pip install --no-index --find-links .\wheelhouse\ pyarrow || exit /B 1
138146

139147
popd

‎ci/scripts/python_build.sh‎

Lines changed: 99 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -39,44 +39,76 @@ if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
3939
. "${ARROW_PYTHON_VENV}/bin/activate"
4040
fi
4141

42-
case "$(uname)" in
43-
Linux)
44-
n_jobs=$(nproc)
45-
;;
46-
Darwin)
47-
n_jobs=$(sysctl -n hw.ncpu)
48-
;;
49-
MINGW*)
50-
n_jobs=${NUMBER_OF_PROCESSORS:-1}
51-
;;
52-
*)
53-
n_jobs=${NPROC:-1}
54-
;;
55-
esac
56-
5742
if [ -n "${CONDA_PREFIX}" ]; then
5843
echo -e "===\n=== Conda environment for build\n==="
5944
conda list
6045
fi
6146

62-
export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja}
63-
export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug}
64-
65-
export PYARROW_WITH_ACERO=${ARROW_ACERO:-OFF}
66-
export PYARROW_WITH_AZURE=${ARROW_AZURE:-OFF}
67-
export PYARROW_WITH_CUDA=${ARROW_CUDA:-OFF}
68-
export PYARROW_WITH_DATASET=${ARROW_DATASET:-ON}
69-
export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT:-OFF}
70-
export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA:-OFF}
71-
export PYARROW_WITH_GCS=${ARROW_GCS:-OFF}
72-
export PYARROW_WITH_HDFS=${ARROW_HDFS:-ON}
73-
export PYARROW_WITH_ORC=${ARROW_ORC:-OFF}
74-
export PYARROW_WITH_PARQUET=${ARROW_PARQUET:-OFF}
75-
export PYARROW_WITH_PARQUET_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION:-ON}
76-
export PYARROW_WITH_S3=${ARROW_S3:-OFF}
77-
export PYARROW_WITH_SUBSTRAIT=${ARROW_SUBSTRAIT:-OFF}
78-
79-
export PYARROW_PARALLEL=${n_jobs}
47+
PYARROW_WITH_ACERO=$(case "$ARROW_ACERO" in
48+
ON) echo "enabled" ;;
49+
OFF) echo "disabled" ;;
50+
*) echo "auto" ;;
51+
esac)
52+
PYARROW_WITH_AZURE=$(case "$ARROW_AZURE" in
53+
ON) echo "enabled" ;;
54+
OFF) echo "disabled" ;;
55+
*) echo "auto" ;;
56+
esac)
57+
PYARROW_WITH_CUDA=$(case "$ARROW_CUDA" in
58+
ON) echo "enabled" ;;
59+
OFF) echo "disabled" ;;
60+
*) echo "auto" ;;
61+
esac)
62+
PYARROW_WITH_DATASET=$(case "$ARROW_DATASET" in
63+
ON) echo "enabled" ;;
64+
OFF) echo "disabled" ;;
65+
*) echo "enabled" ;;
66+
esac)
67+
PYARROW_WITH_FLIGHT=$(case "$ARROW_FLIGHT" in
68+
ON) echo "enabled" ;;
69+
OFF) echo "disabled" ;;
70+
*) echo "auto" ;;
71+
esac)
72+
PYARROW_WITH_GANDIVA=$(case "$ARROW_GANDIVA" in
73+
ON) echo "enabled" ;;
74+
OFF) echo "disabled" ;;
75+
*) echo "auto" ;;
76+
esac)
77+
PYARROW_WITH_GCS=$(case "$ARROW_GCS" in
78+
ON) echo "enabled" ;;
79+
OFF) echo "disabled" ;;
80+
*) echo "auto" ;;
81+
esac)
82+
PYARROW_WITH_HDFS=$(case "$ARROW_HDFS" in
83+
ON) echo "enabled" ;;
84+
OFF) echo "disabled" ;;
85+
*) echo "enabled" ;;
86+
esac)
87+
PYARROW_WITH_ORC=$(case "$ARROW_ORC" in
88+
ON) echo "enabled" ;;
89+
OFF) echo "disabled" ;;
90+
*) echo "auto" ;;
91+
esac)
92+
PYARROW_WITH_PARQUET=$(case "$ARROW_PARQUET" in
93+
ON) echo "enabled" ;;
94+
OFF) echo "disabled" ;;
95+
*) echo "auto" ;;
96+
esac)
97+
PYARROW_WITH_PARQUET_ENCRYPTION=$(case "$PARQUET_REQUIRE_ENCRYPTION" in
98+
ON) echo "enabled" ;;
99+
OFF) echo "disabled" ;;
100+
*) echo "enabled" ;;
101+
esac)
102+
PYARROW_WITH_S3=$(case "$ARROW_S3" in
103+
ON) echo "enabled" ;;
104+
OFF) echo "disabled" ;;
105+
*) echo "auto" ;;
106+
esac)
107+
PYARROW_WITH_SUBSTRAIT=$(case "$ARROW_SUBSTRAIT" in
108+
ON) echo "enabled" ;;
109+
OFF) echo "disabled" ;;
110+
*) echo "auto" ;;
111+
esac)
80112

81113
: "${CMAKE_PREFIX_PATH:=${ARROW_HOME}}"
82114
export CMAKE_PREFIX_PATH
@@ -93,7 +125,40 @@ pushd "${python_build_dir}"
93125
# on Debian/Ubuntu (ARROW-15243).
94126
# - Cannot use build isolation as we want to use specific dependency versions
95127
# (e.g. Numpy, Pandas) on some CI jobs.
96-
${PYTHON:-python} -m pip install --no-deps --no-build-isolation -vv .
128+
129+
# The conda compilers package may mess with C{XX}_FLAGS in a way that interferes
130+
# with the compiler
131+
OLD_CFLAGS=$CFLAGS
132+
OLD_CPPFLAGS=$CPPFLAGS
133+
OLD_CXXFLAGS=$CXXFLAGS
134+
export CFLAGS=
135+
export CPPFLAGS=
136+
export CXXFLAGS=
137+
138+
BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug}
139+
BUILD_TYPE=${BUILD_TYPE,,} # Meson requires lowercase values
140+
141+
${PYTHON:-python} -m pip install --no-deps --no-build-isolation -vv . \
142+
-Csetup-args="-Dbuildtype=${BUILD_TYPE}" \
143+
-Csetup-args="-Dacero=${PYARROW_WITH_ACERO}" \
144+
-Csetup-args="-Dazure=${PYARROW_WITH_AZURE}" \
145+
-Csetup-args="-Dcuda=${PYARROW_WITH_CUDA}" \
146+
-Csetup-args="-Ddataset=${PYARROW_WITH_DATASET}" \
147+
-Csetup-args="-Dflight=${PYARROW_WITH_FLIGHT}" \
148+
-Csetup-args="-Dgandiva=${PYARROW_WITH_GANDIVA}" \
149+
-Csetup-args="-Dgcs=${PYARROW_WITH_GCS}" \
150+
-Csetup-args="-Dhdfs=${PYARROW_WITH_HDFS}" \
151+
-Csetup-args="-Dorc=${PYARROW_WITH_ORC}" \
152+
-Csetup-args="-Dparquet=${PYARROW_WITH_PARQUET}" \
153+
-Csetup-args="-Dparquet_require_encryption=${PYARROW_WITH_PARQUET_ENCRYPTION}" \
154+
-Csetup-args="-Ds3=${PYARROW_WITH_S3}" \
155+
-Csetup-args="-Dsubstrait=${PYARROW_WITH_SUBSTRAIT}" \
156+
-Ccompile-args="-v" \
157+
-Csetup-args="--pkg-config-path=${ARROW_HOME}/lib/pkgconfig"
158+
159+
export CFLAGS=$OLD_CFLAGS
160+
export CPPFLAGS=$OLD_CPPFLAGS
161+
export CXXFLAGS=$OLD_CXXFLAGS
97162
popd
98163

99164
if [ "${BUILD_DOCS_PYTHON}" == "ON" ]; then

‎ci/scripts/python_sdist_build.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,5 +23,9 @@ source_dir=${1}/python
2323

2424
pushd "${source_dir}"
2525
export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION:-}
26-
${PYTHON:-python} setup.py sdist
26+
# Meson dist must be run from a VCS, so initiate a dummy repo
27+
git init .
28+
git add --all .
29+
git commit -m "dummy commit for meson dist"
30+
${PYTHON:-python} -m build --sdist .
2731
popd

‎ci/scripts/python_test.bat‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,5 +37,6 @@ pushd %SOURCE_DIR%\python
3737
popd
3838

3939
@REM Run Python tests
40+
dir C:\hostedtoolcache\windows\Python\3.13.9\x64\Lib\site-packages\pyarrow
4041
%PYTHON_CMD% -c "import pyarrow" || exit /B 1
4142
%PYTHON_CMD% -m pytest -r s --pyargs pyarrow || exit /B 1

‎ci/scripts/python_wheel_macos_build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ rm -rf ${source_dir}/python/pyarrow/*.so.*
3434

3535
echo "=== (${PYTHON_VERSION}) Set SDK, C++ and Wheel flags ==="
3636
export _PYTHON_HOST_PLATFORM="macosx-${MACOSX_DEPLOYMENT_TARGET}-${arch}"
37-
export MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET:-12.0}
37+
export MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET:-14.0}
3838
export SDKROOT=${SDKROOT:-$(xcrun --sdk macosx --show-sdk-path)}
3939

4040
if [ $arch = "arm64" ]; then

‎cpp/cmake_modules/BuildUtils.cmake‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -315,7 +315,7 @@ function(ADD_ARROW_LIB LIB_NAME)
315315
if(ARG_INSTALL_RUNTIME_DIR)
316316
set(INSTALL_RUNTIME_DIR ${ARG_INSTALL_RUNTIME_DIR})
317317
else()
318-
set(INSTALL_RUNTIME_DIR bin)
318+
set(INSTALL_RUNTIME_DIR ${CMAKE_INSTALL_BINDIR})
319319
endif()
320320

321321
if(BUILD_SHARED)

0 commit comments

Comments
 (0)