Skip to content

Commit ad7c633

Browse files
WillAydWillAyd
committed
GH-36411: [C++][Python] Use meson-python for PyArrow build system
1 parent f208e62 commit ad7c633

30 files changed

+986
-1549
lines changed

‎.github/workflows/dev.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ jobs:
103103
shell: bash
104104
run: |
105105
gem install test-unit openssl
106-
pip install "cython>=3.1" setuptools pytest requests setuptools-scm
106+
pip install "cython>=3.1" meson-python pytest requests setuptools-scm
107107
- name: Run Release Test
108108
shell: bash
109109
run: |

‎.github/workflows/python.yml‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ jobs:
118118
with:
119119
python-version: 3.12
120120
- name: Setup Archery
121-
run: pip install -e dev/archery[docker]
121+
run: pip install -e dev/archery[docker] meson-python
122122
- name: Execute Docker Build
123123
env:
124124
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
@@ -175,7 +175,7 @@ jobs:
175175
ARROW_BUILD_TESTS: OFF
176176
PYARROW_TEST_LARGE_MEMORY: ON
177177
# Current oldest supported version according to https://endoflife.date/macos
178-
MACOSX_DEPLOYMENT_TARGET: 12.0
178+
MACOSX_DEPLOYMENT_TARGET: "12.0"
179179
steps:
180180
- name: Checkout Arrow
181181
uses: actions/checkout@v6
@@ -288,11 +288,31 @@ jobs:
288288
env:
289289
# We can invalidate the current cache by updating this.
290290
CACHE_VERSION: "2025-09-16.1"
291+
- name: Install conda
292+
shell: cmd
293+
run: |
294+
curl -L https://github.com/conda-forge/miniforge/releases/download/25.11.0-0/Miniforge3-25.11.0-0-Windows-x86_64.exe -o miniforge.exe
295+
start /wait "" .\miniforge.exe /S /D=C:\miniforge
296+
del .\miniforge.exe
297+
C:\miniforge\Scripts\conda.exe init cmd.exe
298+
echo C:\miniforge\Scripts;C:\miniforge\Library\bin;C:\miniforge\condabin>> %GITHUB_PATH%
299+
- name: Create conda environment
300+
shell: cmd
301+
run: |
302+
conda env create -n pyarrow-dev -f .\ci\conda_env_cpp.txt -y
303+
conda env update -n pyarrow-dev -f .\ci\conda_env_python.txt
291304
- name: Build Arrow C++ and PyArrow
292305
shell: cmd
306+
env:
307+
PYTHON_CMD: "python"
293308
run: |
294-
call "ci\scripts\python_build.bat" %cd% "${{ steps.path-info.outputs.usr-windows-dir }}"
309+
call activate pyarrow-dev || exit /B 1
310+
conda list
311+
call "ci\scripts\python_build.bat" %cd% "%CONDA_PREFIX%"
295312
- name: Test PyArrow
296313
shell: cmd
314+
env:
315+
PYTHON_CMD: "python"
297316
run: |
317+
call activate pyarrow-dev
298318
call "ci\scripts\python_test.bat" %cd%

‎ci/conda_env_cpp.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ libboost-devel
3535
libgrpc
3636
libprotobuf
3737
libutf8proc
38-
lz4-c
38+
lz4-c>=1.10.0
3939
make
4040
meson
4141
ninja

‎ci/conda_env_python.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,9 @@ cython>=3.1
2424
cloudpickle
2525
fsspec
2626
hypothesis
27+
meson-python
2728
numpy>=1.16.6
2829
pytest
2930
pytest-faulthandler
3031
s3fs>=2023.10.0
31-
setuptools>=77
3232
setuptools_scm>=8

‎ci/conda_env_sphinx.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ cython>3.1.1
2121
doxygen
2222
ipython
2323
linkify-it-py
24+
meson-python
2425
# We can't install linuxdoc by conda. We install linuxdoc by pip in
2526
# ci/dockerfiles/conda-python-pandas.dockerfile.
2627
# linuxdoc

‎ci/scripts/python_build.bat‎

Lines changed: 97 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -110,30 +110,110 @@ ccache -sv
110110

111111
echo "=== Building Python ==="
112112
set PYARROW_BUILD_TYPE=%CMAKE_BUILD_TYPE%
113-
set PYARROW_BUILD_VERBOSE=1
114-
set PYARROW_BUNDLE_ARROW_CPP=ON
115-
set PYARROW_CMAKE_GENERATOR=%CMAKE_GENERATOR%
116-
set PYARROW_WITH_ACERO=%ARROW_ACERO%
117-
set PYARROW_WITH_DATASET=%ARROW_DATASET%
118-
set PYARROW_WITH_FLIGHT=%ARROW_FLIGHT%
119-
set PYARROW_WITH_GANDIVA=%ARROW_GANDIVA%
120-
set PYARROW_WITH_GCS=%ARROW_GCS%
121-
set PYARROW_WITH_HDFS=%ARROW_HDFS%
122-
set PYARROW_WITH_ORC=%ARROW_ORC%
123-
set PYARROW_WITH_PARQUET=%ARROW_PARQUET%
124-
set PYARROW_WITH_PARQUET_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION%
125-
set PYARROW_WITH_SUBSTRAIT=%ARROW_SUBSTRAIT%
126-
set PYARROW_WITH_S3=%ARROW_S3%
127-
set ARROW_HOME=%CMAKE_INSTALL_PREFIX%
128-
set CMAKE_PREFIX_PATH=%CMAKE_INSTALL_PREFIX%
113+
if %ARROW_ACERO% == ON (
114+
set PYARROW_WITH_ACERO=enabled
115+
) else if %ARROW_ACERO% == OFF (
116+
set PYARROW_WITH_ACERO=disabled
117+
) else (
118+
set PYARROW_WITH_ACERO=auto
119+
)
120+
if %ARROW_DATASET% == ON (
121+
set PYARROW_WITH_DATASET=enabled
122+
) else if %ARROW_DATASET% == OFF (
123+
set PYARROW_WITH_DATASET=disabled
124+
) else (
125+
set PYARROW_WITH_DATASET=auto
126+
)
127+
if %ARROW_FLIGHT% == ON (
128+
set PYARROW_WITH_FLIGHT=enabled
129+
) else if %ARROW_FLIGHT% == OFF (
130+
set PYARROW_WITH_FLIGHT=disabled
131+
) else (
132+
set PYARROW_WITH_FLIGHT=auto
133+
)
134+
if %ARROW_GANDIVA% == ON (
135+
set PYARROW_WITH_GANDIVA=enabled
136+
) else if %ARROW_GANDIVA% == OFF (
137+
set PYARROW_WITH_GANDIVA=disabled
138+
) else (
139+
set PYARROW_WITH_GANDIVA=auto
140+
)
141+
if %ARROW_GCS% == ON (
142+
set PYARROW_WITH_GCS=enabled
143+
) else if %ARROW_GCS% == OFF (
144+
set PYARROW_WITH_GCS=disabled
145+
) else (
146+
set PYARROW_WITH_GCS=auto
147+
)
148+
if %ARROW_HDFS% == ON (
149+
set PYARROW_WITH_HDFS=enabled
150+
) else if %ARROW_HDFS% == OFF (
151+
set PYARROW_WITH_HDFS=disabled
152+
) else (
153+
set PYARROW_WITH_HDFS=auto
154+
)
155+
if %ARROW_ORC% == ON (
156+
set PYARROW_WITH_ORC=enabled
157+
) else if %ARROW_ORC% == OFF (
158+
set PYARROW_WITH_ORC=disabled
159+
) else (
160+
set PYARROW_WITH_ORC=auto
161+
)
162+
if %ARROW_PARQUET% == ON (
163+
set PYARROW_WITH_PARQUET=enabled
164+
) else if %ARROW_PARQUET% == OFF (
165+
set PYARROW_WITH_PARQUET=disabled
166+
) else (
167+
set PYARROW_WITH_PARQUET=auto
168+
)
169+
if %PARQUET_REQUIRE_ENCRYPTION% == ON (
170+
set PYARROW_WITH_PARQUET_ENCRYPTION=enabled
171+
) else if %ARROW_ACERO% == OFF (
172+
set PYARROW_WITH_PARQUET_ENCRYPTION=disabled
173+
) else (
174+
set PYARROW_WITH_PARQUET_ENCRYPTION=auto
175+
)
176+
if %ARROW_SUBSTRAIT% == ON (
177+
set PYARROW_WITH_SUBSTRAIT=enabled
178+
) else if %ARROW_SUBSTRAIT% == OFF (
179+
set PYARROW_WITH_SUBSTRAIT=disabled
180+
) else (
181+
set PYARROW_WITH_SUBSTRAIT=auto
182+
)
183+
if %ARROW_S3% == ON (
184+
set PYARROW_WITH_S3=enabled
185+
) else if %ARROW_S3% == OFF (
186+
set PYARROW_WITH_S3=disabled
187+
) else (
188+
set PYARROW_WITH_S3=auto
189+
)
190+
if %CMAKE_BUILD_TYPE% == Release (
191+
set MESON_BUILD_TYPE=release
192+
) else (
193+
set MESON_BUILD_TYPE=debug
194+
)
129195

130196
pushd %SOURCE_DIR%\python
131197

132198
@REM Install Python build dependencies
133199
%PYTHON_CMD% -m pip install --upgrade pip || exit /B 1
134200
%PYTHON_CMD% -m pip install -r requirements-build.txt || exit /B 1
201+
%PYTHON_CMD% -m pip install build || exit /B 1
135202

136203
@REM Build PyArrow
137-
%PYTHON_CMD% -m pip install --no-deps --no-build-isolation -vv . || exit /B 1
204+
%PYTHON_CMD% -m build --wheel --no-isolation . ^
205+
-Csetup-args="-Dbuildtype=%MESON_BUILD_TYPE%" ^
206+
-Csetup-args="-Dacero=%PYARROW_WITH_ACERO%" ^
207+
-Csetup-args="-Ddataset=%PYARROW_WITH_DATASET%" ^
208+
-Csetup-args="-Dflight=%PYARROW_WITH_FLIGHT%" ^
209+
-Csetup-args="-Dgandiva=%PYARROW_WITH_GANDIVA%" ^
210+
-Csetup-args="-Dgcs=%PYARROW_WITH_GCS%" ^
211+
-Csetup-args="-Dhdfs=%PYARROW_WITH_HDFS%" ^
212+
-Csetup-args="-Dorc=%PYARROW_WITH_ORC%" ^
213+
-Csetup-args="-Dparquet=%PYARROW_WITH_PARQUET%" ^
214+
-Csetup-args="-Dparquet_require_encryption=%PYARROW_WITH_PARQUET_ENCRYPTION%" ^
215+
-Csetup-args="-Dsubstrait=%PYARROW_WITH_SUBSTRAIT%" ^
216+
-Csetup-args="-Ds3=%PYARROW_WITH_S3%" || exit /B 1
217+
%PYTHON_CMD% -m pip install --no-index --find-links .\dist\ pyarrow
138218

139219
popd

‎ci/scripts/python_build.sh‎

Lines changed: 99 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -39,44 +39,76 @@ if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
3939
. "${ARROW_PYTHON_VENV}/bin/activate"
4040
fi
4141

42-
case "$(uname)" in
43-
Linux)
44-
n_jobs=$(nproc)
45-
;;
46-
Darwin)
47-
n_jobs=$(sysctl -n hw.ncpu)
48-
;;
49-
MINGW*)
50-
n_jobs=${NUMBER_OF_PROCESSORS:-1}
51-
;;
52-
*)
53-
n_jobs=${NPROC:-1}
54-
;;
55-
esac
56-
5742
if [ -n "${CONDA_PREFIX}" ]; then
5843
echo -e "===\n=== Conda environment for build\n==="
5944
conda list
6045
fi
6146

62-
export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja}
63-
export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug}
64-
65-
export PYARROW_WITH_ACERO=${ARROW_ACERO:-OFF}
66-
export PYARROW_WITH_AZURE=${ARROW_AZURE:-OFF}
67-
export PYARROW_WITH_CUDA=${ARROW_CUDA:-OFF}
68-
export PYARROW_WITH_DATASET=${ARROW_DATASET:-ON}
69-
export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT:-OFF}
70-
export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA:-OFF}
71-
export PYARROW_WITH_GCS=${ARROW_GCS:-OFF}
72-
export PYARROW_WITH_HDFS=${ARROW_HDFS:-ON}
73-
export PYARROW_WITH_ORC=${ARROW_ORC:-OFF}
74-
export PYARROW_WITH_PARQUET=${ARROW_PARQUET:-OFF}
75-
export PYARROW_WITH_PARQUET_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION:-ON}
76-
export PYARROW_WITH_S3=${ARROW_S3:-OFF}
77-
export PYARROW_WITH_SUBSTRAIT=${ARROW_SUBSTRAIT:-OFF}
78-
79-
export PYARROW_PARALLEL=${n_jobs}
47+
PYARROW_WITH_ACERO=$(case "$ARROW_ACERO" in
48+
ON) echo "enabled" ;;
49+
OFF) echo "disabled" ;;
50+
*) echo "auto" ;;
51+
esac)
52+
PYARROW_WITH_AZURE=$(case "$ARROW_AZURE" in
53+
ON) echo "enabled" ;;
54+
OFF) echo "disabled" ;;
55+
*) echo "auto" ;;
56+
esac)
57+
PYARROW_WITH_CUDA=$(case "$ARROW_CUDA" in
58+
ON) echo "enabled" ;;
59+
OFF) echo "disabled" ;;
60+
*) echo "auto" ;;
61+
esac)
62+
PYARROW_WITH_DATASET=$(case "$ARROW_DATASET" in
63+
ON) echo "enabled" ;;
64+
OFF) echo "disabled" ;;
65+
*) echo "enabled" ;;
66+
esac)
67+
PYARROW_WITH_FLIGHT=$(case "$ARROW_FLIGHT" in
68+
ON) echo "enabled" ;;
69+
OFF) echo "disabled" ;;
70+
*) echo "auto" ;;
71+
esac)
72+
PYARROW_WITH_GANDIVA=$(case "$ARROW_GANDIVA" in
73+
ON) echo "enabled" ;;
74+
OFF) echo "disabled" ;;
75+
*) echo "auto" ;;
76+
esac)
77+
PYARROW_WITH_GCS=$(case "$ARROW_GCS" in
78+
ON) echo "enabled" ;;
79+
OFF) echo "disabled" ;;
80+
*) echo "auto" ;;
81+
esac)
82+
PYARROW_WITH_HDFS=$(case "$ARROW_HDFS" in
83+
ON) echo "enabled" ;;
84+
OFF) echo "disabled" ;;
85+
*) echo "enabled" ;;
86+
esac)
87+
PYARROW_WITH_ORC=$(case "$ARROW_ORC" in
88+
ON) echo "enabled" ;;
89+
OFF) echo "disabled" ;;
90+
*) echo "auto" ;;
91+
esac)
92+
PYARROW_WITH_PARQUET=$(case "$ARROW_PARQUET" in
93+
ON) echo "enabled" ;;
94+
OFF) echo "disabled" ;;
95+
*) echo "auto" ;;
96+
esac)
97+
PYARROW_WITH_PARQUET_ENCRYPTION=$(case "$PARQUET_REQUIRE_ENCRYPTION" in
98+
ON) echo "enabled" ;;
99+
OFF) echo "disabled" ;;
100+
*) echo "enabled" ;;
101+
esac)
102+
PYARROW_WITH_S3=$(case "$ARROW_S3" in
103+
ON) echo "enabled" ;;
104+
OFF) echo "disabled" ;;
105+
*) echo "auto" ;;
106+
esac)
107+
PYARROW_WITH_SUBSTRAIT=$(case "$ARROW_SUBSTRAIT" in
108+
ON) echo "enabled" ;;
109+
OFF) echo "disabled" ;;
110+
*) echo "auto" ;;
111+
esac)
80112

81113
: "${CMAKE_PREFIX_PATH:=${ARROW_HOME}}"
82114
export CMAKE_PREFIX_PATH
@@ -93,7 +125,40 @@ pushd "${python_build_dir}"
93125
# on Debian/Ubuntu (ARROW-15243).
94126
# - Cannot use build isolation as we want to use specific dependency versions
95127
# (e.g. Numpy, Pandas) on some CI jobs.
96-
${PYTHON:-python} -m pip install --no-deps --no-build-isolation -vv .
128+
129+
# The conda compilers package adds flags for debugging and optimization
130+
# that are unnecessary in a CI context
131+
OLD_CFLAGS=$CFLAGS
132+
OLD_CPPFLAGS=$CPPFLAGS
133+
OLD_CXXFLAGS=$CXXFLAGS
134+
export CFLAGS=
135+
export CPPFLAGS=
136+
export CXXFLAGS=
137+
138+
BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug}
139+
BUILD_TYPE=${BUILD_TYPE,,} # Meson requires lowercase values
140+
141+
${PYTHON:-python} -m pip install --no-deps --no-build-isolation -vv . \
142+
-Csetup-args="-Dbuildtype=${BUILD_TYPE}" \
143+
-Csetup-args="-Dacero=${PYARROW_WITH_ACERO}" \
144+
-Csetup-args="-Dazure=${PYARROW_WITH_AZURE}" \
145+
-Csetup-args="-Dcuda=${PYARROW_WITH_CUDA}" \
146+
-Csetup-args="-Ddataset=${PYARROW_WITH_DATASET}" \
147+
-Csetup-args="-Dflight=${PYARROW_WITH_FLIGHT}" \
148+
-Csetup-args="-Dgandiva=${PYARROW_WITH_GANDIVA}" \
149+
-Csetup-args="-Dgcs=${PYARROW_WITH_GCS}" \
150+
-Csetup-args="-Dhdfs=${PYARROW_WITH_HDFS}" \
151+
-Csetup-args="-Dorc=${PYARROW_WITH_ORC}" \
152+
-Csetup-args="-Dparquet=${PYARROW_WITH_PARQUET}" \
153+
-Csetup-args="-Dparquet_require_encryption=${PYARROW_WITH_PARQUET_ENCRYPTION}" \
154+
-Csetup-args="-Ds3=${PYARROW_WITH_S3}" \
155+
-Csetup-args="-Dsubstrait=${PYARROW_WITH_SUBSTRAIT}" \
156+
-Ccompile-args="-v" \
157+
-Csetup-args="--pkg-config-path=${ARROW_HOME}/lib/pkgconfig"
158+
159+
export CFLAGS=$OLD_CFLAGS
160+
export CPPFLAGS=$OLD_CPPFLAGS
161+
export CXXFLAGS=$OLD_CXXFLAGS
97162
popd
98163

99164
if [ "${BUILD_DOCS_PYTHON}" == "ON" ]; then

‎ci/scripts/python_sdist_build.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,5 +23,9 @@ source_dir=${1}/python
2323

2424
pushd "${source_dir}"
2525
export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION:-}
26-
${PYTHON:-python} setup.py sdist
26+
# Meson dist must be run from a VCS, so initiate a dummy repo
27+
git init .
28+
git add --all .
29+
git commit -m "dummy commit for meson dist"
30+
${PYTHON:-python} -m build --sdist .
2731
popd

‎cpp/cmake_modules/FindSnappyAlt.cmake‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -75,10 +75,6 @@ if(ARROW_SNAPPY_USE_SHARED)
7575
"${CMAKE_SHARED_LIBRARY_PREFIX}snappy${CMAKE_SHARED_LIBRARY_SUFFIX}")
7676
else()
7777
set(SNAPPY_STATIC_LIB_NAME_BASE "snappy")
78-
if(MSVC)
79-
set(SNAPPY_STATIC_LIB_NAME_BASE
80-
"${SNAPPY_STATIC_LIB_NAME_BASE}${SNAPPY_MSVC_STATIC_LIB_SUFFIX}")
81-
endif()
8278
set(SNAPPY_LIB_NAMES
8379
"${CMAKE_STATIC_LIBRARY_PREFIX}${SNAPPY_STATIC_LIB_NAME_BASE}${CMAKE_STATIC_LIBRARY_SUFFIX}"
8480
)

‎cpp/cmake_modules/Findutf8proc.cmake‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,6 @@ if(ARROW_UTF8PROC_USE_SHARED)
7373
list(APPEND utf8proc_LIB_NAMES
7474
"${CMAKE_SHARED_LIBRARY_PREFIX}utf8proc${CMAKE_SHARED_LIBRARY_SUFFIX}")
7575
else()
76-
if(MSVC AND NOT DEFINED utf8proc_MSVC_STATIC_LIB_SUFFIX)
77-
set(utf8proc_MSVC_STATIC_LIB_SUFFIX "_static")
78-
endif()
7976
set(utf8proc_STATIC_LIB_SUFFIX
8077
"${utf8proc_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}")
8178
set(utf8proc_LIB_NAMES

0 commit comments

Comments
 (0)