Skip to content

Commit 3176285

Browse files
raulcdraulcd
committed
Fixes on wheels and sdist
1 parent 9c14ef3 commit 3176285

File tree

11 files changed

+432
-173
lines changed

11 files changed

+432
-173
lines changed

‎ci/conda_env_cpp.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ lz4-c>=1.10.0
3939
make
4040
meson
4141
ninja
42-
nodejs
42+
nodejs>=16
4343
orc<2.1.0
4444
pkg-config
4545
python

‎ci/scripts/python_build.bat‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ if %ARROW_PARQUET% == ON (
168168
)
169169
if %PARQUET_REQUIRE_ENCRYPTION% == ON (
170170
set PYARROW_WITH_PARQUET_ENCRYPTION=enabled
171-
) else if %ARROW_ACERO% == OFF (
171+
) else if %PARQUET_REQUIRE_ENCRYPTION% == OFF (
172172
set PYARROW_WITH_PARQUET_ENCRYPTION=disabled
173173
) else (
174174
set PYARROW_WITH_PARQUET_ENCRYPTION=auto

‎ci/scripts/python_sdist_build.sh‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,8 @@ source_dir=${1}/python
2323

2424
pushd "${source_dir}"
2525
export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION:-}
26-
# Meson dist must be run from a VCS, so initiate a dummy repo
27-
git init .
28-
git add --all .
29-
git commit -m "dummy commit for meson dist"
30-
${PYTHON:-python} -m build --sdist .
26+
# The mounted source directory is already a git repository, so mark it as safe
27+
git config --global --add safe.directory "${1}"
28+
${PYTHON:-python} -m pip install build
29+
${PYTHON:-python} -m build --sdist . -Csetup-args="-Dsdist=true"
3130
popd

‎ci/scripts/python_wheel_macos_build.sh‎

Lines changed: 95 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -156,38 +156,110 @@ popd
156156
echo "=== (${PYTHON_VERSION}) Building wheel ==="
157157
export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE}
158158
export PYARROW_BUNDLE_ARROW_CPP=1
159-
export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR}
160-
export PYARROW_WITH_ACERO=${ARROW_ACERO}
161-
export PYARROW_WITH_AZURE=${ARROW_AZURE}
162-
export PYARROW_WITH_DATASET=${ARROW_DATASET}
163-
export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT}
164-
export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA}
165-
export PYARROW_WITH_GCS=${ARROW_GCS}
166-
export PYARROW_WITH_HDFS=${ARROW_HDFS}
167-
export PYARROW_WITH_ORC=${ARROW_ORC}
168-
export PYARROW_WITH_PARQUET=${ARROW_PARQUET}
169-
export PYARROW_WITH_PARQUET_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION}
170-
export PYARROW_WITH_SUBSTRAIT=${ARROW_SUBSTRAIT}
171-
export PYARROW_WITH_S3=${ARROW_S3}
172-
export PYARROW_CMAKE_OPTIONS="-DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} -DARROW_SIMD_LEVEL=${ARROW_SIMD_LEVEL}"
159+
160+
PYARROW_WITH_ACERO=$(case "$ARROW_ACERO" in
161+
ON) echo "enabled" ;;
162+
OFF) echo "disabled" ;;
163+
*) echo "auto" ;;
164+
esac)
165+
PYARROW_WITH_AZURE=$(case "$ARROW_AZURE" in
166+
ON) echo "enabled" ;;
167+
OFF) echo "disabled" ;;
168+
*) echo "auto" ;;
169+
esac)
170+
# CUDA support for wheels should be disabled?
171+
PYARROW_WITH_CUDA=$(case "$ARROW_CUDA" in
172+
ON) echo "enabled" ;;
173+
OFF) echo "disabled" ;;
174+
*) echo "auto" ;;
175+
esac)
176+
PYARROW_WITH_DATASET=$(case "$ARROW_DATASET" in
177+
ON) echo "enabled" ;;
178+
OFF) echo "disabled" ;;
179+
*) echo "enabled" ;;
180+
esac)
181+
PYARROW_WITH_FLIGHT=$(case "$ARROW_FLIGHT" in
182+
ON) echo "enabled" ;;
183+
OFF) echo "disabled" ;;
184+
*) echo "auto" ;;
185+
esac)
186+
PYARROW_WITH_GANDIVA=$(case "$ARROW_GANDIVA" in
187+
ON) echo "enabled" ;;
188+
OFF) echo "disabled" ;;
189+
*) echo "auto" ;;
190+
esac)
191+
PYARROW_WITH_GCS=$(case "$ARROW_GCS" in
192+
ON) echo "enabled" ;;
193+
OFF) echo "disabled" ;;
194+
*) echo "auto" ;;
195+
esac)
196+
PYARROW_WITH_HDFS=$(case "$ARROW_HDFS" in
197+
ON) echo "enabled" ;;
198+
OFF) echo "disabled" ;;
199+
*) echo "enabled" ;;
200+
esac)
201+
PYARROW_WITH_ORC=$(case "$ARROW_ORC" in
202+
ON) echo "enabled" ;;
203+
OFF) echo "disabled" ;;
204+
*) echo "auto" ;;
205+
esac)
206+
PYARROW_WITH_PARQUET=$(case "$ARROW_PARQUET" in
207+
ON) echo "enabled" ;;
208+
OFF) echo "disabled" ;;
209+
*) echo "auto" ;;
210+
esac)
211+
PYARROW_WITH_PARQUET_ENCRYPTION=$(case "$PARQUET_REQUIRE_ENCRYPTION" in
212+
ON) echo "enabled" ;;
213+
OFF) echo "disabled" ;;
214+
*) echo "enabled" ;;
215+
esac)
216+
PYARROW_WITH_S3=$(case "$ARROW_S3" in
217+
ON) echo "enabled" ;;
218+
OFF) echo "disabled" ;;
219+
*) echo "auto" ;;
220+
esac)
221+
PYARROW_WITH_SUBSTRAIT=$(case "$ARROW_SUBSTRAIT" in
222+
ON) echo "enabled" ;;
223+
OFF) echo "disabled" ;;
224+
*) echo "auto" ;;
225+
esac)
226+
173227
export ARROW_HOME=${build_dir}/install
174-
# PyArrow build configuration
175-
export CMAKE_PREFIX_PATH=${build_dir}/install
176228
# Set PyArrow version explicitly
177229
export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION}
178230

231+
# Meson sdist requires setuptools_scm to be able to get the version from git
232+
git config --global --add safe.directory ${source_dir}
233+
179234
pushd ${source_dir}/python
180-
python setup.py bdist_wheel
235+
python -m build --sdist --wheel . \
236+
-Csetup-args="-Dbuildtype=${PYARROW_BUILD_TYPE}" \
237+
-Csetup-args="-Dacero=${PYARROW_WITH_ACERO}" \
238+
-Csetup-args="-Dazure=${PYARROW_WITH_AZURE}" \
239+
-Csetup-args="-Dcuda=${PYARROW_WITH_CUDA}" \
240+
-Csetup-args="-Ddataset=${PYARROW_WITH_DATASET}" \
241+
-Csetup-args="-Dflight=${PYARROW_WITH_FLIGHT}" \
242+
-Csetup-args="-Dgandiva=${PYARROW_WITH_GANDIVA}" \
243+
-Csetup-args="-Dgcs=${PYARROW_WITH_GCS}" \
244+
-Csetup-args="-Dhdfs=${PYARROW_WITH_HDFS}" \
245+
-Csetup-args="-Dorc=${PYARROW_WITH_ORC}" \
246+
-Csetup-args="-Dparquet=${PYARROW_WITH_PARQUET}" \
247+
-Csetup-args="-Dparquet_require_encryption=${PYARROW_WITH_PARQUET_ENCRYPTION}" \
248+
-Csetup-args="-Ds3=${PYARROW_WITH_S3}" \
249+
-Csetup-args="-Dsubstrait=${PYARROW_WITH_SUBSTRAIT}" \
250+
-Ccompile-args="-v" \
251+
-Csetup-args="--pkg-config-path=${ARROW_HOME}/lib/pkgconfig"
181252
popd
182253

183254
echo "=== (${PYTHON_VERSION}) Show dynamic libraries the wheel depend on ==="
184-
deps=$(delocate-listdeps ${source_dir}/python/dist/*.whl)
255+
delocate-listdeps ${source_dir}/python/dist/*.whl
185256

186-
if echo $deps | grep -v "^pyarrow/lib\(arrow\|gandiva\|parquet\)"; then
257+
echo "=== (${PYTHON_VERSION}) Bundle shared libraries into wheel ==="
258+
delocate-wheel -w ${source_dir}/python/repaired_wheels -v ${source_dir}/python/dist/*.whl
259+
260+
echo "=== (${PYTHON_VERSION}) Validate there are no non-bundled shared dependencies ==="
261+
deps=$(delocate-listdeps ${source_dir}/python/repaired_wheels/*.whl)
262+
if echo $deps | grep -v "^pyarrow/.dylibs/lib\(arrow\|gandiva\|parquet\)"; then
187263
echo "There are non-bundled shared library dependencies."
188264
exit 1
189265
fi
190-
191-
# Move the verified wheels
192-
mkdir -p ${source_dir}/python/repaired_wheels
193-
mv ${source_dir}/python/dist/*.whl ${source_dir}/python/repaired_wheels/

‎ci/scripts/python_wheel_windows_build.bat‎

Lines changed: 105 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -114,26 +114,109 @@ echo "=== (%PYTHON%) Building wheel ==="
114114
set PYARROW_BUILD_TYPE=%CMAKE_BUILD_TYPE%
115115
set PYARROW_BUILD_VERBOSE=1
116116
set PYARROW_BUNDLE_ARROW_CPP=ON
117-
set PYARROW_CMAKE_GENERATOR=%CMAKE_GENERATOR%
118-
set PYARROW_CMAKE_OPTIONS="-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=%CMAKE_INTERPROCEDURAL_OPTIMIZATION%"
119-
set PYARROW_WITH_ACERO=%ARROW_ACERO%
120-
set PYARROW_WITH_DATASET=%ARROW_DATASET%
121-
set PYARROW_WITH_FLIGHT=%ARROW_FLIGHT%
122-
set PYARROW_WITH_GANDIVA=%ARROW_GANDIVA%
123-
set PYARROW_WITH_GCS=%ARROW_GCS%
124-
set PYARROW_WITH_HDFS=%ARROW_HDFS%
125-
set PYARROW_WITH_ORC=%ARROW_ORC%
126-
set PYARROW_WITH_PARQUET=%ARROW_PARQUET%
127-
set PYARROW_WITH_PARQUET_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION%
128-
set PYARROW_WITH_SUBSTRAIT=%ARROW_SUBSTRAIT%
129-
set PYARROW_WITH_S3=%ARROW_S3%
130-
set ARROW_HOME=C:\arrow-dist
131-
set CMAKE_PREFIX_PATH=C:\arrow-dist
117+
118+
if %ARROW_ACERO% == ON (
119+
set PYARROW_WITH_ACERO=enabled
120+
) else if %ARROW_ACERO% == OFF (
121+
set PYARROW_WITH_ACERO=disabled
122+
) else (
123+
set PYARROW_WITH_ACERO=auto
124+
)
125+
if %ARROW_DATASET% == ON (
126+
set PYARROW_WITH_DATASET=enabled
127+
) else if %ARROW_DATASET% == OFF (
128+
set PYARROW_WITH_DATASET=disabled
129+
) else (
130+
set PYARROW_WITH_DATASET=auto
131+
)
132+
if %ARROW_FLIGHT% == ON (
133+
set PYARROW_WITH_FLIGHT=enabled
134+
) else if %ARROW_FLIGHT% == OFF (
135+
set PYARROW_WITH_FLIGHT=disabled
136+
) else (
137+
set PYARROW_WITH_FLIGHT=auto
138+
)
139+
if %ARROW_GANDIVA% == ON (
140+
set PYARROW_WITH_GANDIVA=enabled
141+
) else if %ARROW_GANDIVA% == OFF (
142+
set PYARROW_WITH_GANDIVA=disabled
143+
) else (
144+
set PYARROW_WITH_GANDIVA=auto
145+
)
146+
if %ARROW_GCS% == ON (
147+
set PYARROW_WITH_GCS=enabled
148+
) else if %ARROW_GCS% == OFF (
149+
set PYARROW_WITH_GCS=disabled
150+
) else (
151+
set PYARROW_WITH_GCS=auto
152+
)
153+
if %ARROW_HDFS% == ON (
154+
set PYARROW_WITH_HDFS=enabled
155+
) else if %ARROW_HDFS% == OFF (
156+
set PYARROW_WITH_HDFS=disabled
157+
) else (
158+
set PYARROW_WITH_HDFS=auto
159+
)
160+
if %ARROW_ORC% == ON (
161+
set PYARROW_WITH_ORC=enabled
162+
) else if %ARROW_ORC% == OFF (
163+
set PYARROW_WITH_ORC=disabled
164+
) else (
165+
set PYARROW_WITH_ORC=auto
166+
)
167+
if %ARROW_PARQUET% == ON (
168+
set PYARROW_WITH_PARQUET=enabled
169+
) else if %ARROW_PARQUET% == OFF (
170+
set PYARROW_WITH_PARQUET=disabled
171+
) else (
172+
set PYARROW_WITH_PARQUET=auto
173+
)
174+
if %PARQUET_REQUIRE_ENCRYPTION% == ON (
175+
set PYARROW_WITH_PARQUET_ENCRYPTION=enabled
176+
) else if %PARQUET_REQUIRE_ENCRYPTION% == OFF (
177+
set PYARROW_WITH_PARQUET_ENCRYPTION=disabled
178+
) else (
179+
set PYARROW_WITH_PARQUET_ENCRYPTION=auto
180+
)
181+
if %ARROW_SUBSTRAIT% == ON (
182+
set PYARROW_WITH_SUBSTRAIT=enabled
183+
) else if %ARROW_SUBSTRAIT% == OFF (
184+
set PYARROW_WITH_SUBSTRAIT=disabled
185+
) else (
186+
set PYARROW_WITH_SUBSTRAIT=auto
187+
)
188+
if %ARROW_S3% == ON (
189+
set PYARROW_WITH_S3=enabled
190+
) else if %ARROW_S3% == OFF (
191+
set PYARROW_WITH_S3=disabled
192+
) else (
193+
set PYARROW_WITH_S3=auto
194+
)
195+
196+
@REM Meson sdist requires setuptools_scm to be able to get the version from git
197+
git config --global --add safe.directory C:\arrow
132198

133199
pushd C:\arrow\python
134200

201+
@REM TODO: Remove once docker rebuild works correctly
202+
@REM See: https://github.com/apache/arrow/issues/48947
203+
%PYTHON_CMD% -m pip install -U build || exit /B 1
204+
135205
@REM Build wheel
136-
%PYTHON_CMD% setup.py bdist_wheel || exit /B 1
206+
%PYTHON_CMD% -m build --sdist --wheel . ^
207+
-Csetup-args="-Dbuildtype=%CMAKE_BUILD_TYPE%" ^
208+
-Csetup-args="-Dacero=%PYARROW_WITH_ACERO%" ^
209+
-Csetup-args="-Ddataset=%PYARROW_WITH_DATASET%" ^
210+
-Csetup-args="-Dflight=%PYARROW_WITH_FLIGHT%" ^
211+
-Csetup-args="-Dgandiva=%PYARROW_WITH_GANDIVA%" ^
212+
-Csetup-args="-Dgcs=%PYARROW_WITH_GCS%" ^
213+
-Csetup-args="-Dhdfs=%PYARROW_WITH_HDFS%" ^
214+
-Csetup-args="-Dorc=%PYARROW_WITH_ORC%" ^
215+
-Csetup-args="-Dparquet=%PYARROW_WITH_PARQUET%" ^
216+
-Csetup-args="-Dparquet_require_encryption=%PYARROW_WITH_PARQUET_ENCRYPTION%" ^
217+
-Csetup-args="-Dsubstrait=%PYARROW_WITH_SUBSTRAIT%" ^
218+
-Csetup-args="-Ds3=%PYARROW_WITH_S3%" ^
219+
-Csetup-args="--cmake-prefix-path=C:\arrow-dist" || exit /B 1
137220

138221
@REM Repair the wheel with delvewheel
139222
@REM
@@ -142,11 +225,16 @@ pushd C:\arrow\python
142225
@REM required by multiple Python libraries in the same process.
143226
%PYTHON_CMD% -m pip install delvewheel || exit /B 1
144227

228+
@REM Copy .lib files to bin directory for delvewheel to find them
229+
copy C:\arrow-dist\lib\*.lib C:\arrow-dist\bin\ || exit /B 1
230+
145231
for /f %%i in ('dir dist\pyarrow-*.whl /B') do (set WHEEL_NAME=%cd%\dist\%%i) || exit /B 1
146232
echo "Wheel name: %WHEEL_NAME%"
147233

148234
%PYTHON_CMD% -m delvewheel repair -vv ^
149-
--ignore-existing --with-mangle ^
235+
--ignore-existing --with-mangle --include-imports ^
236+
--no-mangle "arrow.dll;arrow_python.dll;arrow_acero.dll;arrow_dataset.dll;arrow_flight.dll;arrow_substrait.dll;parquet.dll" ^
237+
--add-path "C:\arrow-dist\bin" ^
150238
-w repaired_wheels %WHEEL_NAME% || exit /B 1
151239

152240
popd

0 commit comments

Comments
 (0)