Skip to content

ppc64le(runtime/datascience): fix pyarrow installation (brought in by feast) by adding a dedicated builder stage #2488

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 16, 2025
Merged

ppc64le(runtime/datascience): fix pyarrow installation (brought in by feast) by adding a dedicated builder stage #2488

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 63 additions & 5 deletions runtimes/datascience/ubi9-python-3.12/Dockerfile.cpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ RUN --mount=type=cache,target=/var/cache/dnf \
PACKAGES="$PACKAGES gcc gcc-c++ make openssl-devel autoconf automake libtool cmake python3-devel pybind11-devel openblas-devel unixODBC-devel openssl zlib-devel"; \
fi && \
if [ "$TARGETARCH" = "ppc64le" ]; then \
PACKAGES="$PACKAGES git gcc-toolset-13 make wget unzip unixODBC-devel cmake ninja-build"; \
PACKAGES="$PACKAGES git gcc-toolset-13 make wget unzip rust cargo unixODBC-devel cmake ninja-build"; \
fi && \
if [ -n "$PACKAGES" ]; then \
echo "Installing: $PACKAGES" && \
Expand All @@ -52,6 +52,7 @@ RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
echo 'export LD_LIBRARY_PATH=/usr/local/lib64:/usr/local/lib:/usr/lib64:/usr/lib:$LD_LIBRARY_PATH' >> /etc/profile.d/ppc64le.sh && \
echo 'export OPENBLAS_VERSION=0.3.30' >> /etc/profile.d/ppc64le.sh && \
echo 'export ONNX_VERSION=1.19.0' >> /etc/profile.d/ppc64le.sh && \
echo 'export PYARROW_VERSION=17.0.0' >> /etc/profile.d/ppc64le.sh && \
echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> /etc/profile.d/ppc64le.sh && \
echo 'export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1' >> /etc/profile.d/ppc64le.sh; \
fi
Expand Down Expand Up @@ -199,6 +200,62 @@ RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
echo "Not ppc64le, skipping ONNX build" && mkdir -p /onnx_wheels; \
fi

###################################
# pyarrow builder stage for ppc64le
##################################

FROM cpu-base AS arrow-builder
USER root
WORKDIR /root

ARG TARGETARCH
ENV PYARROW_VERSION=17.0.0

RUN echo "arrow-builder stage TARGETARCH: ${TARGETARCH}"

RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
git clone -b apache-arrow-${PYARROW_VERSION} https://github.com/apache/arrow.git --recursive && \
cd arrow && rm -rf .git && mkdir dist && \
pip3 install -r python/requirements-build.txt && \
export ARROW_HOME=$(pwd)/dist && \
export LD_LIBRARY_PATH=$(pwd)/dist/lib:$LD_LIBRARY_PATH && \
export CMAKE_PREFIX_PATH=$ARROW_HOME:$CMAKE_PREFIX_PATH && \
export PARQUET_TEST_DATA="${PWD}/cpp/submodules/parquet-testing/data" && \
export ARROW_TEST_DATA="${PWD}/testing/data" && \
cmake -S cpp -B cpp/build \
-DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
-DCMAKE_BUILD_TYPE=release \
-DARROW_WITH_BZ2=ON \
-DARROW_WITH_ZLIB=ON \
-DARROW_WITH_ZSTD=ON \
-DARROW_WITH_LZ4=ON \
-DARROW_WITH_SNAPPY=ON \
-DARROW_WITH_BROTLI=ON \
-DARROW_DATASET=ON \
-DARROW_FILESYSTEM=ON \
-DARROW_COMPUTE=ON \
-DARROW_JSON=ON \
-DARROW_CSV=ON \
-DARROW_PYTHON=ON \
-DARROW_PARQUET=ON \
-DARROW_BUILD_SHARED=ON \
-DARROW_BUILD_TESTS=OFF && \
cd cpp/build && \
make -j20 install && \
export PYARROW_PARALLEL=20 && \
export PYARROW_WITH_PARQUET=1 && \
export PYARROW_WITH_DATASET=1 && \
export PYARROW_BUNDLE_ARROW_CPP=1 && \
pip3 install wheel && \
cd ../../python && \
python setup.py build_ext \
--build-type=release \
--bundle-arrow-cpp \
bdist_wheel --dist-dir /arrowwheels; \
else \
echo "Not ppc64le, skipping pyarrow build" && mkdir -p /arrowwheels; \
fi

#######################
# runtime-datascience #
#######################
Expand All @@ -223,16 +280,17 @@ USER 0
# Install ppc64le-built wheels if available
COPY --from=openblas-builder /root/OpenBLAS-* /openblas
COPY --from=onnx-builder /onnx_wheels /tmp/onnx_wheels
COPY --from=arrow-builder /arrowwheels /tmp/arrowwheels

RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
echo "Installing ppc64le ONNX wheels and OpenBLAS..." && \
HOME=/root pip install /tmp/onnx_wheels/*.whl && \
echo "Installing ppc64le ONNX, pyarrow wheels and OpenBLAS..." && \
HOME=/root pip install /tmp/onnx_wheels/*.whl /tmp/arrowwheels/*.whl && \
if [ -d "/openblas" ] && [ "$(ls -A /openblas 2>/dev/null)" ]; then \
PREFIX=/usr/local make -C /openblas install; \
fi && rm -rf /openblas /tmp/onnx_wheels; \
fi && rm -rf /openblas /tmp/onnx_wheels /tmp/arrowwheels; \
else \
echo "Skipping architecture-specific wheel installs for (${TARGETARCH})" && \
rm -rf /tmp/wheels /openblas /tmp/onnx_wheels; \
rm -rf /tmp/wheels /openblas /tmp/onnx_wheels /tmp/arrowwheels; \
fi

USER 0
Expand Down
Loading