ppc64le(runtime/datascience): fix pyarrow installation (brought in by feast) by adding a dedicated builder stage (#2488)

Signed-off-by: AaruniAggarwal <[email protected]>

Author: AaruniAggarwal

runtimes/datascience/ubi9-python-3.12/Dockerfile.cpu

@@ -39,7 +39,7 @@ RUN --mount=type=cache,target=/var/cache/dnf \
         PACKAGES="$PACKAGES gcc gcc-c++ make openssl-devel autoconf automake libtool cmake python3-devel pybind11-devel openblas-devel unixODBC-devel openssl zlib-devel"; \
     fi && \
     if [ "$TARGETARCH" = "ppc64le" ]; then \
-        PACKAGES="$PACKAGES git gcc-toolset-13 make wget unzip unixODBC-devel cmake ninja-build"; \
+        PACKAGES="$PACKAGES git gcc-toolset-13 make wget unzip rust cargo unixODBC-devel cmake ninja-build"; \
     fi && \
     if [ -n "$PACKAGES" ]; then \
         echo "Installing: $PACKAGES" && \
@@ -52,6 +52,7 @@ RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
         echo 'export LD_LIBRARY_PATH=/usr/local/lib64:/usr/local/lib:/usr/lib64:/usr/lib:$LD_LIBRARY_PATH' >> /etc/profile.d/ppc64le.sh && \
         echo 'export OPENBLAS_VERSION=0.3.30' >> /etc/profile.d/ppc64le.sh && \
         echo 'export ONNX_VERSION=1.19.0' >> /etc/profile.d/ppc64le.sh && \
+        echo 'export PYARROW_VERSION=17.0.0' >> /etc/profile.d/ppc64le.sh && \
         echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> /etc/profile.d/ppc64le.sh && \
         echo 'export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1' >> /etc/profile.d/ppc64le.sh; \
     fi
@@ -199,6 +200,62 @@ RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
         echo "Not ppc64le, skipping ONNX build" && mkdir -p /onnx_wheels; \
     fi
 
+###################################
+# pyarrow builder stage for ppc64le
+##################################
+
+FROM cpu-base AS arrow-builder
+USER root
+WORKDIR /root
+
+ARG TARGETARCH
+ENV PYARROW_VERSION=17.0.0
+
+RUN echo "arrow-builder stage TARGETARCH: ${TARGETARCH}"
+
+RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
+        git clone -b apache-arrow-${PYARROW_VERSION} https://github.com/apache/arrow.git --recursive && \
+    	cd arrow && rm -rf .git && mkdir dist                                  && \
+    	pip3 install -r python/requirements-build.txt                          && \
+    	export ARROW_HOME=$(pwd)/dist                                          && \
+    	export LD_LIBRARY_PATH=$(pwd)/dist/lib:$LD_LIBRARY_PATH                && \
+    	export CMAKE_PREFIX_PATH=$ARROW_HOME:$CMAKE_PREFIX_PATH                && \
+    	export PARQUET_TEST_DATA="${PWD}/cpp/submodules/parquet-testing/data"  && \
+    	export ARROW_TEST_DATA="${PWD}/testing/data"                           && \
+    	cmake -S cpp -B cpp/build                                                 \
+            -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+            -DCMAKE_BUILD_TYPE=release         \
+            -DARROW_WITH_BZ2=ON                \
+            -DARROW_WITH_ZLIB=ON               \
+            -DARROW_WITH_ZSTD=ON               \
+            -DARROW_WITH_LZ4=ON                \
+            -DARROW_WITH_SNAPPY=ON             \
+            -DARROW_WITH_BROTLI=ON             \
+            -DARROW_DATASET=ON                 \
+            -DARROW_FILESYSTEM=ON              \
+            -DARROW_COMPUTE=ON                 \
+            -DARROW_JSON=ON                    \
+            -DARROW_CSV=ON                     \
+            -DARROW_PYTHON=ON                  \
+            -DARROW_PARQUET=ON                 \
+            -DARROW_BUILD_SHARED=ON 	       \
+            -DARROW_BUILD_TESTS=OFF         && \
+        cd cpp/build                        && \
+        make -j20 install                   && \
+        export PYARROW_PARALLEL=20          && \
+        export PYARROW_WITH_PARQUET=1       && \
+        export PYARROW_WITH_DATASET=1       && \
+        export PYARROW_BUNDLE_ARROW_CPP=1   && \
+        pip3 install wheel                  && \
+        cd ../../python                     && \
+        python setup.py build_ext              \
+            --build-type=release               \
+            --bundle-arrow-cpp                 \
+            bdist_wheel --dist-dir /arrowwheels; \
+    else \
+        echo "Not ppc64le, skipping pyarrow build" && mkdir -p /arrowwheels; \
+    fi
+
 #######################
 # runtime-datascience #
 #######################
@@ -223,16 +280,17 @@ USER 0
 # Install ppc64le-built wheels if available
 COPY --from=openblas-builder /root/OpenBLAS-* /openblas
 COPY --from=onnx-builder /onnx_wheels /tmp/onnx_wheels
+COPY --from=arrow-builder /arrowwheels /tmp/arrowwheels
 
 RUN if [ "$TARGETARCH" = "ppc64le" ]; then \
-    	echo "Installing ppc64le ONNX wheels and OpenBLAS..." && \
-    	HOME=/root pip install /tmp/onnx_wheels/*.whl && \
+    	echo "Installing ppc64le ONNX, pyarrow wheels and OpenBLAS..." && \
+    	HOME=/root pip install /tmp/onnx_wheels/*.whl /tmp/arrowwheels/*.whl && \
         if [ -d "/openblas" ] && [ "$(ls -A /openblas 2>/dev/null)" ]; then \
     		PREFIX=/usr/local make -C /openblas install; \
-        fi && rm -rf /openblas /tmp/onnx_wheels; \
+        fi && rm -rf /openblas /tmp/onnx_wheels /tmp/arrowwheels; \
     else \
     	echo "Skipping architecture-specific wheel installs for (${TARGETARCH})" && \
-        rm -rf /tmp/wheels /openblas /tmp/onnx_wheels; \
+        rm -rf /tmp/wheels /openblas /tmp/onnx_wheels /tmp/arrowwheels; \
     fi
 
 USER 0