Test : Enable s390x support for runtime datascience 3.11/3.12

ci/cached-builds/gen_gha_matrix_jobs.py

@@ -31,6 +31,8 @@
 S390X_COMPATIBLE = {
     "runtime-minimal-ubi9-python-3.11",
     "runtime-minimal-ubi9-python-3.12",
+    "runtime-datascience-ubi9-python-3.11",
+    "runtime-datascience-ubi9-python-3.12",
     # add more here
 }
 

runtimes/datascience/ubi9-python-3.11/Dockerfile.cpu

@@ -8,8 +8,40 @@ WORKDIR /opt/app-root/bin
 # OS Packages needs to be installed as root
 USER 0
 
-# Install useful OS packages
-RUN dnf install -y mesa-libGL skopeo && dnf clean all && rm -rf /var/cache/yum
+ARG TARGETARCH
+
+# Install useful OS packages (and dev tools for s390x only)
+RUN --mount=type=cache,target=/var/cache/dnf \
+    echo "Building for architecture: ${TARGETARCH}" && \
+    if [ "$TARGETARCH" = "s390x" ]; then \
+        PACKAGES="mesa-libGL skopeo gcc gcc-c++ make openssl-devel autoconf automake libtool cmake python3-devel pybind11-devel openblas-devel unixODBC-devel openssl zlib-devel"; \
+    else \
+        PACKAGES="mesa-libGL skopeo"; \
+    fi && \
+    echo "Installing: $PACKAGES" && \
+    dnf install -y --nogpgcheck --allowerasing --nobest $PACKAGES && \
+    dnf clean all && rm -rf /var/cache/yum
+
+# Install Rust and set environment variables (s390x only)
+RUN if [ "$TARGETARCH" = "s390x" ]; then \
+    mkdir -p /opt/.cargo && \
+    export HOME=/root && \
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o rustup-init.sh && \
+    chmod +x rustup-init.sh && \
+    CARGO_HOME=/opt/.cargo ./rustup-init.sh -y --no-modify-path && \
+    rm -f rustup-init.sh && \
+    chown -R 1001:0 /opt/.cargo && \
+    echo 'export PATH=/opt/.cargo/bin:$PATH' >> /etc/profile.d/cargo.sh && \
+    echo 'export CARGO_HOME=/opt/.cargo' >> /etc/profile.d/cargo.sh && \
+    echo 'export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1' >> /etc/profile.d/cargo.sh; \
+fi
+
+# Set python alternatives for s390x only
+RUN if [ "$TARGETARCH" = "s390x" ]; then \
+    alternatives --install /usr/bin/python python /usr/bin/python3.11 1 && \
+    alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \
+    python --version && python3 --version; \
+fi
 
 # Other apps and tools installed as default user
 USER 1001
@@ -23,11 +55,62 @@ RUN curl -L https://mirror.openshift.com/pub/openshift-v4/$(uname -m)/clients/oc
     tar -xzvf /tmp/openshift-client-linux.tar.gz oc && \
     rm -f /tmp/openshift-client-linux.tar.gz
 
+##############################
+# wheel-builder stage        #
+##############################
+FROM base AS s390x-builder
+
+USER 0
+WORKDIR /tmp/build-wheels
+ARG TARGETARCH
+
+RUN echo "s390x-builder stage TARGETARCH: ${TARGETARCH}"
+
+RUN --mount=type=cache,target=/root/.cache/pip \
+    --mount=type=cache,target=/root/.cache/dnf \
+    if [ "$TARGETARCH" = "s390x" ]; then \
+        echo "Building pyarrow wheel for s390x..." && \
+        dnf install -y cmake make gcc-c++ pybind11-devel wget && \
+        dnf clean all && \
+        git clone --depth 1 https://github.com/apache/arrow.git && \
+        cd arrow/cpp && \
+        mkdir release && cd release && \
+        cmake -DCMAKE_BUILD_TYPE=Release \
+              -DCMAKE_INSTALL_PREFIX=/usr/local \
+              -DARROW_PYTHON=ON \
+              -DARROW_PARQUET=ON \
+              -DARROW_ORC=ON \
+              -DARROW_FILESYSTEM=ON \
+              -DARROW_JSON=ON \
+              -DARROW_CSV=ON \
+              -DARROW_DATASET=ON \
+              -DARROW_DEPENDENCY_SOURCE=BUNDLED \
+              -DCMAKE_CXX_FLAGS="-O3 -march=z14 -mtune=z14" \
+              -DCMAKE_C_FLAGS="-O3 -march=z14 -mtune=z14" \
+              .. && \
+        make -j$(nproc) && \
+        make install && \
+        cd ../../python && \
+        pip install --no-cache-dir -U pip wheel setuptools && \
+        pip install --no-cache-dir -r requirements-build.txt && \
+        export PYARROW_PARALLEL=$(nproc) && \
+        export ARROW_BUILD_TYPE=release && \
+        CFLAGS="-O3 -march=z14 -mtune=z14" \
+        CXXFLAGS="-O3 -march=z14 -mtune=z14" \
+        python setup.py build_ext --build-type=release --bundle-arrow-cpp bdist_wheel && \
+        mkdir -p /tmp/wheels && \
+        cp dist/pyarrow-*.whl /tmp/wheels/ && \
+        ls -la /tmp/wheels/; \
+    else \
+        echo "Not s390x, skipping wheel build" && mkdir -p /tmp/wheels; \
+    fi
+
 #######################
 # runtime-datascience #
 #######################
-FROM base AS runtime-datascience  
+FROM base AS runtime-datascience
 
+ARG TARGETARCH
 ARG DATASCIENCE_SOURCE_CODE=runtimes/datascience/ubi9-python-3.11
 
 LABEL name="odh-notebook-runtime-datascience-ubi9-python-3.11" \
@@ -41,16 +124,37 @@ LABEL name="odh-notebook-runtime-datascience-ubi9-python-3.11" \
     io.openshift.build.image="quay.io/opendatahub/workbench-images:runtime-datascience-ubi9-python-3.11"
 
 WORKDIR /opt/app-root/bin
+USER 0
+
+# Install s390x-built wheels if available
+COPY --from=s390x-builder /tmp/wheels /tmp/wheels
+RUN if [ "$TARGETARCH" = "s390x" ]; then \
+    echo "Installing s390x wheels..." && \
+    WHEELS=$(find /tmp/wheels/ -name "pyarrow-*.whl") && \
+    if [ -n "$WHEELS" ]; then \
+        pip install --no-cache-dir $WHEELS && \
+        echo "Wheel install complete"; \
+    else \
+        echo "No pyarrow wheel found!" && exit 1; \
+    fi && rm -rf /tmp/wheels; \
+else \
+    echo "Skipping wheel install on non-s390x (${TARGETARCH})"; \
+fi
 
 # Install Python packages from Pipfile.lock
 COPY ${DATASCIENCE_SOURCE_CODE}/Pipfile.lock ./
 # Copy Elyra dependencies for air-gapped enviroment
 COPY ${DATASCIENCE_SOURCE_CODE}/utils ./utils/
 
-RUN echo "Installing softwares and packages" && \
-    micropipenv install && \
+RUN --mount=type=cache,target=/root/.cache/pip \
+    echo "Installing softwares and packages" && \
+    if [ "$TARGETARCH" = "s390x" ]; then \
+        GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 CFLAGS="-O3" CXXFLAGS="-O3" micropipenv install; \
+    else \
+        micropipenv install; \
+    fi && \
     rm -f ./Pipfile.lock && \
-    # Fix permissions to support pip in Openshift environments \
+    # Fix permissions to support pip in Openshift environments
     chmod -R g+w /opt/app-root/lib/python3.11/site-packages && \
     fix-permissions /opt/app-root -P