Enable s390x support for runtime-datascience (Python 3.11 & 3.12)

Nash-123 · Nash-123 · commit 12c97e9b1f6b · 2025-08-08T18:24:38.000+02:00
Signed-off-by: Nishan Acharya &lt;Nishan.Acharya@ibm.com&gt;
diff --git a/ci/cached-builds/gen_gha_matrix_jobs.py b/ci/cached-builds/gen_gha_matrix_jobs.py
@@ -35,6 +35,8 @@
     "runtime-minimal-ubi9-python-3.12",
     "jupyter-minimal-ubi9-python-3.11",
     "jupyter-minimal-ubi9-python-3.12",
+    "runtime-datascience-ubi9-python-3.11",
+    "runtime-datascience-ubi9-python-3.12",
     # add more here
 }
 
diff --git a/runtimes/datascience/ubi9-python-3.11/Dockerfile.cpu b/runtimes/datascience/ubi9-python-3.11/Dockerfile.cpu
@@ -8,8 +8,40 @@ WORKDIR /opt/app-root/bin
 # OS Packages needs to be installed as root
 USER 0
 
-# Install useful OS packages
-RUN dnf install -y mesa-libGL skopeo && dnf clean all && rm -rf /var/cache/yum
+ARG TARGETARCH
+
+# Install useful OS packages (and dev tools for s390x only)
+RUN --mount=type=cache,target=/var/cache/dnf \
+    echo "Building for architecture: ${TARGETARCH}" && \
+    if [ "$TARGETARCH" = "s390x" ]; then \
+        PACKAGES="mesa-libGL skopeo gcc gcc-c++ make openssl-devel autoconf automake libtool cmake python3-devel pybind11-devel openblas-devel unixODBC-devel openssl zlib-devel"; \
+    else \
+        PACKAGES="mesa-libGL skopeo"; \
+    fi && \
+    echo "Installing: $PACKAGES" && \
+    dnf install -y --nogpgcheck --allowerasing --nobest $PACKAGES && \
+    dnf clean all && rm -rf /var/cache/yum
+
+# Install Rust and set environment variables (s390x only)
+RUN if [ "$TARGETARCH" = "s390x" ]; then \
+    mkdir -p /opt/.cargo && \
+    export HOME=/root && \
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o rustup-init.sh && \
+    chmod +x rustup-init.sh && \
+    CARGO_HOME=/opt/.cargo ./rustup-init.sh -y --no-modify-path && \
+    rm -f rustup-init.sh && \
+    chown -R 1001:0 /opt/.cargo && \
+    echo 'export PATH=/opt/.cargo/bin:$PATH' >> /etc/profile.d/cargo.sh && \
+    echo 'export CARGO_HOME=/opt/.cargo' >> /etc/profile.d/cargo.sh && \
+    echo 'export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1' >> /etc/profile.d/cargo.sh; \
+fi
+
+# Set python alternatives for s390x only
+RUN if [ "$TARGETARCH" = "s390x" ]; then \
+    alternatives --install /usr/bin/python python /usr/bin/python3.11 1 && \
+    alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \
+    python --version && python3 --version; \
+fi
 
 # Other apps and tools installed as default user
 USER 1001
@@ -25,11 +57,62 @@ RUN curl -L https://mirror.openshift.com/pub/openshift-v4/$(uname -m)/clients/oc
     rm -f /tmp/openshift-client-linux.tar.gz
 # Install the oc client end
 
+##############################
+# wheel-builder stage        #
+##############################
+FROM base AS s390x-builder
+
+USER 0
+WORKDIR /tmp/build-wheels
+ARG TARGETARCH
+
+RUN echo "s390x-builder stage TARGETARCH: ${TARGETARCH}"
+
+RUN --mount=type=cache,target=/root/.cache/pip \
+    --mount=type=cache,target=/root/.cache/dnf \
+    if [ "$TARGETARCH" = "s390x" ]; then \
+        echo "Building pyarrow wheel for s390x..." && \
+        dnf install -y cmake make gcc-c++ pybind11-devel wget && \
+        dnf clean all && \
+        git clone --depth 1 https://github.com/apache/arrow.git && \
+        cd arrow/cpp && \
+        mkdir release && cd release && \
+        cmake -DCMAKE_BUILD_TYPE=Release \
+              -DCMAKE_INSTALL_PREFIX=/usr/local \
+              -DARROW_PYTHON=ON \
+              -DARROW_PARQUET=ON \
+              -DARROW_ORC=ON \
+              -DARROW_FILESYSTEM=ON \
+              -DARROW_JSON=ON \
+              -DARROW_CSV=ON \
+              -DARROW_DATASET=ON \
+              -DARROW_DEPENDENCY_SOURCE=BUNDLED \
+              -DCMAKE_CXX_FLAGS="-O3 -march=z14 -mtune=z14" \
+              -DCMAKE_C_FLAGS="-O3 -march=z14 -mtune=z14" \
+              .. && \
+        make -j$(nproc) && \
+        make install && \
+        cd ../../python && \
+        pip install --no-cache-dir -U pip wheel setuptools && \
+        pip install --no-cache-dir -r requirements-build.txt && \
+        export PYARROW_PARALLEL=$(nproc) && \
+        export ARROW_BUILD_TYPE=release && \
+        CFLAGS="-O3 -march=z14 -mtune=z14" \
+        CXXFLAGS="-O3 -march=z14 -mtune=z14" \
+        python setup.py build_ext --build-type=release --bundle-arrow-cpp bdist_wheel && \
+        mkdir -p /tmp/wheels && \
+        cp dist/pyarrow-*.whl /tmp/wheels/ && \
+        ls -la /tmp/wheels/; \
+    else \
+        echo "Not s390x, skipping wheel build" && mkdir -p /tmp/wheels; \
+    fi
+
 #######################
 # runtime-datascience #
 #######################
 FROM base AS runtime-datascience
 
+ARG TARGETARCH
 ARG DATASCIENCE_SOURCE_CODE=runtimes/datascience/ubi9-python-3.11
 
 LABEL name="odh-notebook-runtime-datascience-ubi9-python-3.11" \
@@ -43,17 +126,44 @@ LABEL name="odh-notebook-runtime-datascience-ubi9-python-3.11" \
     io.openshift.build.image="quay.io/opendatahub/workbench-images:runtime-datascience-ubi9-python-3.11"
 
 WORKDIR /opt/app-root/bin
+USER 0
+
+# Install s390x-built wheels if available
+COPY --from=s390x-builder /tmp/wheels /tmp/wheels
+RUN if [ "$TARGETARCH" = "s390x" ]; then \
+    echo "Installing s390x wheels..." && \
+    WHEELS=$(find /tmp/wheels/ -name "pyarrow-*.whl") && \
+    if [ -n "$WHEELS" ]; then \
+        pip install --no-cache-dir $WHEELS && \
+        echo "Wheel install complete"; \
+    else \
+        echo "No pyarrow wheel found!" && exit 1; \
+    fi && rm -rf /tmp/wheels; \
+else \
+    echo "Skipping wheel install on non-s390x (${TARGETARCH})"; \
+fi
 
 # Install Python packages from requirements.txt
 COPY ${DATASCIENCE_SOURCE_CODE}/requirements.txt ./
 # Copy Elyra dependencies for air-gapped enviroment
 COPY ${DATASCIENCE_SOURCE_CODE}/utils ./utils/
 
-RUN echo "Installing softwares and packages" && \
-    # This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`,
-    #  we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common.
-    uv pip install --strict --no-deps --no-cache --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --requirements=./requirements.txt --build-constraints=./requirements.txt && \
-    # Fix permissions to support pip in Openshift environments \
+RUN --mount=type=cache,target=/root/.cache/pip \
+    echo "Installing softwares and packages" && \
+    if [ "$TARGETARCH" = "s390x" ]; then \
+        # For s390x, we need special flags and environment variables for building packages
+        GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 \
+        CFLAGS="-O3" CXXFLAGS="-O3" \
+        uv pip install --strict --no-deps --no-cache --no-config --no-progress \
+            --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match \
+            --requirements=./requirements.txt --build-constraints=./requirements.txt; \
+    else \
+        # This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`,
+        #  we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common.
+        uv pip install --strict --no-deps --no-cache --no-config --no-progress \
+            --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match \
+            --requirements=./requirements.txt --build-constraints=./requirements.txt; \
+    fi && \
     chmod -R g+w /opt/app-root/lib/python3.11/site-packages && \
     fix-permissions /opt/app-root -P
 
diff --git a/runtimes/datascience/ubi9-python-3.11/Pipfile b/runtimes/datascience/ubi9-python-3.11/Pipfile
@@ -17,7 +17,12 @@ scikit-learn = "~=1.6.1"
 scipy = "~=1.15.2"
 skl2onnx = "~=1.18.0"
 onnxconverter-common = "~=1.13.0" # Required for skl2onnx, as upgraded version is not compatible with protobuf
-codeflare-sdk = "~=0.30.0"
+
+# Exclude these packages on s390x architecture due to compatibility
+codeflare-sdk = {version = "~=0.30.0", markers = "platform_machine != 's390x'"}
+py-spy = {version = "~=0.4.0", markers = "platform_machine != 's390x'"}
+ray = {version = "~=2.47.1", markers = "platform_machine != 's390x'", extras = ["data", "default"]}
+pyarrow = {version = "~=21.0.0", markers = "platform_machine != 's390x'"}
 
 # DB connectors
 pymongo = "~=4.11.2"
diff --git a/runtimes/datascience/ubi9-python-3.12/Dockerfile.cpu b/runtimes/datascience/ubi9-python-3.12/Dockerfile.cpu
@@ -3,13 +3,48 @@
 ####################
 FROM registry.access.redhat.com/ubi9/python-312:latest AS base
 
+ARG TARGETARCH
+
 WORKDIR /opt/app-root/bin
 
 # OS Packages needs to be installed as root
 USER 0
 
 # Install useful OS packages
-RUN dnf install -y mesa-libGL skopeo libxcrypt-compat && dnf clean all && rm -rf /var/cache/yum
+RUN --mount=type=cache,target=/var/cache/dnf \
+    echo "Building for architecture: ${TARGETARCH}" && \
+    PACKAGES="mesa-libGL skopeo libxcrypt-compat" && \
+    # Additional dev tools only for s390x
+    if [ "$TARGETARCH" = "s390x" ]; then \
+        PACKAGES="$PACKAGES gcc gcc-c++ make openssl-devel autoconf automake libtool cmake python3-devel pybind11-devel openblas-devel unixODBC-devel openssl zlib-devel"; \
+    fi && \
+    if [ -n "$PACKAGES" ]; then \
+        dnf install -y --nogpgcheck --allowerasing --nobest $PACKAGES && \
+        dnf clean all && rm -rf /var/cache/yum; \
+    fi
+
+# For s390x only, set ENV vars and install Rust
+RUN if [ "$TARGETARCH" = "s390x" ]; then \
+    # Install Rust and set up environment
+    mkdir -p /opt/.cargo && \
+    export HOME=/root && \
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o rustup-init.sh && \
+    chmod +x rustup-init.sh && \
+    CARGO_HOME=/opt/.cargo HOME=/root ./rustup-init.sh -y --no-modify-path && \
+    rm -f rustup-init.sh && \
+    chown -R 1001:0 /opt/.cargo && \
+    # Set environment variables
+    echo 'export PATH=/opt/.cargo/bin:$PATH' >> /etc/profile.d/cargo.sh && \
+    echo 'export CARGO_HOME=/opt/.cargo' >> /etc/profile.d/cargo.sh && \
+    echo 'export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1' >> /etc/profile.d/cargo.sh; \
+fi
+
+# Set python alternatives only for s390x (not needed for other arches)
+RUN if [ "$TARGETARCH" = "s390x" ]; then \
+    alternatives --install /usr/bin/python python /usr/bin/python3.12 1 && \
+    alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 && \
+    python --version && python3 --version; \
+fi
 
 # Other apps and tools installed as default user
 USER 1001
@@ -25,11 +60,69 @@ RUN curl -L https://mirror.openshift.com/pub/openshift-v4/$(uname -m)/clients/oc
     rm -f /tmp/openshift-client-linux.tar.gz
 # Install the oc client end
 
+##############################
+# wheel-builder stage        #
+# NOTE: Only used in s390x
+##############################
+FROM base AS s390x-builder
+
+ARG TARGETARCH 
+USER 0
+WORKDIR /tmp/build-wheels
+
+# Build pyarrow optimized for s390x
+RUN --mount=type=cache,target=/root/.cache/pip \
+    --mount=type=cache,target=/root/.cache/dnf \
+    if [ "$TARGETARCH" = "s390x" ]; then \
+        # Install build dependencies (shared for pyarrow and onnx)
+        dnf install -y cmake make gcc-c++ pybind11-devel wget && \
+        dnf clean all && \
+        # Build and collect pyarrow wheel
+        git clone --depth 1 https://github.com/apache/arrow.git && \
+        cd arrow/cpp && \
+        mkdir release && cd release && \
+        cmake -DCMAKE_BUILD_TYPE=Release \
+              -DCMAKE_INSTALL_PREFIX=/usr/local \
+              -DARROW_PYTHON=ON \
+              -DARROW_PARQUET=ON \
+              -DARROW_ORC=ON \
+              -DARROW_FILESYSTEM=ON \
+              -DARROW_JSON=ON \
+              -DARROW_CSV=ON \
+              -DARROW_DATASET=ON \
+              -DARROW_DEPENDENCY_SOURCE=BUNDLED \
+              -DARROW_WITH_LZ4=OFF \
+              -DARROW_WITH_ZSTD=OFF \
+              -DARROW_WITH_SNAPPY=OFF \
+              -DARROW_BUILD_TESTS=OFF \
+              -DARROW_BUILD_BENCHMARKS=OFF \
+              .. && \
+        make -j$(nproc) VERBOSE=1 && \
+        make install -j$(nproc) && \
+        cd ../../python && \
+        pip install --no-cache-dir -r requirements-build.txt && \
+        PYARROW_WITH_PARQUET=1 \
+        PYARROW_WITH_DATASET=1 \
+        PYARROW_WITH_FILESYSTEM=1 \
+        PYARROW_WITH_JSON=1 \
+        PYARROW_WITH_CSV=1 \
+        PYARROW_PARALLEL=$(nproc) \
+        python setup.py build_ext --build-type=release --bundle-arrow-cpp bdist_wheel && \
+        mkdir -p /tmp/wheels && \
+        cp dist/pyarrow-*.whl /tmp/wheels/ && \
+        # Ensure wheels directory exists and has content
+        ls -la /tmp/wheels/; \
+    else \
+        # Create empty wheels directory for non-s390x
+        mkdir -p /tmp/wheels; \
+    fi
+
 #######################
 # runtime-datascience #
 #######################
 FROM base AS runtime-datascience
 
+ARG TARGETARCH
 ARG DATASCIENCE_SOURCE_CODE=runtimes/datascience/ubi9-python-3.12
 
 LABEL name="odh-notebook-runtime-datascience-ubi9-python-3.12" \
@@ -44,16 +137,37 @@ LABEL name="odh-notebook-runtime-datascience-ubi9-python-3.12" \
 
 WORKDIR /opt/app-root/bin
 
+USER 0
+# Copy wheels from build stage (s390x only)
+COPY --from=s390x-builder /tmp/wheels /tmp/wheels
+RUN if [ "$TARGETARCH" = "s390x" ]; then \
+    pip install --no-cache-dir /tmp/wheels/*.whl && rm -rf /tmp/wheels; \
+else \
+    echo "Skipping wheel install for $TARGETARCH"; \
+fi
+
 # Install Python packages from requirements.txt
 COPY ${DATASCIENCE_SOURCE_CODE}/requirements.txt ./
 # Copy Elyra dependencies for air-gapped enviroment
 COPY ${DATASCIENCE_SOURCE_CODE}/utils ./utils/
 
-RUN echo "Installing softwares and packages" && \
-    # This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`,
-    #  we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common.
-    uv pip install --strict --no-deps --no-cache --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --requirements=./requirements.txt --build-constraints=./requirements.txt && \
-    # Fix permissions to support pip in Openshift environments \
+RUN --mount=type=cache,target=/root/.cache/pip \
+    echo "Installing softwares and packages" && \
+    if [ "$TARGETARCH" = "s390x" ]; then \
+        # For s390x, we need special flags and environment variables for building packages
+        GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 \
+        CFLAGS="-O3" CXXFLAGS="-O3" \
+        uv pip install --strict --no-deps --no-cache --no-config --no-progress \
+            --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match \
+            --requirements=./requirements.txt --build-constraints=./requirements.txt; \
+    else \
+        # This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`,
+        #  we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common.
+        uv pip install --strict --no-deps --no-cache --no-config --no-progress \
+            --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match \
+            --requirements=./requirements.txt --build-constraints=./requirements.txt; \
+    fi && \
+    # Fix permissions to support pip in Openshift environments
     chmod -R g+w /opt/app-root/lib/python3.12/site-packages && \
     fix-permissions /opt/app-root -P
 
diff --git a/runtimes/datascience/ubi9-python-3.12/Pipfile b/runtimes/datascience/ubi9-python-3.12/Pipfile
@@ -17,7 +17,12 @@ scikit-learn = "~=1.6.1"
 scipy = "~=1.15.2"
 skl2onnx = "~=1.18.0"
 onnxconverter-common = "~=1.13.0" # Required for skl2onnx, as upgraded version is not compatible with protobuf
-codeflare-sdk = "~=0.29.0"
+
+# Exclude these packages on s390x architecture due to compatibility
+codeflare-sdk = {version = "~=0.29.0", markers = "platform_machine != 's390x'"}
+py-spy = {version = "~=0.4.0", markers = "platform_machine != 's390x'"}
+ray = {version = "~=2.46.0", markers = "platform_machine != 's390x'", extras = ["data", "default"]}
+pyarrow = {version = "~=21.0.0", markers = "platform_machine != 's390x'"}
 
 # DB connectors
 pymongo = "~=4.11.2"

Original file line number	Diff line number	Diff line change
`@@ -35,6 +35,8 @@`
`35`	`35`	`"runtime-minimal-ubi9-python-3.12",`
`36`	`36`	`"jupyter-minimal-ubi9-python-3.11",`
`37`	`37`	`"jupyter-minimal-ubi9-python-3.12",`
	`38`	`+ "runtime-datascience-ubi9-python-3.11",`
	`39`	`+ "runtime-datascience-ubi9-python-3.12",`
`38`	`40`	`# add more here`
`39`	`41`	`}`
`40`	`42`