From 401533c357cbef67cef9c9225337993981047601 Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Thu, 5 Feb 2026 17:28:43 +0800
Subject: [PATCH 01/21] - upgrade dev base image from Ubuntu jammy to noble -
 upgrade usr base image from RedHat ubi8 to ubi9

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 .buildbot/jenkins-build-llvm-project.py |  4 +--
 docker/Dockerfile.llvm-project          | 44 +++++++++++--------------
 docker/Dockerfile.onnx-mlir             | 16 +++++----
 docker/Dockerfile.onnx-mlir-dev         | 21 ++++++------
 4 files changed, 42 insertions(+), 43 deletions(-)

diff --git a/.buildbot/jenkins-build-llvm-project.py b/.buildbot/jenkins-build-llvm-project.py
index 06c5bb115e..079d8686a1 100755
--- a/.buildbot/jenkins-build-llvm-project.py
+++ b/.buildbot/jenkins-build-llvm-project.py
@@ -7,8 +7,8 @@
 LLVM_PROJECT_DOCKERFILE = "docker/Dockerfile.llvm-project"
 LLVM_PROJECT_GITHUB_URL = "https://api.github.com/repos/llvm/llvm-project"
 LLVM_PROJECT_BASE_IMAGE = {
-    "static": "ghcr.io/onnxmlir/ubuntu:jammy",
-    "shared": "registry.access.redhat.com/ubi8-minimal:latest",
+    "static": "ghcr.io/onnxmlir/ubuntu:noble",
+    "shared": "registry.access.redhat.com/ubi9-minimal:latest",
 }
 LLVM_PROJECT_IMAGE = {
     "static": docker_static_image_name,
diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project
index 40f14a79cd..b43e7efbf2 100644
--- a/docker/Dockerfile.llvm-project
+++ b/docker/Dockerfile.llvm-project
@@ -1,5 +1,5 @@
-# By default, use ubuntu:jammy, remember to change Jenkins build script as well
-ARG BASE_IMAGE="ghcr.io/onnxmlir/ubuntu:jammy"
+# By default, use ubuntu:noble, remember to change Jenkins build script as well
+ARG BASE_IMAGE="ghcr.io/onnxmlir/ubuntu:noble"
 FROM ${BASE_IMAGE}
 
 # Label the image for various checking and cleanup
@@ -31,8 +31,8 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"]
                   autoconf automake ca-certificates clang cmake cppcheck \
                   curl default-jdk-headless gcc g++ git libncurses-dev \
                   libtool make maven ninja-build openjdk-11-jdk-headless \
-                  python3 python3-dev python3-distutils python3-numpy \
-                  python3-pip python3-pytest-xdist python3-setuptools \
+                  python3 python3-dev python3-numpy python3-pip \
+                  python3-pytest-xdist python3-setuptools \
                   python3-typing-extensions unzip zip zlib1g-dev && \
           rm -rf /var/lib/apt/lists/* && \
           ln -sf /usr/bin/pytest-3 /usr/bin/pytest; \
@@ -45,27 +45,21 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"]
           yum update -q -y && \
           yum install -q -y \
               autoconf automake ca-certificates clang cmake diffutils \
-              file java-11-openjdk-devel java-11-openjdk-headless \
-              gcc gcc-c++ git libtool make ncurses-devel ninja-build \
-              python39 python39-devel python39-pip python39-setuptools \
-              python39-wheel tzdata-java unzip which zip zlib-devel && \
-          # Workaround broken ubi8 amd64 image installing python3.12 as
-          # dependency of clang, which also breaks the /usr/bin/pip3
-          # symlink creation
-          ([ -f /usr/bin/python3.12 ] && yum remove -y python3.12 || true) && \
-          # Use same versions as those in ubuntu:jammy
+              file java-21-devel gcc gcc-c++ git libtool make ncurses-devel \
+              ninja-build python3 python3-devel python3-pip python3-setuptools \
+              python3-wheel tzdata-java unzip which zip zlib-devel && \
+          # Use same versions as those in ubuntu:noble
           pip3 install -q \
-               Cython pytest==6.2.5 numpy==1.21.5 pytest-forked==1.4.0 \
-               pytest-xdist==2.5.0 typing-extensions==3.10.0.2 && \
+               Cython pytest==7.4.4 numpy==1.26.4 pytest-xdist==3.4.0 \
+               typing-extensions==4.10.0 && \
           rm -rf /var/cache/dnf/* && \
           echo -e "/usr/local/lib" > /etc/ld.so.conf.d/local.conf; \
        fi \
     && ln -sf /usr/bin/python3 /usr/bin/python \
-    && pip3 install --upgrade pip \
     && rm -rf ${HOME}/.cache
 
 # Install bazel
-ARG BAZEL_VERSION=6.5.0
+ARG BAZEL_VERSION=7.4.1
 ARG BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download
 RUN curl -sL ${BAZEL_URL}/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-dist.zip \
          -o bazel-${BAZEL_VERSION}-dist.zip \
@@ -76,7 +70,8 @@ RUN curl -sL ${BAZEL_URL}/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-dist.zip \
                          --color=no \
                          --curses=yes \
                          --copt=-Wno-error=implicit-fallthrough \
-                         --host_javabase=@local_jdk//:jdk \
+                         --tool_java_runtime_version=local_jdk \
+                         --java_runtime_version=local_jdk \
                          --verbose_failures" \
        BAZEL_JAVAC_OPTS="-J-Xmx4g -J-Xms1g" ./compile.sh \
     && cp output/bazel /usr/local/bin \
@@ -100,11 +95,10 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
     && cd ${PROTOBUF_DIR}/build \
     # Must specify -Dprotobuf_BUILD_TESTS=OFF otherwise find_package(absl)
     # in onnx will fail due to missing protobuf::gmock target
-    # Must specify -DCMAKE_CXX_STANDARD=17 since protobuf requires c++17 but
-    # clang 14 on Ubuntu Jammy defaults to c++14
+    # If clang default C++ standard is lower than 17, you must specify
+    # -DCMAKE_CXX_STANDARD=17 which is required by protobuf
     && CC=clang CXX=clang++ \
        cmake -DCMAKE_INSTALL_LIBDIR=lib \
-             -DCMAKE_CXX_STANDARD=17 \
              -DBUILD_SHARED_LIBS=ON \
              -Dprotobuf_BUILD_TESTS=OFF .. \
     && make -j${NPROC} install && ldconfig \
@@ -115,11 +109,13 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
     # small patch.
     && export CARGO_BAZEL_GENERATOR_URL=file:///root/.cargo/bin/cargo-bazel \
     && export CARGO_BAZEL_REPIN=true \
-    && CC=clang CXX=clang++ bazel fetch //python/dist:binary_wheel \
+    && CC=clang CXX=clang++ \
+       bazel fetch //python/dist:binary_wheel --noenable_bzlmod \
     && sed -i -e 's/\["arm64", "amd64"\]/\["arm64", "amd64", "s390x"\]/g' \
            ${HOME}/.cache/bazel/_bazel_root/*/external/rules_buf/buf/internal/toolchain.bzl \
-    && CC=clang CXX=clang++ bazel build //python/dist:binary_wheel \
-    && pip3 install bazel-bin/python/dist/protobuf-${PROTOBUF_VERSION}-*.whl \
+    && CC=clang CXX=clang++ \
+       bazel build //python/dist:binary_wheel --noenable_bzlmod \
+    && pip3 install bazel-bin/python/dist/protobuf-${PROTOBUF_VERSION}-*.whl --prefix=/usr \
     && cd .. && rm -rf ${PROTOBUF_DIR} ${HOME}/.cache
 
 # Install jsoniter
diff --git a/docker/Dockerfile.onnx-mlir b/docker/Dockerfile.onnx-mlir
index 3753b11f1d..0e651cb345 100644
--- a/docker/Dockerfile.onnx-mlir
+++ b/docker/Dockerfile.onnx-mlir
@@ -27,11 +27,13 @@ RUN ONNX_ROOT=${WORK_DIR}/onnx-mlir/third_party/onnx \
           absl::log_internal_check_op\
           absl::log_internal_message\
           absl::log_internal_nullguard' CMakeLists.txt \
-    # Required for pip install with `--no-build-isolation` flag.
-    # setuptools >= 70.x creates conflicts with pip packaging versions.
-    && python3 -m pip install --upgrade setuptools==68.2.2 \
+    # To avoid the "externally managed environment" error on ubuntu,
+    # use --prefix=/usr to install into /usr/local/lib (default is
+    # /usr/lib)
+    # ubi9 doesn't have this problem and --prefix=/usr will install
+    # into /usr/lib64 (default is /usr/lib)
     && CC=clang CXX=clang++ CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib" \
-       python3 -m pip install . --no-build-isolation \
+       python3 -m pip install . --prefix=/usr \
     && rm -rf ${HOME}/.cache
 
 ARG NPROC=4
@@ -94,11 +96,11 @@ RUN LLVM_PROJECT_ROOT=${WORK_DIR}/llvm-project \
                   make ninja-build python3 && \
           rm -rf /var/lib/apt/lists/*; \
        elif [ "${distro}" = "rhel" ] || [ "${distro}" = "fedora" ]; then \
-          pip3 uninstall -q -y Cython pybind11 pytest pytest-forked \
-               pytest-xdist typing-extensions && \
+          pip3 uninstall -q -y Cython pybind11 pytest pytest-xdist \
+               typing-extensions && \
           yum remove -q -y \
               adwaita-icon-theme autoconf automake cmake file \
-              git libtool ninja-build python39 && \
+              git libtool ninja-build python3 && \
           rm -rf /var/cache/dnf/*; \
        fi \
     && rm -rf /tmp/* /usr/bin/python ${HOME}/.cache \
diff --git a/docker/Dockerfile.onnx-mlir-dev b/docker/Dockerfile.onnx-mlir-dev
index 8d312ffb81..065d4675d6 100644
--- a/docker/Dockerfile.onnx-mlir-dev
+++ b/docker/Dockerfile.onnx-mlir-dev
@@ -22,11 +22,13 @@ RUN ONNX_ROOT=${WORK_DIR}/onnx-mlir/third_party/onnx \
           absl::log_internal_check_op\
           absl::log_internal_message\
           absl::log_internal_nullguard' CMakeLists.txt \
-    # Required for pip install with `--no-build-isolation` flag.
-    # setuptools >= 70.x creates conflicts with pip packaging versions.
-    && python3 -m pip install --upgrade setuptools==68.2.2 \
+    # To avoid the "externally managed environment" error on ubuntu,
+    # use --prefix=/usr to install into /usr/local/lib (default is
+    # /usr/lib)
+    # ubi9 doesn't have this problem and --prefix=/usr will install
+    # into /usr/lib64 (default is /usr/lib)
     && CC=clang CXX=clang++ CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib" \
-       python3 -m pip install . --no-build-isolation
+       python3 -m pip install . --prefix=/usr
 
 ARG NPROC=4
 ARG ACCEL=NNPA
@@ -39,9 +41,8 @@ RUN if [ "$(uname -m)" = "s390x" ]; then \
       cd ${WORK_DIR}/onnx-mlir && \
       rm -rf build-light && mkdir -p build-light && cd build-light && \
       cmake -DCMAKE_BUILD_TYPE=Debug \
-            -DONNX_MLIR_TARGET_TO_BUILD=OMPyInfer \
-            .. &&\
-      cmake --build . && \
+            -DONNX_MLIR_TARGET_TO_BUILD=OMPyInfer .. && \
+      cmake --build . --parallel ${NPROC} && \
       cd .. && rm -rf build-light; \
     fi
 
@@ -60,9 +61,9 @@ RUN LLVM_PROJECT_ROOT=${WORK_DIR}/llvm-project \
 # Run the cppcheck scanner on s390x
 # onnx-{data|ml|operator-ml}.pb.cc are not generated by cmake so
 # run make onnx to generate them. It wasn't a problem with
-# cppcheck v1.90 in Ubuntu focal since it apparently silently
-# ignores nonexisting files. But cppcheck v2.7 in Ubuntu jammy
-# will report error on nonexisting files.
+# cppcheck v1.90 in Ubuntu Focal since it apparently silently
+# ignores nonexisting files. But cppcheck v2.7 in Jammy and
+# v2.13 in Noble will report error on nonexisting files.
     && if [ "$(uname -m)" = "s390x" ]; then \
            make -j${NPROC} onnx && \
            NPROC=${NPROC} ${ONNX_MLIR_ROOT}/utils/cppcheck_scan/cppcheck-scan.py; \

From 6f916e7c2e5096afbe8b62bcbcc2261318bbbb66 Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Thu, 5 Feb 2026 20:20:56 +0800
Subject: [PATCH 02/21] symlink pytest to pytest-3 no longer needed

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 docker/Dockerfile.llvm-project | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project
index b43e7efbf2..bf909337b8 100644
--- a/docker/Dockerfile.llvm-project
+++ b/docker/Dockerfile.llvm-project
@@ -35,7 +35,6 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"]
                   python3-pytest-xdist python3-setuptools \
                   python3-typing-extensions unzip zip zlib1g-dev && \
           rm -rf /var/lib/apt/lists/* && \
-          ln -sf /usr/bin/pytest-3 /usr/bin/pytest; \
        elif [ "${distro}" = "rhel" ] || [ "${distro}" = "fedora" ]; then \
           ln -sf /usr/share/zoneinfo/${TZ} /etc/localtime && \
           ([ -x /usr/bin/microdnf ] && microdnf install -y yum) && \

From 9af0d54d2099b23515d4e81793500dcb4c61a95c Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Thu, 5 Feb 2026 20:29:03 +0800
Subject: [PATCH 03/21] Fix elif syntax error

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 docker/Dockerfile.llvm-project | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project
index bf909337b8..d52d235c08 100644
--- a/docker/Dockerfile.llvm-project
+++ b/docker/Dockerfile.llvm-project
@@ -34,7 +34,7 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"]
                   python3 python3-dev python3-numpy python3-pip \
                   python3-pytest-xdist python3-setuptools \
                   python3-typing-extensions unzip zip zlib1g-dev && \
-          rm -rf /var/lib/apt/lists/* && \
+          rm -rf /var/lib/apt/lists/*; \
        elif [ "${distro}" = "rhel" ] || [ "${distro}" = "fedora" ]; then \
           ln -sf /usr/share/zoneinfo/${TZ} /etc/localtime && \
           ([ -x /usr/bin/microdnf ] && microdnf install -y yum) && \

From 2ad7e845f75eaf03366e8e740c64f51a5bf84840 Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Fri, 6 Feb 2026 10:42:16 +0800
Subject: [PATCH 04/21] Use protobuf v33.5

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 docker/Dockerfile.llvm-project | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project
index d52d235c08..88ebf42f57 100644
--- a/docker/Dockerfile.llvm-project
+++ b/docker/Dockerfile.llvm-project
@@ -30,10 +30,10 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"]
           apt-get install -qq -y --no-install-recommends \
                   autoconf automake ca-certificates clang cmake cppcheck \
                   curl default-jdk-headless gcc g++ git libncurses-dev \
-                  libtool make maven ninja-build openjdk-11-jdk-headless \
-                  python3 python3-dev python3-numpy python3-pip \
-                  python3-pytest-xdist python3-setuptools \
-                  python3-typing-extensions unzip zip zlib1g-dev && \
+                  libtool make maven ninja-build python3 python3-dev \
+                  python3-numpy python3-pip python3-pytest-xdist \
+                  python3-setuptools python3-typing-extensions \
+                  unzip zip zlib1g-dev && \
           rm -rf /var/lib/apt/lists/*; \
        elif [ "${distro}" = "rhel" ] || [ "${distro}" = "fedora" ]; then \
           ln -sf /usr/share/zoneinfo/${TZ} /etc/localtime && \
@@ -86,7 +86,9 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
     && cargo install cargo-bazel --version ${CARGO_BAZEL_VERSION}
 
 # Install protobuf
-ARG PROTOBUF_VERSION=6.31.1
+# v33.5 includes the bzlmod fix for rules_python version bug
+# (commit fa9ceb3d4f3fbb1542059d931bf4151a0bee316c)
+ARG PROTOBUF_VERSION=33.5
 ARG PROTOBUF_URL=https://github.com/protocolbuffers/protobuf.git
 ARG PROTOBUF_DIR=protobuf
 RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR} \
@@ -108,12 +110,10 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
     # small patch.
     && export CARGO_BAZEL_GENERATOR_URL=file:///root/.cargo/bin/cargo-bazel \
     && export CARGO_BAZEL_REPIN=true \
-    && CC=clang CXX=clang++ \
-       bazel fetch //python/dist:binary_wheel --noenable_bzlmod \
+    && CC=clang CXX=clang++ bazel fetch //python/dist:binary_wheel \
     && sed -i -e 's/\["arm64", "amd64"\]/\["arm64", "amd64", "s390x"\]/g' \
            ${HOME}/.cache/bazel/_bazel_root/*/external/rules_buf/buf/internal/toolchain.bzl \
-    && CC=clang CXX=clang++ \
-       bazel build //python/dist:binary_wheel --noenable_bzlmod \
+    && CC=clang CXX=clang++ bazel build //python/dist:binary_wheel \
     && pip3 install bazel-bin/python/dist/protobuf-${PROTOBUF_VERSION}-*.whl --prefix=/usr \
     && cd .. && rm -rf ${PROTOBUF_DIR} ${HOME}/.cache
 

From d18e6bb456d03e1914f1716091d1c1e0820dc159 Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Fri, 6 Feb 2026 11:48:55 +0800
Subject: [PATCH 05/21] Update rules_buf toolchain.bzl path

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 docker/Dockerfile.llvm-project | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project
index 88ebf42f57..3c1055f524 100644
--- a/docker/Dockerfile.llvm-project
+++ b/docker/Dockerfile.llvm-project
@@ -112,7 +112,7 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
     && export CARGO_BAZEL_REPIN=true \
     && CC=clang CXX=clang++ bazel fetch //python/dist:binary_wheel \
     && sed -i -e 's/\["arm64", "amd64"\]/\["arm64", "amd64", "s390x"\]/g' \
-           ${HOME}/.cache/bazel/_bazel_root/*/external/rules_buf/buf/internal/toolchain.bzl \
+           ${HOME}/.cache/bazel/_bazel_root/*/external/rules_buf~/buf/internal/toolchain.bzl \
     && CC=clang CXX=clang++ bazel build //python/dist:binary_wheel \
     && pip3 install bazel-bin/python/dist/protobuf-${PROTOBUF_VERSION}-*.whl --prefix=/usr \
     && cd .. && rm -rf ${PROTOBUF_DIR} ${HOME}/.cache

From 0769a44479bee2ab513e8acf50b2d5cb34c7ef9c Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Fri, 6 Feb 2026 12:46:18 +0800
Subject: [PATCH 06/21] Fix stupid protobuf version number

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 docker/Dockerfile.llvm-project | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project
index 3c1055f524..776e3baafc 100644
--- a/docker/Dockerfile.llvm-project
+++ b/docker/Dockerfile.llvm-project
@@ -114,7 +114,7 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
     && sed -i -e 's/\["arm64", "amd64"\]/\["arm64", "amd64", "s390x"\]/g' \
            ${HOME}/.cache/bazel/_bazel_root/*/external/rules_buf~/buf/internal/toolchain.bzl \
     && CC=clang CXX=clang++ bazel build //python/dist:binary_wheel \
-    && pip3 install bazel-bin/python/dist/protobuf-${PROTOBUF_VERSION}-*.whl --prefix=/usr \
+    && pip3 install bazel-bin/python/dist/protobuf-*.whl --prefix=/usr \
     && cd .. && rm -rf ${PROTOBUF_DIR} ${HOME}/.cache
 
 # Install jsoniter

From 4b0709d39627c73d01b76de58ac6688bea42f3e8 Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Fri, 6 Feb 2026 14:47:31 +0800
Subject: [PATCH 07/21] Bump rules_python to 1.8.0 since s390x no longer has
 python 3.8 interpreter

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 docker/Dockerfile.llvm-project  | 19 +++++++++++++------
 docker/Dockerfile.onnx-mlir-dev |  2 +-
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project
index 776e3baafc..4a8cefed25 100644
--- a/docker/Dockerfile.llvm-project
+++ b/docker/Dockerfile.llvm-project
@@ -86,8 +86,6 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
     && cargo install cargo-bazel --version ${CARGO_BAZEL_VERSION}
 
 # Install protobuf
-# v33.5 includes the bzlmod fix for rules_python version bug
-# (commit fa9ceb3d4f3fbb1542059d931bf4151a0bee316c)
 ARG PROTOBUF_VERSION=33.5
 ARG PROTOBUF_URL=https://github.com/protocolbuffers/protobuf.git
 ARG PROTOBUF_DIR=protobuf
@@ -104,12 +102,21 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
              -Dprotobuf_BUILD_TESTS=OFF .. \
     && make -j${NPROC} install && ldconfig \
     && cd .. \
-    # New version of python protobuf can no longer be built with setup.py.
-    # Must use bazel to build. protobuf v6.31.1 is the first version using
-    # rules_rust 0.56.0 which has s390x support. rules_buf still needs a
-    # small patch.
+    # New version of python protobuf can no longer be built with setup.py
+    # and must be built with bazel.
+    #
+    # v31.1 is the first version using rules_rust 0.56.0 which has s390x
+    # support.
+    # v33.5 includes the bzlmod fix for rules_python version bug
+    # (commit fa9ceb3d4f3fbb1542059d931bf4151a0bee316c).
+    #
+    # MODULE.bazel needs to be patched to bump rules_python to 1.8.0
+    # since s390x no longer has python 3.8 interpreter.
+    # rules_buf needs to be patched to support s390x arch.
     && export CARGO_BAZEL_GENERATOR_URL=file:///root/.cargo/bin/cargo-bazel \
     && export CARGO_BAZEL_REPIN=true \
+    && sed -i -e 's/"rules_python", version = "1.6.0"/"rules_python", version = "1.8.0"/g'
+           MODULE.bazel \
     && CC=clang CXX=clang++ bazel fetch //python/dist:binary_wheel \
     && sed -i -e 's/\["arm64", "amd64"\]/\["arm64", "amd64", "s390x"\]/g' \
            ${HOME}/.cache/bazel/_bazel_root/*/external/rules_buf~/buf/internal/toolchain.bzl \
diff --git a/docker/Dockerfile.onnx-mlir-dev b/docker/Dockerfile.onnx-mlir-dev
index 065d4675d6..74e6a458c1 100644
--- a/docker/Dockerfile.onnx-mlir-dev
+++ b/docker/Dockerfile.onnx-mlir-dev
@@ -99,7 +99,7 @@ RUN LLVM_PROJECT_ROOT=${WORK_DIR}/llvm-project \
           git remote rename origin upstream; \
        fi \
 # Install joblib for model zoo tests
-    && pip3 install -q joblib \
+    && pip3 install -q joblib --prefix=/usr \
     && rm -rf /tmp/*
 
 # Copy from onnx-mlir-dev-test to squash removed source layer and

From 3e9e119ff0b57ae11622d66e7d620a605b46ec8c Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Fri, 6 Feb 2026 15:51:22 +0800
Subject: [PATCH 08/21] Fix syntax error (missing backslash)

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 docker/Dockerfile.llvm-project | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project
index 4a8cefed25..a1b2f1096b 100644
--- a/docker/Dockerfile.llvm-project
+++ b/docker/Dockerfile.llvm-project
@@ -115,7 +115,7 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
     # rules_buf needs to be patched to support s390x arch.
     && export CARGO_BAZEL_GENERATOR_URL=file:///root/.cargo/bin/cargo-bazel \
     && export CARGO_BAZEL_REPIN=true \
-    && sed -i -e 's/"rules_python", version = "1.6.0"/"rules_python", version = "1.8.0"/g'
+    && sed -i -e 's/"rules_python", version = "1.6.0"/"rules_python", version = "1.8.0"/g' \
            MODULE.bazel \
     && CC=clang CXX=clang++ bazel fetch //python/dist:binary_wheel \
     && sed -i -e 's/\["arm64", "amd64"\]/\["arm64", "amd64", "s390x"\]/g' \

From ab53849746fa0b346ca426959e27bbabc64173a5 Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Fri, 6 Feb 2026 16:11:54 +0800
Subject: [PATCH 09/21] Try bazel fetch --nobuild

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 docker/Dockerfile.llvm-project | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project
index a1b2f1096b..2f3b2422de 100644
--- a/docker/Dockerfile.llvm-project
+++ b/docker/Dockerfile.llvm-project
@@ -117,7 +117,7 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
     && export CARGO_BAZEL_REPIN=true \
     && sed -i -e 's/"rules_python", version = "1.6.0"/"rules_python", version = "1.8.0"/g' \
            MODULE.bazel \
-    && CC=clang CXX=clang++ bazel fetch //python/dist:binary_wheel \
+    && CC=clang CXX=clang++ bazel fetch --nobuild //python/dist:binary_wheel \
     && sed -i -e 's/\["arm64", "amd64"\]/\["arm64", "amd64", "s390x"\]/g' \
            ${HOME}/.cache/bazel/_bazel_root/*/external/rules_buf~/buf/internal/toolchain.bzl \
     && CC=clang CXX=clang++ bazel build //python/dist:binary_wheel \

From 33265f601a4b6e8221f4306c9a1d8c3b5d9862fb Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Fri, 6 Feb 2026 16:45:49 +0800
Subject: [PATCH 10/21] Try bazel fetch rules_buf repo only

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 docker/Dockerfile.llvm-project | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project
index 2f3b2422de..ce04f1bfad 100644
--- a/docker/Dockerfile.llvm-project
+++ b/docker/Dockerfile.llvm-project
@@ -117,7 +117,7 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
     && export CARGO_BAZEL_REPIN=true \
     && sed -i -e 's/"rules_python", version = "1.6.0"/"rules_python", version = "1.8.0"/g' \
            MODULE.bazel \
-    && CC=clang CXX=clang++ bazel fetch --nobuild //python/dist:binary_wheel \
+    && CC=clang CXX=clang++ bazel fetch --nobuild --repo=@rules_buf \
     && sed -i -e 's/\["arm64", "amd64"\]/\["arm64", "amd64", "s390x"\]/g' \
            ${HOME}/.cache/bazel/_bazel_root/*/external/rules_buf~/buf/internal/toolchain.bzl \
     && CC=clang CXX=clang++ bazel build //python/dist:binary_wheel \

From 3314184ff9c592fb9ab9dab602de085275a646f5 Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Fri, 6 Feb 2026 19:51:05 +0800
Subject: [PATCH 11/21] - try std=c++18 - don't remove python3 on ubi9 which
 dnf depends on

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 docker/Dockerfile.llvm-project | 5 +++--
 docker/Dockerfile.onnx-mlir    | 4 ++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project
index ce04f1bfad..7e07051812 100644
--- a/docker/Dockerfile.llvm-project
+++ b/docker/Dockerfile.llvm-project
@@ -95,9 +95,10 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
     # Must specify -Dprotobuf_BUILD_TESTS=OFF otherwise find_package(absl)
     # in onnx will fail due to missing protobuf::gmock target
     # If clang default C++ standard is lower than 17, you must specify
-    # -DCMAKE_CXX_STANDARD=17 which is required by protobuf
+    # -DCMAKE_CXX_STANDARD=17 (or higher) which is required by protobuf
     && CC=clang CXX=clang++ \
        cmake -DCMAKE_INSTALL_LIBDIR=lib \
+             -DCMAKE_CXX_STANDARD=18 \
              -DBUILD_SHARED_LIBS=ON \
              -Dprotobuf_BUILD_TESTS=OFF .. \
     && make -j${NPROC} install && ldconfig \
@@ -120,7 +121,7 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
     && CC=clang CXX=clang++ bazel fetch --nobuild --repo=@rules_buf \
     && sed -i -e 's/\["arm64", "amd64"\]/\["arm64", "amd64", "s390x"\]/g' \
            ${HOME}/.cache/bazel/_bazel_root/*/external/rules_buf~/buf/internal/toolchain.bzl \
-    && CC=clang CXX=clang++ bazel build //python/dist:binary_wheel \
+    && CC=clang CXX=clang++ bazel build --cxxopt=-std=c++18 //python/dist:binary_wheel \
     && pip3 install bazel-bin/python/dist/protobuf-*.whl --prefix=/usr \
     && cd .. && rm -rf ${PROTOBUF_DIR} ${HOME}/.cache
 
diff --git a/docker/Dockerfile.onnx-mlir b/docker/Dockerfile.onnx-mlir
index 0e651cb345..20f30541cf 100644
--- a/docker/Dockerfile.onnx-mlir
+++ b/docker/Dockerfile.onnx-mlir
@@ -96,11 +96,11 @@ RUN LLVM_PROJECT_ROOT=${WORK_DIR}/llvm-project \
                   make ninja-build python3 && \
           rm -rf /var/lib/apt/lists/*; \
        elif [ "${distro}" = "rhel" ] || [ "${distro}" = "fedora" ]; then \
-          pip3 uninstall -q -y Cython pybind11 pytest pytest-xdist \
+          pip3 uninstall -q -y Cython pytest pytest-xdist \
                typing-extensions && \
           yum remove -q -y \
               adwaita-icon-theme autoconf automake cmake file \
-              git libtool ninja-build python3 && \
+              git libtool ninja-build && \
           rm -rf /var/cache/dnf/*; \
        fi \
     && rm -rf /tmp/* /usr/bin/python ${HOME}/.cache \

From ebcc5c9ab83946e2743f9628e819f407e2581d93 Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Fri, 6 Feb 2026 20:03:36 +0800
Subject: [PATCH 12/21] There is no c++18, try c++20 instead

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 docker/Dockerfile.llvm-project | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project
index 7e07051812..09257971be 100644
--- a/docker/Dockerfile.llvm-project
+++ b/docker/Dockerfile.llvm-project
@@ -98,7 +98,7 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
     # -DCMAKE_CXX_STANDARD=17 (or higher) which is required by protobuf
     && CC=clang CXX=clang++ \
        cmake -DCMAKE_INSTALL_LIBDIR=lib \
-             -DCMAKE_CXX_STANDARD=18 \
+             -DCMAKE_CXX_STANDARD=20 \
              -DBUILD_SHARED_LIBS=ON \
              -Dprotobuf_BUILD_TESTS=OFF .. \
     && make -j${NPROC} install && ldconfig \
@@ -121,7 +121,7 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
     && CC=clang CXX=clang++ bazel fetch --nobuild --repo=@rules_buf \
     && sed -i -e 's/\["arm64", "amd64"\]/\["arm64", "amd64", "s390x"\]/g' \
            ${HOME}/.cache/bazel/_bazel_root/*/external/rules_buf~/buf/internal/toolchain.bzl \
-    && CC=clang CXX=clang++ bazel build --cxxopt=-std=c++18 //python/dist:binary_wheel \
+    && CC=clang CXX=clang++ bazel build --cxxopt=-std=c++20 //python/dist:binary_wheel \
     && pip3 install bazel-bin/python/dist/protobuf-*.whl --prefix=/usr \
     && cd .. && rm -rf ${PROTOBUF_DIR} ${HOME}/.cache
 

From e48a6ac89c73cf85c1e2374cc567db947d362d08 Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Fri, 6 Feb 2026 21:38:27 +0800
Subject: [PATCH 13/21] Remove std=c++20

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 docker/Dockerfile.llvm-project | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project
index 09257971be..8055b8197a 100644
--- a/docker/Dockerfile.llvm-project
+++ b/docker/Dockerfile.llvm-project
@@ -98,7 +98,6 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
     # -DCMAKE_CXX_STANDARD=17 (or higher) which is required by protobuf
     && CC=clang CXX=clang++ \
        cmake -DCMAKE_INSTALL_LIBDIR=lib \
-             -DCMAKE_CXX_STANDARD=20 \
              -DBUILD_SHARED_LIBS=ON \
              -Dprotobuf_BUILD_TESTS=OFF .. \
     && make -j${NPROC} install && ldconfig \
@@ -121,7 +120,7 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
     && CC=clang CXX=clang++ bazel fetch --nobuild --repo=@rules_buf \
     && sed -i -e 's/\["arm64", "amd64"\]/\["arm64", "amd64", "s390x"\]/g' \
            ${HOME}/.cache/bazel/_bazel_root/*/external/rules_buf~/buf/internal/toolchain.bzl \
-    && CC=clang CXX=clang++ bazel build --cxxopt=-std=c++20 //python/dist:binary_wheel \
+    && CC=clang CXX=clang++ bazel build //python/dist:binary_wheel \
     && pip3 install bazel-bin/python/dist/protobuf-*.whl --prefix=/usr \
     && cd .. && rm -rf ${PROTOBUF_DIR} ${HOME}/.cache
 

From 516d4ecbb014bd5696f1340c90b602a272229823 Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Sat, 7 Feb 2026 00:08:40 +0800
Subject: [PATCH 14/21] Don't remove adwaita-icon-theme on ubi9 which
 systemd-udev depends on

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 docker/Dockerfile.onnx-mlir | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/docker/Dockerfile.onnx-mlir b/docker/Dockerfile.onnx-mlir
index 20f30541cf..f38e425d14 100644
--- a/docker/Dockerfile.onnx-mlir
+++ b/docker/Dockerfile.onnx-mlir
@@ -96,11 +96,10 @@ RUN LLVM_PROJECT_ROOT=${WORK_DIR}/llvm-project \
                   make ninja-build python3 && \
           rm -rf /var/lib/apt/lists/*; \
        elif [ "${distro}" = "rhel" ] || [ "${distro}" = "fedora" ]; then \
-          pip3 uninstall -q -y Cython pytest pytest-xdist \
-               typing-extensions && \
+          pip3 uninstall -q -y \
+               Cython pytest pytest-xdist typing-extensions && \
           yum remove -q -y \
-              adwaita-icon-theme autoconf automake cmake file \
-              git libtool ninja-build && \
+              autoconf automake cmake file git libtool ninja-build && \
           rm -rf /var/cache/dnf/*; \
        fi \
     && rm -rf /tmp/* /usr/bin/python ${HOME}/.cache \

From a6fe637a5aab71507f8541c9db0ba3bb3297fc4a Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Mon, 9 Feb 2026 10:55:48 +0800
Subject: [PATCH 15/21] Fix clang-18 crash when compiling abseil-cpp
 duration.cc under bazel

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 docker/Dockerfile.llvm-project | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project
index 8055b8197a..7716de6a43 100644
--- a/docker/Dockerfile.llvm-project
+++ b/docker/Dockerfile.llvm-project
@@ -113,14 +113,24 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
     # MODULE.bazel needs to be patched to bump rules_python to 1.8.0
     # since s390x no longer has python 3.8 interpreter.
     # rules_buf needs to be patched to support s390x arch.
+    #
+    # clang-18 crashes when compiling absl/time/duration.cc but works
+    # fine when running under cmake. This is because cmake and bazel
+    # are using different optimization levels. Override compiler options
+    # for duration.cc with --host_per_file_copt since abseil is being
+    # built as a tool dependency. If abseil is being built as a target
+    # dependency, we would use --per_file_copt instead.
     && export CARGO_BAZEL_GENERATOR_URL=file:///root/.cargo/bin/cargo-bazel \
     && export CARGO_BAZEL_REPIN=true \
     && sed -i -e 's/"rules_python", version = "1.6.0"/"rules_python", version = "1.8.0"/g' \
            MODULE.bazel \
-    && CC=clang CXX=clang++ bazel fetch --nobuild --repo=@rules_buf \
+    && CC=clang CXX=clang++ \
+       bazel fetch --nobuild --repo=@rules_buf \
     && sed -i -e 's/\["arm64", "amd64"\]/\["arm64", "amd64", "s390x"\]/g' \
            ${HOME}/.cache/bazel/_bazel_root/*/external/rules_buf~/buf/internal/toolchain.bzl \
-    && CC=clang CXX=clang++ bazel build //python/dist:binary_wheel \
+    && CC=clang CXX=clang++ \
+       bazel build --host_per_file_copt=".*/absl/time/duration.cc@-O0" \
+                   //python/dist:binary_wheel \
     && pip3 install bazel-bin/python/dist/protobuf-*.whl --prefix=/usr \
     && cd .. && rm -rf ${PROTOBUF_DIR} ${HOME}/.cache
 

From 27a99be599f4113b96a8f16d2ec8eb549a240748 Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Tue, 10 Feb 2026 11:00:55 +0800
Subject: [PATCH 16/21] Fix protoc-gen-upb_minitable_stage0 segfault

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 docker/Dockerfile.llvm-project | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project
index 7716de6a43..1e353263dc 100644
--- a/docker/Dockerfile.llvm-project
+++ b/docker/Dockerfile.llvm-project
@@ -90,6 +90,10 @@ ARG PROTOBUF_VERSION=33.5
 ARG PROTOBUF_URL=https://github.com/protocolbuffers/protobuf.git
 ARG PROTOBUF_DIR=protobuf
 RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR} \
+    # Fix protoc-gen-upb_minitable_stage0 segfault in
+    # _upb_DefPool_SetPlatform on s390x until upstreamed
+    && sed -i -Ez 's/(.*\n)([[:blank:]]*if \(_upb_Decoder_Reserve\(d, arr, 1\)\) \{)/\1    _upb_Decoder_MungeInt32\(\&elem\);\n\2/' ${PROTOBUF_DIR}/upb/wire/decode.c \
+    && sed -i 'N; /} else {\n        _upb_Decoder_Munge(field, val);/ { s/^/        _upb_Decoder_MungeInt32(val);\n/; b }; P; D' ${PROTOBUF_DIR}/upb/wire/decode.c \
     && mkdir -p ${PROTOBUF_DIR}/build \
     && cd ${PROTOBUF_DIR}/build \
     # Must specify -Dprotobuf_BUILD_TESTS=OFF otherwise find_package(absl)
@@ -114,8 +118,8 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
     # since s390x no longer has python 3.8 interpreter.
     # rules_buf needs to be patched to support s390x arch.
     #
-    # clang-18 crashes when compiling absl/time/duration.cc but works
-    # fine when running under cmake. This is because cmake and bazel
+    # clang-18 crashes on s390x when compiling absl/time/duration.cc under
+    # bazel but works fine under cmake. This is because cmake and bazel
     # are using different optimization levels. Override compiler options
     # for duration.cc with --host_per_file_copt since abseil is being
     # built as a tool dependency. If abseil is being built as a target

From 451ded33470e4c7d7505507865d8d30e6eb84839 Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Tue, 10 Feb 2026 21:22:54 +0800
Subject: [PATCH 17/21] Stay with protobuf v31.1 due to known problems with
 v33.5

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 docker/Dockerfile.llvm-project | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project
index 1e353263dc..58d1a3044a 100644
--- a/docker/Dockerfile.llvm-project
+++ b/docker/Dockerfile.llvm-project
@@ -86,14 +86,10 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
     && cargo install cargo-bazel --version ${CARGO_BAZEL_VERSION}
 
 # Install protobuf
-ARG PROTOBUF_VERSION=33.5
+ARG PROTOBUF_VERSION=31.1
 ARG PROTOBUF_URL=https://github.com/protocolbuffers/protobuf.git
 ARG PROTOBUF_DIR=protobuf
 RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR} \
-    # Fix protoc-gen-upb_minitable_stage0 segfault in
-    # _upb_DefPool_SetPlatform on s390x until upstreamed
-    && sed -i -Ez 's/(.*\n)([[:blank:]]*if \(_upb_Decoder_Reserve\(d, arr, 1\)\) \{)/\1    _upb_Decoder_MungeInt32\(\&elem\);\n\2/' ${PROTOBUF_DIR}/upb/wire/decode.c \
-    && sed -i 'N; /} else {\n        _upb_Decoder_Munge(field, val);/ { s/^/        _upb_Decoder_MungeInt32(val);\n/; b }; P; D' ${PROTOBUF_DIR}/upb/wire/decode.c \
     && mkdir -p ${PROTOBUF_DIR}/build \
     && cd ${PROTOBUF_DIR}/build \
     # Must specify -Dprotobuf_BUILD_TESTS=OFF otherwise find_package(absl)
@@ -124,16 +120,16 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
     # for duration.cc with --host_per_file_copt since abseil is being
     # built as a tool dependency. If abseil is being built as a target
     # dependency, we would use --per_file_copt instead.
+    #
+    # After fixing all the v33.5 bugs on s390x, it turns out it has known
+    # compatibility problems and also causes onnx (and others) to segfault.
+    # So staying with v31.1 for now.
+    && sed -i -e "/strip_prefix = \"rules_buf-0.3.0\"/a \    patch_cmds = [\n        \x27\x27\x27sed -i -e \x27s|\\\\\\\\[\"arm64\", \"amd64\"\\\\\\\\]|\\\\\\\\[\"arm64\", \"amd64\", \"s390x\"\\\\\\\\]|g\x27 buf/internal/toolchain.bzl\x27\x27\x27,\n    ]," WORKSPACE \
     && export CARGO_BAZEL_GENERATOR_URL=file:///root/.cargo/bin/cargo-bazel \
     && export CARGO_BAZEL_REPIN=true \
-    && sed -i -e 's/"rules_python", version = "1.6.0"/"rules_python", version = "1.8.0"/g' \
-           MODULE.bazel \
-    && CC=clang CXX=clang++ \
-       bazel fetch --nobuild --repo=@rules_buf \
-    && sed -i -e 's/\["arm64", "amd64"\]/\["arm64", "amd64", "s390x"\]/g' \
-           ${HOME}/.cache/bazel/_bazel_root/*/external/rules_buf~/buf/internal/toolchain.bzl \
     && CC=clang CXX=clang++ \
-       bazel build --host_per_file_copt=".*/absl/time/duration.cc@-O0" \
+       bazel build --noenable_bzlmod \
+                   --host_per_file_copt=".*/absl/time/duration.cc@-O0" \
                    //python/dist:binary_wheel \
     && pip3 install bazel-bin/python/dist/protobuf-*.whl --prefix=/usr \
     && cd .. && rm -rf ${PROTOBUF_DIR} ${HOME}/.cache

From 0a1dddb1595e22182fed7dcce3793651774c8fb9 Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Tue, 10 Feb 2026 22:36:21 +0800
Subject: [PATCH 18/21] Onnx still segfaults with protobuf v31.1, try turning
 off ONNX_GEN_PB_TYPE_STUBS

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 docker/Dockerfile.onnx-mlir     | 3 ++-
 docker/Dockerfile.onnx-mlir-dev | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/docker/Dockerfile.onnx-mlir b/docker/Dockerfile.onnx-mlir
index f38e425d14..2208a3747a 100644
--- a/docker/Dockerfile.onnx-mlir
+++ b/docker/Dockerfile.onnx-mlir
@@ -32,7 +32,8 @@ RUN ONNX_ROOT=${WORK_DIR}/onnx-mlir/third_party/onnx \
     # /usr/lib)
     # ubi9 doesn't have this problem and --prefix=/usr will install
     # into /usr/lib64 (default is /usr/lib)
-    && CC=clang CXX=clang++ CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib" \
+    && CC=clang CXX=clang++ \
+       CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib -DONNX_GEN_PB_TYPE_STUBS=OFF" \
        python3 -m pip install . --prefix=/usr \
     && rm -rf ${HOME}/.cache
 
diff --git a/docker/Dockerfile.onnx-mlir-dev b/docker/Dockerfile.onnx-mlir-dev
index 74e6a458c1..db13a0c674 100644
--- a/docker/Dockerfile.onnx-mlir-dev
+++ b/docker/Dockerfile.onnx-mlir-dev
@@ -27,7 +27,8 @@ RUN ONNX_ROOT=${WORK_DIR}/onnx-mlir/third_party/onnx \
     # /usr/lib)
     # ubi9 doesn't have this problem and --prefix=/usr will install
     # into /usr/lib64 (default is /usr/lib)
-    && CC=clang CXX=clang++ CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib" \
+    && CC=clang CXX=clang++ \
+       CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib -DONNX_GEN_PB_TYPE_STUBS=OFF" \
        python3 -m pip install . --prefix=/usr
 
 ARG NPROC=4

From b6536168e6e1a9f3a50a3cefa9a07840dfe05c45 Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Wed, 11 Feb 2026 16:03:17 +0800
Subject: [PATCH 19/21] onnx protoc-gen-mypy.py segfault was due to pip build
 isolation, not protobuf. So proceed with protobuf v33.5

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 docker/Dockerfile.llvm-project  | 32 +++++++++++++++++++++++---------
 docker/Dockerfile.onnx-mlir     | 12 +++++++++---
 docker/Dockerfile.onnx-mlir-dev | 12 +++++++++---
 3 files changed, 41 insertions(+), 15 deletions(-)

diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project
index 58d1a3044a..7a9176e234 100644
--- a/docker/Dockerfile.llvm-project
+++ b/docker/Dockerfile.llvm-project
@@ -86,22 +86,32 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
     && cargo install cargo-bazel --version ${CARGO_BAZEL_VERSION}
 
 # Install protobuf
-ARG PROTOBUF_VERSION=31.1
+ARG PROTOBUF_VERSION=33.5
 ARG PROTOBUF_URL=https://github.com/protocolbuffers/protobuf.git
 ARG PROTOBUF_DIR=protobuf
 RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR} \
+    #
+    # Fix protoc-gen-upb_minitable_stage0 segfault in
+    # _upb_DefPool_SetPlatform on s390x until upstreamed
+    #
+    && sed -i -Ez 's/(.*\n)([[:blank:]]*if \(_upb_Decoder_Reserve\(d, arr, 1\)\) \{)/\1    _upb_Decoder_MungeInt32\(\&elem\);\n\2/' ${PROTOBUF_DIR}/upb/wire/decode.c \
+    && sed -i 'N; /} else {\n        _upb_Decoder_Munge(field, val);/ { s/^/        _upb_Decoder_MungeInt32(val);\n/; b }; P; D' ${PROTOBUF_DIR}/upb/wire/decode.c \
+    #
     && mkdir -p ${PROTOBUF_DIR}/build \
     && cd ${PROTOBUF_DIR}/build \
+    #
     # Must specify -Dprotobuf_BUILD_TESTS=OFF otherwise find_package(absl)
     # in onnx will fail due to missing protobuf::gmock target
     # If clang default C++ standard is lower than 17, you must specify
     # -DCMAKE_CXX_STANDARD=17 (or higher) which is required by protobuf
+    #
     && CC=clang CXX=clang++ \
        cmake -DCMAKE_INSTALL_LIBDIR=lib \
              -DBUILD_SHARED_LIBS=ON \
              -Dprotobuf_BUILD_TESTS=OFF .. \
     && make -j${NPROC} install && ldconfig \
     && cd .. \
+    #
     # New version of python protobuf can no longer be built with setup.py
     # and must be built with bazel.
     #
@@ -111,8 +121,8 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
     # (commit fa9ceb3d4f3fbb1542059d931bf4151a0bee316c).
     #
     # MODULE.bazel needs to be patched to bump rules_python to 1.8.0
-    # since s390x no longer has python 3.8 interpreter.
-    # rules_buf needs to be patched to support s390x arch.
+    # since s390x no longer has python 3.8 interpreter. And rules_buf
+    # needs to be patched to support s390x arch.
     #
     # clang-18 crashes on s390x when compiling absl/time/duration.cc under
     # bazel but works fine under cmake. This is because cmake and bazel
@@ -121,15 +131,19 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
     # built as a tool dependency. If abseil is being built as a target
     # dependency, we would use --per_file_copt instead.
     #
-    # After fixing all the v33.5 bugs on s390x, it turns out it has known
-    # compatibility problems and also causes onnx (and others) to segfault.
-    # So staying with v31.1 for now.
-    && sed -i -e "/strip_prefix = \"rules_buf-0.3.0\"/a \    patch_cmds = [\n        \x27\x27\x27sed -i -e \x27s|\\\\\\\\[\"arm64\", \"amd64\"\\\\\\\\]|\\\\\\\\[\"arm64\", \"amd64\", \"s390x\"\\\\\\\\]|g\x27 buf/internal/toolchain.bzl\x27\x27\x27,\n    ]," WORKSPACE \
+    && sed -i -e 's/"rules_python", version = "1.6.0"/"rules_python", version = "1.8.0"/g' \
+           MODULE.bazel \
+    && printf "\n\
+single_version_override(\n\
+    module_name = \"rules_buf\",\n\
+    patch_cmds = [\n\
+        '''sed -i -e 's|\\\\\\\\[\"arm64\", \"amd64\"\\\\\\\\]|\\\\\\\\[\"arm64\", \"amd64\", \"s390x\"\\\\\\\\]|g' buf/internal/toolchain.bzl''',\n\
+    ],\n\
+)" >> MODULE.bazel \
     && export CARGO_BAZEL_GENERATOR_URL=file:///root/.cargo/bin/cargo-bazel \
     && export CARGO_BAZEL_REPIN=true \
     && CC=clang CXX=clang++ \
-       bazel build --noenable_bzlmod \
-                   --host_per_file_copt=".*/absl/time/duration.cc@-O0" \
+       bazel build --host_per_file_copt=".*/absl/time/duration.cc@-O0" \
                    //python/dist:binary_wheel \
     && pip3 install bazel-bin/python/dist/protobuf-*.whl --prefix=/usr \
     && cd .. && rm -rf ${PROTOBUF_DIR} ${HOME}/.cache
diff --git a/docker/Dockerfile.onnx-mlir b/docker/Dockerfile.onnx-mlir
index 2208a3747a..cbb398fd90 100644
--- a/docker/Dockerfile.onnx-mlir
+++ b/docker/Dockerfile.onnx-mlir
@@ -21,20 +21,26 @@ COPY . onnx-mlir
 # CMAKE_INSTALL_LIBDIR to be lib.
 RUN ONNX_ROOT=${WORK_DIR}/onnx-mlir/third_party/onnx \
     && cd ${ONNX_ROOT} \
+    #
     # Require patching until upstreamed
+    #
     && sed -i -e 's/target_link_libraries(onnx PUBLIC onnx_proto)/target_link_libraries(onnx PUBLIC onnx_proto PUBLIC ${protobuf_ABSL_USED_TARGETS})/g' \
            -e '/absl::log_initialize/a \
           absl::log_internal_check_op\
           absl::log_internal_message\
           absl::log_internal_nullguard' CMakeLists.txt \
+    #
     # To avoid the "externally managed environment" error on ubuntu,
     # use --prefix=/usr to install into /usr/local/lib (default is
     # /usr/lib)
     # ubi9 doesn't have this problem and --prefix=/usr will install
     # into /usr/lib64 (default is /usr/lib)
-    && CC=clang CXX=clang++ \
-       CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib -DONNX_GEN_PB_TYPE_STUBS=OFF" \
-       python3 -m pip install . --prefix=/usr \
+    #
+    # Must specify --no-build-isolation to avoid protoc-gen-mypy.py
+    # segfault on s390x
+    #
+    && CC=clang CXX=clang++ CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib" \
+       python3 -m pip install . --prefix=/usr --no-build-isolation \
     && rm -rf ${HOME}/.cache
 
 ARG NPROC=4
diff --git a/docker/Dockerfile.onnx-mlir-dev b/docker/Dockerfile.onnx-mlir-dev
index db13a0c674..5aa2794719 100644
--- a/docker/Dockerfile.onnx-mlir-dev
+++ b/docker/Dockerfile.onnx-mlir-dev
@@ -16,20 +16,26 @@ COPY . onnx-mlir
 # Setup onnx
 RUN ONNX_ROOT=${WORK_DIR}/onnx-mlir/third_party/onnx \
     && cd ${ONNX_ROOT} \
+    #
     # Require patching until upstreamed
+    #
     && sed -i -e 's/target_link_libraries(onnx PUBLIC onnx_proto)/target_link_libraries(onnx PUBLIC onnx_proto PUBLIC ${protobuf_ABSL_USED_TARGETS})/g' \
            -e '/absl::log_initialize/a \
           absl::log_internal_check_op\
           absl::log_internal_message\
           absl::log_internal_nullguard' CMakeLists.txt \
+    #
     # To avoid the "externally managed environment" error on ubuntu,
     # use --prefix=/usr to install into /usr/local/lib (default is
     # /usr/lib)
     # ubi9 doesn't have this problem and --prefix=/usr will install
     # into /usr/lib64 (default is /usr/lib)
-    && CC=clang CXX=clang++ \
-       CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib -DONNX_GEN_PB_TYPE_STUBS=OFF" \
-       python3 -m pip install . --prefix=/usr
+    #
+    # Must specify --no-build-isolation to avoid protoc-gen-mypy.py
+    # segfault on s390x
+    #
+    && CC=clang CXX=clang++ CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib" \
+       python3 -m pip install . --prefix=/usr --no-build-isolation
 
 ARG NPROC=4
 ARG ACCEL=NNPA

From f06a9cd03563fecea55a66fc561fbd954d565a77 Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Wed, 11 Feb 2026 16:54:05 +0800
Subject: [PATCH 20/21] Must use archive_override instead of
 single_version_override

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 docker/Dockerfile.llvm-project | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project
index 7a9176e234..a7dc05f2ed 100644
--- a/docker/Dockerfile.llvm-project
+++ b/docker/Dockerfile.llvm-project
@@ -134,8 +134,11 @@ RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR
     && sed -i -e 's/"rules_python", version = "1.6.0"/"rules_python", version = "1.8.0"/g' \
            MODULE.bazel \
     && printf "\n\
-single_version_override(\n\
+archive_override(\n\
     module_name = \"rules_buf\",\n\
+    integrity = \"sha256-Hr64Q/CaYr0E3ptAjEOgdZd1yc+cBjp7OG1wzuf3DIs=\",\n\
+    strip_prefix = \"rules_buf-0.3.0\",\n\
+    urls = [ \"https://github.com/bufbuild/rules_buf/archive/refs/tags/v0.3.0.zip\", ],\n\
     patch_cmds = [\n\
         '''sed -i -e 's|\\\\\\\\[\"arm64\", \"amd64\"\\\\\\\\]|\\\\\\\\[\"arm64\", \"amd64\", \"s390x\"\\\\\\\\]|g' buf/internal/toolchain.bzl''',\n\
     ],\n\

From 6eee0d075ae58ee2ff854e28fbe1869accda2f11 Mon Sep 17 00:00:00 2001
From: Gong Su <gong_su@hotmail.com>
Date: Wed, 11 Feb 2026 20:21:36 +0800
Subject: [PATCH 21/21] Upgrade setuptools to avoid UNKNOWN wheel when building
 onnx on ubi9

Signed-off-by: Gong Su <gong_su@hotmail.com>
---
 docker/Dockerfile.llvm-project | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project
index a7dc05f2ed..9c3fe0231f 100644
--- a/docker/Dockerfile.llvm-project
+++ b/docker/Dockerfile.llvm-project
@@ -47,7 +47,9 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"]
               file java-21-devel gcc gcc-c++ git libtool make ncurses-devel \
               ninja-build python3 python3-devel python3-pip python3-setuptools \
               python3-wheel tzdata-java unzip which zip zlib-devel && \
-          # Use same versions as those in ubuntu:noble
+          # Upgrade setuptools to avoid UNKNOWN wheel when building onnx
+          # Use same package versions as those in ubuntu:noble
+          pip3 install -q setuptools --upgrade && \
           pip3 install -q \
                Cython pytest==7.4.4 numpy==1.26.4 pytest-xdist==3.4.0 \
                typing-extensions==4.10.0 && \