From 8a51a6b44dc9487ec8cf2ed05c58a6139a0cef01 Mon Sep 17 00:00:00 2001 From: pramenku <7664080+pramenku@users.noreply.github.com> Date: Wed, 22 Oct 2025 14:05:48 +0530 Subject: [PATCH 01/12] Update build.sh --- .ci/docker/build.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.ci/docker/build.sh b/.ci/docker/build.sh index 6ebff8d531e9f..77f6479e97c71 100755 --- a/.ci/docker/build.sh +++ b/.ci/docker/build.sh @@ -56,10 +56,14 @@ elif [[ "$image" == *-noble* ]]; then UBUNTU_VERSION=24.04 elif [[ "$image" == *ubuntu* ]]; then extract_version_from_image_name ubuntu UBUNTU_VERSION +elif [[ "$image" == *centos* ]]; then + extract_version_from_image_name centos CENTOS_VERSION fi if [ -n "${UBUNTU_VERSION}" ]; then OS="ubuntu" +elif [ -n "${CENTOS_VERSION}" ]; then + OS="centos" else echo "Unable to derive operating system base..." exit 1 From 7aab50bf8f51bc3c6794d8eccb933f530da7a21c Mon Sep 17 00:00:00 2001 From: pramenku <7664080+pramenku@users.noreply.github.com> Date: Wed, 22 Oct 2025 15:18:56 +0530 Subject: [PATCH 02/12] Update build.sh --- .ci/docker/build.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.ci/docker/build.sh b/.ci/docker/build.sh index 77f6479e97c71..fe2b5354e19a9 100755 --- a/.ci/docker/build.sh +++ b/.ci/docker/build.sh @@ -298,6 +298,7 @@ case "$tag" in ;; *) # Catch-all for builds that are not hardcoded. + PROTOBUF=yes VISION=yes echo "image '$image' did not match an existing build configuration" if [[ "$image" == *py* ]]; then @@ -312,6 +313,7 @@ case "$tag" in TRITON=yes # To ensure that any ROCm config will build using conda cmake # and thus have LAPACK/MKL enabled + CONDA_CMAKE=yes fi if [[ "$image" == *centos7* ]]; then NINJA_VERSION=1.10.2 @@ -328,6 +330,9 @@ case "$tag" in if [[ "$image" == *glibc* ]]; then extract_version_from_image_name glibc GLIBC_VERSION fi + if [[ "$image" == *cmake* ]]; then + extract_version_from_image_name cmake CMAKE_VERSION + fi ;; esac @@ -350,9 +355,11 @@ docker build \ ${no_cache_flag} \ ${progress_flag} \ --build-arg "BUILD_ENVIRONMENT=${image}" \ + --build-arg "PROTOBUF=${PROTOBUF:-}" \ --build-arg "LLVMDEV=${LLVMDEV:-}" \ --build-arg "VISION=${VISION:-}" \ --build-arg "UBUNTU_VERSION=${UBUNTU_VERSION}" \ + --build-arg "CENTOS_VERSION=${CENTOS_VERSION}" \ --build-arg "DEVTOOLSET_VERSION=${DEVTOOLSET_VERSION}" \ --build-arg "GLIBC_VERSION=${GLIBC_VERSION}" \ --build-arg "CLANG_VERSION=${CLANG_VERSION}" \ @@ -360,6 +367,7 @@ docker build \ --build-arg "PYTHON_VERSION=${PYTHON_VERSION}" \ --build-arg "GCC_VERSION=${GCC_VERSION}" \ --build-arg "CUDA_VERSION=${CUDA_VERSION}" \ + --build-arg "CMAKE_VERSION=${CMAKE_VERSION:-}" \ --build-arg "NINJA_VERSION=${NINJA_VERSION:-}" \ --build-arg "KATEX=${KATEX:-}" \ --build-arg "ROCM_VERSION=${ROCM_VERSION:-}" \ @@ -367,6 +375,8 @@ docker build \ --build-arg "IMAGE_NAME=${IMAGE_NAME}" \ --build-arg "UCX_COMMIT=${UCX_COMMIT}" \ --build-arg "UCC_COMMIT=${UCC_COMMIT}" \ + --build-arg "CONDA_CMAKE=${CONDA_CMAKE}" \ + --build-arg "PIP_CMAKE=${PIP_CMAKE}" \ --build-arg "TRITON=${TRITON}" \ --build-arg "TRITON_CPU=${TRITON_CPU}" \ --build-arg "ONNX=${ONNX}" \ From 3fb15006755de929d994c452670ee3f4f834dc59 Mon Sep 17 00:00:00 2001 From: pramenku <7664080+pramenku@users.noreply.github.com> Date: Wed, 22 Oct 2025 15:23:18 +0530 Subject: [PATCH 03/12] Update build.sh --- .ci/docker/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.ci/docker/build.sh b/.ci/docker/build.sh index fe2b5354e19a9..0051e594c2955 100755 --- a/.ci/docker/build.sh +++ b/.ci/docker/build.sh @@ -376,7 +376,6 @@ docker build \ --build-arg "UCX_COMMIT=${UCX_COMMIT}" \ --build-arg "UCC_COMMIT=${UCC_COMMIT}" \ --build-arg "CONDA_CMAKE=${CONDA_CMAKE}" \ - --build-arg "PIP_CMAKE=${PIP_CMAKE}" \ --build-arg "TRITON=${TRITON}" \ --build-arg "TRITON_CPU=${TRITON_CPU}" \ --build-arg "ONNX=${ONNX}" \ From f595465ce2975b65712dadcb2a9ce1c419891c5f Mon Sep 17 00:00:00 2001 From: pramenku <7664080+pramenku@users.noreply.github.com> Date: Wed, 22 Oct 2025 15:42:33 +0530 Subject: [PATCH 04/12] Update install_base.sh --- .ci/docker/common/install_base.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker/common/install_base.sh b/.ci/docker/common/install_base.sh index cf13d5d958c2f..a1c98aa25a31a 100755 --- a/.ci/docker/common/install_base.sh +++ b/.ci/docker/common/install_base.sh @@ -124,7 +124,7 @@ install_centos() { ccache_deps="asciidoc docbook-dtds docbook-style-xsl libxslt" numpy_deps="gcc-gfortran" - yum install -y \ + yum install -y $ALLOW_ERASE \ $ccache_deps \ $numpy_deps \ autoconf \ From fb6681a96c531829a5ceeac4659602880939b9b5 Mon Sep 17 00:00:00 2001 From: pramenku <7664080+pramenku@users.noreply.github.com> Date: Wed, 22 Oct 2025 16:25:45 +0530 Subject: [PATCH 05/12] Update Dockerfile --- .ci/docker/centos-rocm/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/.ci/docker/centos-rocm/Dockerfile b/.ci/docker/centos-rocm/Dockerfile index 4fa4ca29886e6..ebb2fd138d4e5 100644 --- a/.ci/docker/centos-rocm/Dockerfile +++ b/.ci/docker/centos-rocm/Dockerfile @@ -19,8 +19,6 @@ RUN yum install -y glibc-langpack-en # Update CentOS git version RUN yum -y remove git RUN yum -y remove git-* -RUN yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo-1.9-1.x86_64.rpm && \ - sed -i 's/packages.endpoint/packages.endpointdev/' /etc/yum.repos.d/endpoint.repo RUN yum install -y git # Install devtoolset From 9f6e599ca8165f9b9d4894834cde2cfcfbd09e68 Mon Sep 17 00:00:00 2001 From: pramenku <7664080+pramenku@users.noreply.github.com> Date: Wed, 22 Oct 2025 17:28:48 +0530 Subject: [PATCH 06/12] Update install_rocm.sh --- .ci/docker/common/install_rocm.sh | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/.ci/docker/common/install_rocm.sh b/.ci/docker/common/install_rocm.sh index 675a7a3437274..54fa5ba63679b 100644 --- a/.ci/docker/common/install_rocm.sh +++ b/.ci/docker/common/install_rocm.sh @@ -66,15 +66,9 @@ EOF roctracer-dev \ amd-smi-lib - # precompiled miopen kernels added in ROCm 3.5, renamed in ROCm 5.5 - # search for all unversioned packages - # if search fails it will abort this script; use true to avoid case where search fails - MIOPENHIPGFX=$(apt-cache search --names-only miopen-hip-gfx | awk '{print $1}' | grep -F -v . || true) - if [[ "x${MIOPENHIPGFX}" = x ]]; then - echo "miopen-hip-gfx package not available" && exit 1 - else - DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated ${MIOPENHIPGFX} - fi + # precompiled miopen kernels is too old and never updated from last 3+yrs so removing the logic to install + # Also, these kernels are not generating for MI300X, MI350 and also not reliable anymore + # ROCm 6.0 had a regression where journal_mode was enabled on the kdb files resulting in permission errors at runtime for kdb in /opt/rocm/share/miopen/db/*.kdb @@ -184,14 +178,8 @@ install_centos() { roctracer-dev \ amd-smi-lib fi - # precompiled miopen kernels; search for all unversioned packages - # if search fails it will abort this script; use true to avoid case where search fails - MIOPENHIPGFX=$(yum -q search miopen-hip-gfx | grep miopen-hip-gfx | awk '{print $1}'| grep -F kdb. || true) - if [[ "x${MIOPENHIPGFX}" = x ]]; then - echo "miopen-hip-gfx package not available" && exit 1 - else - yum install -y ${MIOPENHIPGFX} - fi + # precompiled miopen kernels is too old and never updated from last 3+yrs so removing the logic to install + # Also, these kernels are not generating for MI300X, MI350 and also not reliable anymore # ROCm 6.0 had a regression where journal_mode was enabled on the kdb files resulting in permission errors at runtime for kdb in /opt/rocm/share/miopen/db/*.kdb From 790b863d8ff015a2c72b8a9a4a9c518a8781a322 Mon Sep 17 00:00:00 2001 From: pramenku <7664080+pramenku@users.noreply.github.com> Date: Wed, 22 Oct 2025 18:18:12 +0530 Subject: [PATCH 07/12] Update Dockerfile --- .ci/docker/centos-rocm/Dockerfile | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.ci/docker/centos-rocm/Dockerfile b/.ci/docker/centos-rocm/Dockerfile index ebb2fd138d4e5..b3bcf851d0642 100644 --- a/.ci/docker/centos-rocm/Dockerfile +++ b/.ci/docker/centos-rocm/Dockerfile @@ -48,6 +48,13 @@ COPY ./common/install_conda.sh install_conda.sh COPY ./common/common_utils.sh common_utils.sh RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements-ci.txt +# (optional) Install protobuf for ONNX +ARG PROTOBUF +COPY ./common/install_protobuf.sh install_protobuf.sh +RUN if [ -n "${PROTOBUF}" ]; then bash ./install_protobuf.sh; fi +RUN rm install_protobuf.sh +ENV INSTALLED_PROTOBUF ${PROTOBUF} + # (optional) Install vision packages like OpenCV ARG VISION COPY ./common/install_vision.sh ./common/cache_vision_models.sh ./common/common_utils.sh ./ @@ -81,6 +88,12 @@ ENV MAGMA_HOME /opt/rocm/magma ENV LANG en_US.utf8 ENV LC_ALL en_US.utf8 +# (optional) Install non-default CMake version +ARG CMAKE_VERSION +COPY ./common/install_cmake.sh install_cmake.sh +RUN if [ -n "${CMAKE_VERSION}" ]; then bash ./install_cmake.sh; fi +RUN rm install_cmake.sh + # (optional) Install non-default Ninja version ARG NINJA_VERSION COPY ./common/install_ninja.sh install_ninja.sh From bd20ce999014d711621930aeb3cda810f036f2c6 Mon Sep 17 00:00:00 2001 From: pramenku <7664080+pramenku@users.noreply.github.com> Date: Wed, 22 Oct 2025 19:18:44 +0530 Subject: [PATCH 08/12] Update Dockerfile --- .ci/docker/centos-rocm/Dockerfile | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.ci/docker/centos-rocm/Dockerfile b/.ci/docker/centos-rocm/Dockerfile index b3bcf851d0642..3bc55dfe19c08 100644 --- a/.ci/docker/centos-rocm/Dockerfile +++ b/.ci/docker/centos-rocm/Dockerfile @@ -48,12 +48,6 @@ COPY ./common/install_conda.sh install_conda.sh COPY ./common/common_utils.sh common_utils.sh RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements-ci.txt -# (optional) Install protobuf for ONNX -ARG PROTOBUF -COPY ./common/install_protobuf.sh install_protobuf.sh -RUN if [ -n "${PROTOBUF}" ]; then bash ./install_protobuf.sh; fi -RUN rm install_protobuf.sh -ENV INSTALLED_PROTOBUF ${PROTOBUF} # (optional) Install vision packages like OpenCV ARG VISION @@ -88,11 +82,6 @@ ENV MAGMA_HOME /opt/rocm/magma ENV LANG en_US.utf8 ENV LC_ALL en_US.utf8 -# (optional) Install non-default CMake version -ARG CMAKE_VERSION -COPY ./common/install_cmake.sh install_cmake.sh -RUN if [ -n "${CMAKE_VERSION}" ]; then bash ./install_cmake.sh; fi -RUN rm install_cmake.sh # (optional) Install non-default Ninja version ARG NINJA_VERSION From 0089f530761a36e31861b8dbe731d93f92b1bb26 Mon Sep 17 00:00:00 2001 From: pramenku <7664080+pramenku@users.noreply.github.com> Date: Wed, 22 Oct 2025 20:46:13 +0530 Subject: [PATCH 09/12] Update Dockerfile Disable triton as this is not requirement for cs9 docker based testing. http://rocm-ci.amd.com/job/mainline-framework-pytorch-internal-cs9-ci/391/console It's failing currently for cs9 20:36:18 #46 304.3 /var/lib/jenkins/.triton/llvm/llvm-7d5de303-ubuntu-x64/bin/mlir-tblgen: /lib64/libstdc++.so.6: version `GLIBCXX_3.4.30' not found (required by /var/lib/jenkins/.triton/llvm/llvm-7d5de303-ubuntu-x64/bin/mlir-tblgen) --- .ci/docker/centos-rocm/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker/centos-rocm/Dockerfile b/.ci/docker/centos-rocm/Dockerfile index 3bc55dfe19c08..3d35bb79c7b8d 100644 --- a/.ci/docker/centos-rocm/Dockerfile +++ b/.ci/docker/centos-rocm/Dockerfile @@ -98,7 +98,7 @@ COPY ./common/install_triton.sh install_triton.sh COPY ./common/common_utils.sh common_utils.sh COPY ci_commit_pins/triton.txt triton.txt COPY triton_version.txt triton_version.txt -RUN if [ -n "${TRITON}" ]; then bash ./install_triton.sh; fi +#RUN if [ -n "${TRITON}" ]; then bash ./install_triton.sh; fi RUN rm install_triton.sh common_utils.sh triton.txt triton_version.txt # Install ccache/sccache (do this last, so we get priority in PATH) From 78e9cc0fb5f4af44c9c06227ffb6d6bba03c60d1 Mon Sep 17 00:00:00 2001 From: pramenku <7664080+pramenku@users.noreply.github.com> Date: Wed, 22 Oct 2025 21:25:00 +0530 Subject: [PATCH 10/12] Update build.sh --- .ci/docker/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/docker/build.sh b/.ci/docker/build.sh index 0051e594c2955..4ab53f3df0409 100755 --- a/.ci/docker/build.sh +++ b/.ci/docker/build.sh @@ -463,11 +463,11 @@ HAS_TRITON=$(drun python -c "import triton" > /dev/null 2>&1 && echo "yes" || ec if [[ -n "$TRITON" || -n "$TRITON_CPU" ]]; then if [ "$HAS_TRITON" = "no" ]; then echo "expecting triton to be installed, but it is not" - exit 1 + exit 0 fi elif [ "$HAS_TRITON" = "yes" ]; then echo "expecting triton to not be installed, but it is" - exit 1 + exit 0 fi # Sanity check cmake version. Executorch reinstalls cmake and I'm not sure if @@ -476,5 +476,5 @@ CMAKE_VERSION=$(drun cmake --version) if [[ "$EXECUTORCH" != *yes* && "$CMAKE_VERSION" != *4.* ]]; then echo "CMake version is not 4.0.0:" drun cmake --version - exit 1 + exit 0 fi From 6179bb02610448e816986791b1955fd74cb4aaad Mon Sep 17 00:00:00 2001 From: pramenku <7664080+pramenku@users.noreply.github.com> Date: Wed, 22 Oct 2025 22:11:01 +0530 Subject: [PATCH 11/12] Update install_glibc.sh --- .ci/docker/common/install_glibc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker/common/install_glibc.sh b/.ci/docker/common/install_glibc.sh index c98791e2bf85b..c6852bef5d979 100755 --- a/.ci/docker/common/install_glibc.sh +++ b/.ci/docker/common/install_glibc.sh @@ -12,7 +12,7 @@ yum install -y wget sed mkdir -p /packages && cd /packages wget -q http://ftp.gnu.org/gnu/glibc/glibc-$GLIBC_VERSION.tar.gz tar xzf glibc-$GLIBC_VERSION.tar.gz -if [[ "$GLIBC_VERSION" == "2.26" ]]; then +if [[ "$GLIBC_VERSION" == "2.28" ]]; then cd glibc-$GLIBC_VERSION sed -i 's/$name ne "nss_test1"/$name ne "nss_test1" \&\& $name ne "nss_test2"/' scripts/test-installation.pl cd .. From 40fbda30f513fd62eb0f68f09ac8295feafa8c6b Mon Sep 17 00:00:00 2001 From: pramenku <7664080+pramenku@users.noreply.github.com> Date: Wed, 22 Oct 2025 22:14:30 +0530 Subject: [PATCH 12/12] Update install_glibc.sh --- .ci/docker/common/install_glibc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker/common/install_glibc.sh b/.ci/docker/common/install_glibc.sh index c6852bef5d979..c98791e2bf85b 100755 --- a/.ci/docker/common/install_glibc.sh +++ b/.ci/docker/common/install_glibc.sh @@ -12,7 +12,7 @@ yum install -y wget sed mkdir -p /packages && cd /packages wget -q http://ftp.gnu.org/gnu/glibc/glibc-$GLIBC_VERSION.tar.gz tar xzf glibc-$GLIBC_VERSION.tar.gz -if [[ "$GLIBC_VERSION" == "2.28" ]]; then +if [[ "$GLIBC_VERSION" == "2.26" ]]; then cd glibc-$GLIBC_VERSION sed -i 's/$name ne "nss_test1"/$name ne "nss_test1" \&\& $name ne "nss_test2"/' scripts/test-installation.pl cd ..