Merge branch 'main' into ppc64le-ci-wheel-build-v1

Author: sandeepgupta12

.ci/aarch64_linux/build_aarch64_wheel.py

@@ -1012,7 +1012,7 @@ def parse_arguments():
         install_condaforge_python(host, args.python_version)
         sys.exit(0)
 
-    python_version = args.python_version if args.python_version is not None else "3.8"
+    python_version = args.python_version if args.python_version is not None else "3.9"
 
     if args.use_torch_from_pypi:
         configure_system(host, compiler=args.compiler, python_version=python_version)

.ci/docker/ci_commit_pins/executorch.txt

@@ -1 +1 @@
-01a22b6f16d117454b7d21ebdc691b0785b84a7f
+cedf52aa8e4df879886270a5920da6fe84cbaa67

.ci/docker/common/install_base.sh

@@ -37,7 +37,7 @@ install_ubuntu() {
   if [[ "$UBUNTU_VERSION" == "20.04"* && "$CUDA_VERSION" == "11.8"* ]]; then
     maybe_libnccl_dev="libnccl2=2.15.5-1+cuda11.8 libnccl-dev=2.15.5-1+cuda11.8 --allow-downgrades --allow-change-held-packages"
   elif [[ "$UBUNTU_VERSION" == "20.04"* && "$CUDA_VERSION" == "12.4"* ]]; then
-    maybe_libnccl_dev="libnccl2=2.25.1-1+cuda12.4 libnccl-dev=2.25.1-1+cuda12.4 --allow-downgrades --allow-change-held-packages"
+    maybe_libnccl_dev="libnccl2=2.26.2-1+cuda12.4 libnccl-dev=2.26.2-1+cuda12.4 --allow-downgrades --allow-change-held-packages"
   else
     maybe_libnccl_dev=""
   fi

.ci/docker/common/install_cache.sh

@@ -2,43 +2,27 @@
 
 set -ex
 
-SCCACHE_VERSION="0.9.1"
-
-CARGO_FLAGS=""
-
-install_prereqs_ubuntu() {
+install_ubuntu() {
+  echo "Preparing to build sccache from source"
   apt-get update
   # libssl-dev will not work as it is upgraded to libssl3 in Ubuntu-22.04.
   # Instead use lib and headers from OpenSSL1.1 installed in `install_openssl.sh``
   apt-get install -y cargo
-
-  # cleanup after ourselves
-  trap 'cleanup_ubuntu' EXIT
-
-  echo "Downloading old sccache binary from S3 repo for PCH builds"
-  curl --retry 3 https://s3.amazonaws.com/ossci-linux/sccache -o /opt/cache/bin/sccache-0.2.14a
-  chmod 755 /opt/cache/bin/sccache-0.2.14a
-}
-
-cleanup_ubuntu() {
+  echo "Checking out sccache repo"
+  git clone https://github.com/mozilla/sccache -b v0.9.1
+  cd sccache
+  echo "Building sccache"
+  cargo build --release
+  cp target/release/sccache /opt/cache/bin
+  echo "Cleaning up"
+  cd ..
   rm -rf sccache
   apt-get remove -y cargo rustc
   apt-get autoclean && apt-get clean
-}
 
-install_prereqs_almalinux() {
-  dnf install -y cargo
-  # use vendored openssl, we're not going to use the dist-server anyways
-  CARGO_FEATURES="--bin sccache --features openssl/vendored"
-}
-
-build_and_install_sccache() {
-  # modern version of git don't like openssl1.1
-  wget -q -O sccache.tar.gz "https://github.com/mozilla/sccache/archive/refs/tags/v${SCCACHE_VERSION}.tar.gz"
-  tar xzf sccache.tar.gz
-  pushd "sccache-${SCCACHE_VERSION}"
-  cargo build --release ${CARGO_FEATURES}
-  cp target/release/sccache /opt/cache/bin
+  echo "Downloading old sccache binary from S3 repo for PCH builds"
+  curl --retry 3 https://s3.amazonaws.com/ossci-linux/sccache -o /opt/cache/bin/sccache-0.2.14a
+  chmod 755 /opt/cache/bin/sccache-0.2.14a
 }
 
 install_binary() {
@@ -51,21 +35,8 @@ mkdir -p /opt/cache/lib
 sed -e 's|PATH="\(.*\)"|PATH="/opt/cache/bin:\1"|g' -i /etc/environment
 export PATH="/opt/cache/bin:$PATH"
 
-echo "Preparing to build sccache from source"
-DIST_ID=$(. /etc/os-release && echo "$ID")
-case ${DIST_ID} in
-  ubuntu)
-    install_prereqs_ubuntu
-    ;;
-  almalinux)
-    install_prereqs_almalinux
-    ;;
-  *)
-    echo "ERROR: Unknown distribution ${DIST_ID}"
-    exit 1
-    ;;
-esac
-build_and_install_sccache
+# Setup compiler cache
+install_ubuntu
 chmod a+x /opt/cache/bin/sccache
 
 function write_sccache_stub() {

.ci/docker/common/install_conda.sh

@@ -62,7 +62,7 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
 
   # libstdcxx from conda default channels are too old, we need GLIBCXX_3.4.30
   # which is provided in libstdcxx 12 and up.
-  conda_install libstdcxx-ng=12.3.0 -c conda-forge
+  conda_install libstdcxx-ng=12.3.0 --update-deps -c conda-forge
 
   # Install PyTorch conda deps, as per https://github.com/pytorch/pytorch README
   if [[ $(uname -m) == "aarch64" ]]; then

.ci/docker/common/install_cpython.sh

@@ -7,7 +7,7 @@ PYTHON_DOWNLOAD_GITHUB_BRANCH=https://github.com/python/cpython/archive/refs/hea
 GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py
 
 # Python versions to be installed in /opt/$VERSION_NO
-CPYTHON_VERSIONS=${CPYTHON_VERSIONS:-"3.8.1 3.9.0 3.10.1 3.11.0 3.12.0 3.13.0 3.13.0t"}
+CPYTHON_VERSIONS=${CPYTHON_VERSIONS:-"3.9.0 3.10.1 3.11.0 3.12.0 3.13.0 3.13.0t"}
 
 function check_var {
     if [ -z "$1" ]; then

.ci/docker/common/install_cuda.sh

@@ -2,7 +2,7 @@
 
 set -ex
 
-NCCL_VERSION=v2.25.1-1
+NCCL_VERSION=v2.26.2-1
 CUDNN_VERSION=9.5.1.17
 
 function install_cusparselt_040 {

.ci/docker/manywheel/Dockerfile_2_28

@@ -60,11 +60,6 @@ FROM base as libpng
 ADD ./common/install_libpng.sh install_libpng.sh
 RUN bash ./install_libpng.sh && rm install_libpng.sh
 
-FROM base as cache
-# Install sccache
-ADD ./common/install_cache.sh install_cache.sh
-RUN bash ./install_cache.sh && rm install_cache.sh
-
 FROM ${GPU_IMAGE} as common
 ARG DEVTOOLSET_VERSION=11
 ENV LC_ALL en_US.UTF-8
@@ -117,10 +112,6 @@ COPY --from=libpng             /usr/local/include/libpng*            /usr/local/
 COPY --from=libpng             /usr/local/lib/libpng*                /usr/local/lib/
 COPY --from=libpng             /usr/local/lib/pkgconfig              /usr/local/lib/pkgconfig
 COPY --from=jni                /usr/local/include/jni.h              /usr/local/include/jni.h
-COPY --from=cache              /opt/cache                            /opt/cache
-
-# Ensure sccache is included in the path
-ENV PATH /opt/cache/bin:$PATH
 
 FROM common as cpu_final
 ARG BASE_CUDA_VERSION=11.8

.ci/docker/manywheel/Dockerfile_2_28_aarch64

@@ -59,11 +59,6 @@ FROM base as openblas
 ADD ./common/install_openblas.sh install_openblas.sh
 RUN bash ./install_openblas.sh && rm install_openblas.sh
 
-FROM base as cache
-# Install sccache
-ADD ./common/install_cache.sh install_cache.sh
-RUN bash ./install_cache.sh && rm install_cache.sh
-
 FROM base as final
 
 # remove unncessary python versions
@@ -72,8 +67,4 @@ RUN rm -rf /opt/python/cp26-cp26mu /opt/_internal/cpython-2.6.9-ucs4
 RUN rm -rf /opt/python/cp33-cp33m /opt/_internal/cpython-3.3.6
 RUN rm -rf /opt/python/cp34-cp34m /opt/_internal/cpython-3.4.6
 COPY --from=openblas     /opt/OpenBLAS/  /opt/OpenBLAS/
-COPY --from=cache              /opt/cache                            /opt/cache
-
 ENV LD_LIBRARY_PATH=/opt/OpenBLAS/lib:$LD_LIBRARY_PATH
-# Ensure sccache is included in the path
-ENV PATH /opt/cache/bin:$PATH

.ci/docker/manywheel/Dockerfile_s390x

@@ -86,20 +86,11 @@ ADD ./common/install_cpython.sh /build_scripts/install_cpython.sh
 ENV SSL_CERT_FILE=
 RUN bash build_scripts/build.sh && rm -r build_scripts
 
-FROM base as cache
-# Install sccache
-ADD ./common/install_cache.sh install_cache.sh
-RUN bash ./install_cache.sh && rm install_cache.sh
-
 FROM base as final
 COPY --from=python             /opt/python                           /opt/python
 COPY --from=python             /opt/_internal                        /opt/_internal
 COPY --from=python             /opt/python/cp39-cp39/bin/auditwheel  /usr/local/bin/auditwheel
 COPY --from=patchelf           /usr/local/bin/patchelf               /usr/local/bin/patchelf
-COPY --from=cache              /opt/cache                            /opt/cache
-
-# Ensure sccache is included in the path
-ENV PATH /opt/cache/bin:$PATH
 
 RUN alternatives --set python /usr/bin/python3.12
 RUN alternatives --set python3 /usr/bin/python3.12