first changes after merge

Author: sandeepgupta12

.ci/docker/manywheel/Dockerfile_ppc64le

@@ -0,0 +1,138 @@
+# Use the manylinux_2_28 base image for ppc64le
+FROM quay.io/pypa/manylinux_2_28_ppc64le as base
+
+# Language variables
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+ENV LANGUAGE=C.UTF-8
+
+ARG DEVTOOLSET_VERSION=14
+
+# Create symbolic links for Python 3.12
+#RUN ln -sf /opt/python/cp312-cp312/bin/python3.12 /usr/bin/python3 && \
+#    ln -sf /opt/python/cp312-cp312/bin/python3.12 /usr/bin/python
+
+# Install required system dependencies
+RUN yum -y install epel-release && \
+    yum -y update && \
+    yum install -y \
+      sudo \
+      autoconf \
+      automake \
+      bison \
+      bzip2 \
+      curl \
+      diffutils \
+      file \
+      git \
+      make \
+      patch \
+      perl \
+      unzip \
+      util-linux \
+      wget \
+      which \
+      xz \
+      yasm \
+      less \
+      zstd \
+      libgomp \
+      gcc-toolset-${DEVTOOLSET_VERSION}-gcc \
+      gcc-toolset-${DEVTOOLSET_VERSION}-gcc-c++ \
+      gcc-toolset-${DEVTOOLSET_VERSION}-binutils \
+      gcc-toolset-${DEVTOOLSET_VERSION}-gcc-gfortran \
+      cmake \
+      ninja-build \
+      rust \
+      cargo \
+      llvm-devel \
+      libzstd-devel \
+      python3.12-devel \
+      python3.12-setuptools \
+      python3.12-pip \
+      python3-virtualenv \
+      python3.12-pyyaml \
+      python3.12-numpy \
+      python3.12-wheel \
+      python3.12-cryptography \
+      blas-devel \
+      openblas-devel \
+      lapack-devel \
+      atlas-devel \
+      libjpeg-devel \
+      libxslt-devel \
+      libxml2-devel \
+      openssl-devel \
+      valgrind
+  
+
+# Ensure the correct Python version is used
+#ENV PATH=/opt/python/cp312-cp312/bin:$PATH
+# Add gcc-toolset to the path
+ENV PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH
+ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH
+
+# Configure git to avoid safe directory issues
+RUN git config --global --add safe.directory "*"
+
+# installed python doesn't have development parts. Rebuild it from scratch
+RUN /bin/rm -rf /opt/_internal /opt/python /usr/local/*/*
+
+# EPEL for cmake
+FROM base as patchelf
+
+# Instead of ADD ./common/install_cpython.sh ...
+RUN git clone --depth 1 --branch temp-gha-runner-v2 https://github.com/sandeepgupta12/pytorch.git /tmp/pytorch && \
+    cp /tmp/pytorch/.ci/docker/common/install_patchelf.sh /build_scripts/install_patchelf.sh 
+# Install patchelf
+# ADD ./common/install_patchelf.sh install_patchelf.sh
+#RUN bash ./install_patchelf.sh && rm install_patchelf.sh
+#ENV CPYTHON_VERSIONS="3.10.1"
+#RUN echo "CPYTHON_VERSIONS=$CPYTHON_VERSIONS" && bash -x /build_scripts/install_cpython.sh
+#ENV SSL_CERT_FILE=""
+#RUN curl -fL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
+RUN bash /build_scripts/install_patchelf.sh && rm -r /build_scripts
+RUN cp $(which patchelf) /patchelf
+
+FROM patchelf as python
+
+# Clone only required scripts from the PyTorch repo
+RUN mkdir -p /build_scripts && \
+    cp -r /tmp/pytorch/.ci/docker/manywheel/build_scripts/* /build_scripts/ && \
+    cp /tmp/pytorch/.ci/docker/common/install_cpython.sh /build_scripts/install_cpython.sh && \
+    rm -rf /tmp/pytorch
+# build python
+#COPY manywheel/build_scripts /build_scripts
+#ADD ./common/install_cpython.sh /build_scripts/install_cpython.sh
+ENV SSL_CERT_FILE=
+# Run the build script
+RUN bash /build_scripts/build.sh && rm -r /build_scripts
+#RUN bash build_scripts/build.sh && rm -r build_scripts
+#RUN bash build_scripts/build.sh || (echo "Checksum verification failed!" && exit 1)
+
+FROM base as final
+COPY --from=python             /opt/python                           /opt/python
+COPY --from=python             /opt/_internal                        /opt/_internal
+COPY --from=python             /opt/python/cp39-cp39/bin/auditwheel  /usr/local/bin/auditwheel
+COPY --from=patchelf           /usr/local/bin/patchelf               /usr/local/bin/patchelf
+
+RUN alternatives --set python /usr/bin/python3.12
+RUN alternatives --set python3 /usr/bin/python3.12
+
+RUN pip-3.12 install typing_extensions
+
+# Install required Python packages
+#RUN pip install --upgrade pip
+#RUN pip install typing_extensions pyyaml setuptools
+ 
+# Install test dependencies
+RUN dnf install -y \
+      protobuf-devel \
+      protobuf-c-devel \
+      protobuf-lite-devel \
+      wget \
+      patch
+
+# Set default entrypoint
+ENTRYPOINT []
+CMD ["/bin/bash"]

.ci/docker/manywheel/build.sh

@@ -55,6 +55,12 @@ case ${image} in
         DOCKER_GPU_BUILD_ARG=""
         MANY_LINUX_VERSION="s390x"
         ;;
+    manylinuxppc64le-builder:cpu-ppc64le)
+        TARGET=final
+        GPU_IMAGE=redhat/ubi9
+        DOCKER_GPU_BUILD_ARG=""
+        MANY_LINUX_VERSION="ppc64le"
+        ;;
     manylinux2_28-builder:cuda11*)
         TARGET=cuda_final
         GPU_IMAGE=amd64/almalinux:8
@@ -102,7 +108,7 @@ if [[ -n ${MANY_LINUX_VERSION} && -z ${DOCKERFILE_SUFFIX} ]]; then
     DOCKERFILE_SUFFIX=_${MANY_LINUX_VERSION}
 fi
 # Only activate this if in CI
-if [ "$(uname -m)" != "s390x" ] && [ -v CI ]; then
+if [ "$(uname -m)" != "s390x" ] && "$(uname -m)" != "ppc64le" ] && [ -v CI ]; then
     # TODO: Remove LimitNOFILE=1048576 patch once https://github.com/pytorch/test-infra/issues/5712
     # is resolved. This patch is required in order to fix timing out of Docker build on Amazon Linux 2023.
     sudo sed -i s/LimitNOFILE=infinity/LimitNOFILE=1048576/ /usr/lib/systemd/system/docker.service

.ci/docker/manywheel/build_scripts/build.sh

@@ -20,7 +20,7 @@ AUTOCONF_HASH=954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969
 # the final image after compiling Python
 PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel libpcap-devel xz-devel libffi-devel"
 
-if [ "$(uname -m)" != "s390x" ] ; then
+if [ "$(uname -m)" != "s390x" && "$(uname -m)" != "ppc64le" ] ; then
     PYTHON_COMPILE_DEPS="${PYTHON_COMPILE_DEPS} db4-devel"
 else
     PYTHON_COMPILE_DEPS="${PYTHON_COMPILE_DEPS} libdb-devel"
@@ -39,7 +39,37 @@ yum -y install bzip2 make git patch unzip bison yasm diffutils \
     ${PYTHON_COMPILE_DEPS}
 
 # Install newest autoconf
-build_autoconf $AUTOCONF_ROOT $AUTOCONF_HASH
+# If the architecture is not ppc64le, use the existing build_autoconf function
+if [ "$(uname -m)" != "ppc64le" ] ; then
+    build_autoconf $AUTOCONF_ROOT $AUTOCONF_HASH
+else
+    # Download and extract Autoconf
+    curl -sLO http://ftp.gnu.org/gnu/autoconf/$AUTOCONF_ROOT.tar.gz
+
+    # Verify the integrity of the downloaded file using SHA-256 checksum
+    echo "$AUTOCONF_HASH  $AUTOCONF_ROOT.tar.gz" | sha256sum -c -
+
+    # Extract the downloaded tarball 
+    tar -xzf $AUTOCONF_ROOT.tar.gz
+    cd $AUTOCONF_ROOT
+
+    # Update config.guess and config.sub scripts to ensure proper architecture detection
+    curl -sLo build-aux/config.guess https://git.savannah.gnu.org/cgit/config.git/plain/config.guess
+    curl -sLo build-aux/config.sub https://git.savannah.gnu.org/cgit/config.git/plain/config.sub
+
+    chmod +x build-aux/config.guess build-aux/config.sub
+
+    # Configure the Autoconf build system with the correct host type for ppc64le
+    ./configure --host=powerpc64le-pc-linux-gnu
+
+    # Build and install
+    make -j$(nproc)
+    make install
+
+    # Clean up
+    cd ..
+    rm -rf $AUTOCONF_ROOT $AUTOCONF_ROOT.tar.gz
+fi
 autoconf --version
 
 # Compile the latest Python releases.

.ci/docker/manywheel/build_scripts/manylinux1-check.py

@@ -5,7 +5,7 @@ def is_manylinux1_compatible():
     # Only Linux, and only x86-64 / i686
     from distutils.util import get_platform
 
-    if get_platform() not in ["linux-x86_64", "linux-i686", "linux-s390x"]:
+    if get_platform() not in ["linux-x86_64", "linux-i686", "linux-s390x", "linux-ppc64le"]:
         return False
 
     # Check for presence of _manylinux module

.ci/manywheel/build.sh

@@ -15,7 +15,7 @@ case "${GPU_ARCH_TYPE:-BLANK}" in
     rocm)
         bash "${SCRIPTPATH}/build_rocm.sh"
         ;;
-    cpu | cpu-cxx11-abi | cpu-s390x)
+    cpu | cpu-cxx11-abi | cpu-s390x | cpu-ppc64le)
         bash "${SCRIPTPATH}/build_cpu.sh"
         ;;
     xpu)

.ci/manywheel/build_common.sh

@@ -367,7 +367,7 @@ for pkg in /$WHEELHOUSE_DIR/torch_no_python*.whl /$WHEELHOUSE_DIR/torch*linux*.w
     done
 
     # create Manylinux 2_28 tag this needs to happen before regenerate the RECORD
-    if [[ $PLATFORM == "manylinux_2_28_x86_64" && $GPU_ARCH_TYPE != "cpu-s390x" && $GPU_ARCH_TYPE != "xpu" ]]; then
+    if [[ $PLATFORM == "manylinux_2_28_x86_64" && $GPU_ARCH_TYPE != "cpu-s390x" && $GPU_ARCH_TYPE != "cpu-ppc64le" && $GPU_ARCH_TYPE != "xpu" ]]; then
         wheel_file=$(echo $(basename $pkg) | sed -e 's/-cp.*$/.dist-info\/WHEEL/g')
         sed -i -e s#linux_x86_64#"${PLATFORM}"# $wheel_file;
     fi
@@ -412,7 +412,7 @@ for pkg in /$WHEELHOUSE_DIR/torch_no_python*.whl /$WHEELHOUSE_DIR/torch*linux*.w
     fi
 
     # Rename wheel for Manylinux 2_28
-    if [[ $PLATFORM == "manylinux_2_28_x86_64" && $GPU_ARCH_TYPE != "cpu-s390x" && $GPU_ARCH_TYPE != "xpu" ]]; then
+    if [[ $PLATFORM == "manylinux_2_28_x86_64" && $GPU_ARCH_TYPE != "cpu-s390x" && $GPU_ARCH_TYPE != "cpu-ppc64le" && $GPU_ARCH_TYPE != "xpu" ]]; then
         pkg_name=$(echo $(basename $pkg) | sed -e s#linux_x86_64#"${PLATFORM}"#)
         zip -rq $pkg_name $PREIX*
         rm -f $pkg

.ci/manywheel/build_cpu.sh

@@ -36,6 +36,8 @@ elif [[ "$OS_NAME" == *"AlmaLinux"* ]]; then
 elif [[ "$OS_NAME" == *"Ubuntu"* ]]; then
     if [[ "$(uname -m)" == "s390x" ]]; then
         LIBGOMP_PATH="/usr/lib/s390x-linux-gnu/libgomp.so.1"
+    elif [[ "$(uname -m)" == "ppc64le" ]]; then
+        LIBGOMP_PATH="/usr/lib64/libgomp.so.1"
     else
         LIBGOMP_PATH="/usr/lib/x86_64-linux-gnu/libgomp.so.1"
     fi

.ci/pytorch/build.sh

@@ -237,7 +237,7 @@ fi
 
 # Do not change workspace permissions for ROCm and s390x CI jobs
 # as it can leave workspace with bad permissions for cancelled jobs
-if [[ "$BUILD_ENVIRONMENT" != *rocm* && "$BUILD_ENVIRONMENT" != *s390x* && -d /var/lib/jenkins/workspace ]]; then
+if [[ "$BUILD_ENVIRONMENT" != *rocm* && "$BUILD_ENVIRONMENT" != *s390x* && "$BUILD_ENVIRONMENT" != *ppc64le* && -d /var/lib/jenkins/workspace ]]; then
   # Workaround for dind-rootless userid mapping (https://github.com/pytorch/ci-infra/issues/96)
   WORKSPACE_ORIGINAL_OWNER_ID=$(stat -c '%u' "/var/lib/jenkins/workspace")
   cleanup_workspace() {
@@ -416,6 +416,6 @@ if [[ "$BUILD_ENVIRONMENT" != *libtorch* && "$BUILD_ENVIRONMENT" != *bazel* ]];
   python tools/stats/export_test_times.py
 fi
 # don't do this for bazel or s390x as they don't use sccache
-if [[ "$BUILD_ENVIRONMENT" != *s390x* && "$BUILD_ENVIRONMENT" != *-bazel-* ]]; then
+if [[ "$BUILD_ENVIRONMENT" != *s390x* && "$BUILD_ENVIRONMENT" != *ppc64le* && "$BUILD_ENVIRONMENT" != *-bazel-* ]]; then
   print_sccache_stats
 fi

.ci/pytorch/check_binary.sh

@@ -184,7 +184,7 @@ fi
 if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then
   echo "Checking that MKL is available"
   build_and_run_example_cpp check-torch-mkl
-elif [[ "$(uname -m)" != "arm64" && "$(uname -m)" != "s390x" ]]; then
+elif [[ "$(uname -m)" != "arm64" && "$(uname -m)" != "s390x" && "$(uname -m)" != "ppc64le" ]]; then
   if [[ "$(uname)" != 'Darwin' || "$PACKAGE_TYPE" != *wheel ]]; then
     if [[ "$(uname -m)" == "aarch64" ]]; then
       echo "Checking that MKLDNN is available on aarch64"
@@ -208,7 +208,7 @@ if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then
   echo "Checking that XNNPACK is available"
   build_and_run_example_cpp check-torch-xnnpack
 else
-  if [[ "$(uname)" != 'Darwin' || "$PACKAGE_TYPE" != *wheel ]] && [[ "$(uname -m)" != "s390x"  ]]; then
+  if [[ "$(uname)" != 'Darwin' || "$PACKAGE_TYPE" != *wheel ]] && [[ "$(uname -m)" != "s390x" && "$(uname -m)" != "ppc64le"  ]]; then
     echo "Checking that XNNPACK is available"
     pushd /tmp
     python -c 'import torch.backends.xnnpack; exit(0 if torch.backends.xnnpack.enabled else 1)'
@@ -237,7 +237,7 @@ if [[ "$OSTYPE" == "msys" ]]; then
 fi
 
 # Test that CUDA builds are setup correctly
-if [[ "$DESIRED_CUDA" != 'cpu' && "$DESIRED_CUDA" != 'xpu' && "$DESIRED_CUDA" != 'cpu-cxx11-abi' && "$DESIRED_CUDA" != *"rocm"* && "$(uname -m)" != "s390x" ]]; then
+if [[ "$DESIRED_CUDA" != 'cpu' && "$DESIRED_CUDA" != 'xpu' && "$DESIRED_CUDA" != 'cpu-cxx11-abi' && "$DESIRED_CUDA" != *"rocm"* && "$(uname -m)" != "s390x" && "$(uname -m)" != "ppc64le" ]]; then
   if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then
     build_and_run_example_cpp check-torch-cuda
   else
@@ -311,7 +311,7 @@ if [[ "$(uname)" == 'Linux' &&  "$PACKAGE_TYPE" == 'manywheel' ]]; then
   # gcc 11 - CUDA 11.8, xpu, rocm
   # gcc 13 - CUDA 12.6, 12.8 and cpu
   # Please see issue for reference: https://github.com/pytorch/pytorch/issues/152426
-  if [[ "$(uname -m)" == "s390x" ]]; then
+  if [[ "$(uname -m)" == "s390x" && "$(uname -m)" != "ppc64le" ]]; then
     cxx_abi="19"
   elif [[ "$DESIRED_CUDA" != 'xpu' && "$DESIRED_CUDA" != 'rocm'* ]]; then
     cxx_abi="18"

.github/actions/test-pytorch-binary/action.yml

@@ -34,7 +34,7 @@ runs:
         )
 
         echo "CONTAINER_NAME=${container_name}" >> "$GITHUB_ENV"
-        if [[ "${GPU_ARCH_TYPE}" != "rocm" && "${BUILD_ENVIRONMENT}" != "linux-aarch64-binary-manywheel" && "${BUILD_ENVIRONMENT}" != "linux-s390x-binary-manywheel" && "${GPU_ARCH_TYPE}" != "xpu" ]]; then
+        if [[ "${GPU_ARCH_TYPE}" != "rocm" && "${BUILD_ENVIRONMENT}" != "linux-aarch64-binary-manywheel" && "${BUILD_ENVIRONMENT}" != "linux-s390x-binary-manywheel" && "${BUILD_ENVIRONMENT}" != "linux-ppc64le-binary-manywheel" && "${GPU_ARCH_TYPE}" != "xpu" ]]; then
           # Propagate download.pytorch.org IP to container. This is only needed on Linux non aarch64 runner
           grep download.pytorch.org /etc/hosts | docker exec -i "${container_name}" bash -c "/bin/cat >> /etc/hosts"
         fi
@@ -45,9 +45,9 @@ runs:
         docker exec -t "${container_name}" bash -c "source ${BINARY_ENV_FILE} && bash -x /run.sh"
 
     - name: Cleanup docker
-      if: always() && (env.BUILD_ENVIRONMENT == 'linux-s390x-binary-manywheel' || env.GPU_ARCH_TYPE == 'xpu')
+      if: always() && (env.BUILD_ENVIRONMENT == 'linux-s390x-binary-manywheel' || env.BUILD_ENVIRONMENT != 'linux-ppc64le-binary-manywheel' || env.GPU_ARCH_TYPE == 'xpu')
       shell: bash
       run: |
-        # on s390x or xpu stop the container for clean worker stop
+        # on s390x or ppc64le or xpu stop the container for clean worker stop
         # shellcheck disable=SC2046
         docker stop "${{ env.CONTAINER_NAME }}" || true