Address review comments:Fix issue with Docker image load as per review

Update runner registration flow based on feedback

Enhance security by unmounting and removing runner token file

To prevent any potential token leakage, unmount and remove /run/runner_secret immediately after generating the token. This ensures that the token is inaccessible beyond its intended use, even within the job execution.

Author: sandeepgupta12

.ci/docker/manywheel/Dockerfile_ppc64le

@@ -1,5 +1,5 @@
-# Use UBI 9.3 as base image
-FROM registry.access.redhat.com/ubi9/ubi:9.5
+# Use UBI 9 as base image
+FROM registry.access.redhat.com/ubi9/ubi:9.5 AS base
 
 # Install necessary dependencies
 RUN dnf install -y \
@@ -20,20 +20,12 @@ RUN ln -sf /usr/bin/python3 /usr/bin/python && \
 
 COPY requirements.txt .
 # Install Python packages via pip
-RUN pip install wheel setuptools pyyaml typing_extensions expecttest
-
-#RUN source /opt/rh/gcc-toolset-13/enable && pip install -r requirements.txt
+RUN pip install wheel 
 RUN pip install -r requirements.txt
 
-# Copy the PyTorch source code into the container
-COPY . /workspace/pytorch
-
-WORKDIR /workspace/pytorch
-
-# Ensure submodules are initialized and updated
-RUN git submodule update --init --recursive
+RUN mkdir -p /workspace/pytorch
 
-# Copy the build script and make it executable
-COPY .github/scripts/ppc64le-build.sh /ppc64le-build.sh
+ENTRYPOINT []
+CMD ["/bin/bash"]
 
 

.ci/docker/manywheel/build.sh

@@ -65,6 +65,13 @@ case ${GPU_ARCH_TYPE} in
         DOCKER_GPU_BUILD_ARG=""
         MANY_LINUX_VERSION="s390x"
         ;;
+    cpu-ppc64le)
+        TARGET=base
+        DOCKER_TAG=ppc64le
+        GPU_IMAGE=redhat/ubi9
+        DOCKER_GPU_BUILD_ARG=""
+        MANY_LINUX_VERSION="ppc64le"
+        ;;
     cuda)
         TARGET=cuda_final
         DOCKER_TAG=cuda${GPU_ARCH_VERSION}
@@ -121,7 +128,7 @@ fi
 (
     set -x
 
-    if [ "$(uname -m)" != "s390x" ]; then
+    if [ "$(uname -m)" != "s390x" && "$(uname -m)" != "ppc64le" ]; then
         # TODO: Remove LimitNOFILE=1048576 patch once https://github.com/pytorch/test-infra/issues/5712
         # is resolved. This patch is required in order to fix timing out of Docker build on Amazon Linux 2023.
         sudo sed -i s/LimitNOFILE=infinity/LimitNOFILE=1048576/ /usr/lib/systemd/system/docker.service

.github/scripts/ppc64le-ci/README.md

@@ -3,16 +3,30 @@
 ## Install prerequisites.
 
 ```
-Install Docker 
+$ sudo dnf install podman podman-docker jq
 ```
-## Clone pytorch repository
-
 ## Add services.
 
 ```
 $ sudo cp self-hosted-builder/*.service /etc/systemd/system/
 $ sudo systemctl daemon-reload
 ```
+
+## Rebuild the image
+
+First build ppc64le builder image `docker.io/pytorch/ubippc64le-builder`,
+using following commands:
+
+```
+$ cd ~
+$ git clone https://github.com/pytorch/pytorch
+$ cd pytorch
+$ git submodule update --init --recursive
+$ GPU_ARCH_TYPE=cpu-ppc64le "$(pwd)/.ci/docker/manywheel/build.sh" ubippc64le-builder
+$ docker image tag localhost/pytorch/ubippc64le-builder docker.io/pytorch/ubippc64le-builder:cpu-ppc64le
+$ docker image save -o ~/ubi-ppc64le.tar docker.io/pytorch/ubippc64le-builder:cpu-ppc64le
+```
+
 Next step is to build `actions-runner` image using:
 
 ```
@@ -36,8 +50,7 @@ $ sudo /bin/cp <github_app_private_key_file> /etc/actions-runner/<name>/key_priv
 $ sudo echo <github_app_id> | sudo tee /etc/actions-runner/<name>/appid.env
 $ sudo echo <github_app_install_id> | sudo tee /etc/actions-runner/<name>/installid.env
 $ sudo echo NAME=<worker_name> | sudo tee    /etc/actions-runner/<name>/env
-$ sudo echo OWNER=<github_owner>   | sudo tee -a /etc/actions-runner/<name>/env
-$ sudo echo REPO=pytorch   | sudo tee -a /etc/actions-runner/<name>/env
+$ sudo echo ORG=<github_org>   | sudo tee -a /etc/actions-runner/<name>/env
 $ cd self-hosted-builder
 $ sudo /bin/cp helpers/*.sh /usr/local/bin/
 $ sudo chmod 755 /usr/local/bin/app_token.sh /usr/local/bin/gh_token_generator.sh

.github/scripts/ppc64le-ci/self-hosted-builder/actions-runner.Dockerfile

@@ -32,10 +32,8 @@ RUN update-alternatives --set iptables /usr/sbin/iptables-legacy && \
     update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
 
 
-# Add Docker GPG key and repository
-RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \
-    echo "deb [arch=ppc64el signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" > /etc/apt/sources.list.d/docker.list && \
-    apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io && \
+# Install Podman and podman-docker (Docker compatibility)
+RUN apt-get update && apt-get install -y podman podman-docker && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # Install dotnet SDK and other dependencies
@@ -56,10 +54,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 RUN useradd -c "Action Runner" -m runner && \
     usermod -L runner && \
     echo "runner ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/runner && \
-    groupadd docker || true && \
-    usermod -aG docker runner && \
-    (test -S /var/run/docker.sock && chmod 660 /var/run/docker.sock && chgrp docker /var/run/docker.sock || true)
+    groupadd podman || true && \
+    usermod -aG podman runner
 
+# Configure Podman cgroup manager
+RUN mkdir -p /etc/containers && \
+    echo "[engine]\ncgroup_manager = \"cgroupfs\"" | sudo tee /etc/containers/containers.conf
 
 # Add and configure GitHub Actions runner
 ARG RUNNERREPO="https://github.com/actions/runner"
@@ -96,6 +96,8 @@ USER runner
 # Set working directory
 WORKDIR /opt/runner
 
+COPY --chown=runner:runner pytorch-ubi-ppc64le.tar /opt/runner/pytorch-ubi-ppc64le.tar
+
 # Define entry point and command
 ENTRYPOINT ["/usr/bin/entrypoint"]
 CMD ["/usr/bin/actions-runner"]

.github/scripts/ppc64le-ci/self-hosted-builder/fs/usr/bin/actions-runner

@@ -2,38 +2,15 @@
 
 set -e -u
 
-trap cleanup EXIT
+# first import docker image
+if [ -f ./pytorch-ubi-ppc64le.tar ] ; then
+        docker image load --input pytorch-ubi-ppc64le.tar
+        docker image tag docker.io/pytorch/ubippc64le-builder:cpu-ppc64le docker.io/pytorch/ubippc64le-builder:cpu-ppc64le-main
+        rm -f ubi-ppc64le.tar
+fi
 
 token_file=registration-token.json
 
-# Function to clean up and unregister the runner
-cleanup() {
-    echo "Cleaning up temporary files..."
-    [ -f "$token_file" ] && rm -f "$token_file"
-    [ -f "runner-id.json" ] && rm -f "runner-id.json"
-
-    echo "Unregistering the runner from GitHub..."
-    ACCESS_TOKEN="$(cat /run/runner_secret)"
-    runner_id=$(curl -s \
-        -H "Accept: application/vnd.github.v3+json" \
-        -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-        "https://api.github.com/repos/${OWNER}/${REPO}/actions/runners" | \
-        jq --raw-output '.runners[] | select(.name=="'"${NAME}"'") | .id')
-
-    if [ -n "$runner_id" ]; then
-        curl -s \
-            -X DELETE \
-            -H "Accept: application/vnd.github.v3+json" \
-            -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-            "https://api.github.com/repos/${OWNER}/${REPO}/actions/runners/$runner_id"
-        echo "Runner unregistered successfully."
-    else
-        echo "Warning: Runner ID for ${NAME} not found. It may already be removed."
-    fi
-
-    unset ACCESS_TOKEN runner_id
-}
-
 # Fetch GitHub access token
 if [ ! -f /run/runner_secret ]; then
     echo "Error: Access token file not found at /run/runner_secret."
@@ -48,19 +25,22 @@ curl \
         -X POST \
         -H "Accept: application/vnd.github.v3+json" \
         -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-        "https://api.github.com/repos/${OWNER}/${REPO}/actions/runners/registration-token" \
+        "https://api.github.com/orgs/${ORG}/actions/runners/registration-token" \
         -o "$token_file"
 
 unset ACCESS_TOKEN
 
+sudo umount /run/runner_secret
+sudo rm -f /run/runner_secret
+
 # register runner as ephemeral runner
 # it does one job, stops and unregisters
 registration_token=$(jq --raw-output .token "$token_file")
 
 ./config.sh \
         --unattended \
         --ephemeral \
-        --url "https://github.com/${OWNER}/${REPO}" \
+        --url "https://github.com/${ORG}" \
         --token "${registration_token}" \
         --name "${NAME}" \
         --no-default-labels \

.github/workflows/_linux-build.yml

@@ -109,7 +109,7 @@ jobs:
     steps:
       - name: Setup SSH (Click me for login details)
         uses: pytorch/test-infra/.github/actions/setup-ssh@main
-        if: inputs.build-environment != 'linux-s390x-binary-manywheel' && inputs.build-environment != 'linux-ppc64le-binary-manywheel-ubi9'
+        if: inputs.build-environment != 'linux-s390x-binary-manywheel'
         with:
           github-secret: ${{ secrets.GITHUB_TOKEN }}
 
@@ -119,17 +119,16 @@ jobs:
       # checkout. In other cases you should prefer a local checkout.
       - name: Checkout PyTorch
         uses: pytorch/pytorch/.github/actions/checkout-pytorch@main
-        if: inputs.build-environment != 'linux-ppc64le-binary-manywheel-ubi9'
         with:
           no-sudo: true
 
       - name: Setup Linux
         uses: ./.github/actions/setup-linux
-        if: inputs.build-environment != 'linux-s390x-binary-manywheel' && inputs.build-environment != 'linux-ppc64le-binary-manywheel-ubi9'
+        if: inputs.build-environment != 'linux-s390x-binary-manywheel'
 
       - name: configure aws credentials
         uses: aws-actions/configure-aws-credentials@v3
-        if: ${{ inputs.aws-role-to-assume != '' && inputs.build-environment != 'linux-s390x-binary-manywheel' && inputs.build-environment != 'linux-ppc64le-binary-manywheel-ubi9' }}
+        if: ${{ inputs.aws-role-to-assume != '' && inputs.build-environment != 'linux-s390x-binary-manywheel' }}
         with:
           role-to-assume: ${{ inputs.aws-role-to-assume }}
           role-session-name: gha-linux-build
@@ -138,13 +137,13 @@ jobs:
       - name: Calculate docker image
         id: calculate-docker-image
         uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
-        if: inputs.build-environment != 'linux-s390x-binary-manywheel' && inputs.build-environment != 'linux-ppc64le-binary-manywheel-ubi9'
+        if: inputs.build-environment != 'linux-s390x-binary-manywheel'
         with:
           docker-image-name: ${{ inputs.docker-image-name }}
 
       - name: Use following to pull public copy of the image
         id: print-ghcr-mirror
-        if: inputs.build-environment != 'linux-s390x-binary-manywheel' && inputs.build-environment != 'linux-ppc64le-binary-manywheel-ubi9'
+        if: inputs.build-environment != 'linux-s390x-binary-manywheel'
         env:
           ECR_DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }}
         shell: bash
@@ -154,26 +153,24 @@ jobs:
 
       - name: Pull docker image
         uses: pytorch/test-infra/.github/actions/pull-docker-image@main
-        if: inputs.build-environment != 'linux-s390x-binary-manywheel' && inputs.build-environment != 'linux-ppc64le-binary-manywheel-ubi9'
+        if: inputs.build-environment != 'linux-s390x-binary-manywheel'
         with:
           docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }}
 
       - name: Parse ref
         id: parse-ref
-        if: inputs.build-environment != 'linux-ppc64le-binary-manywheel-ubi9'
         run: .github/scripts/parse_ref.py
 
       - name: Get workflow job id
         id: get-job-id
         uses: ./.github/actions/get-workflow-job-id
-        if: always() && inputs.build-environment != 'linux-ppc64le-binary-manywheel-ubi9'
+        if: always()
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
       # Apply the filter logic to the build step too if the test-config label is already there
       - name: Select all requested test configurations (if the test matrix is available)
         id: filter
-        if: inputs.build-environment != 'linux-ppc64le-binary-manywheel-ubi9'
         uses: ./.github/actions/filter-test-configs
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -184,14 +181,14 @@ jobs:
       - name: Download pytest cache
         uses: ./.github/actions/pytest-cache-download
         continue-on-error: true
-        if: inputs.build-environment != 'linux-s390x-binary-manywheel' && inputs.build-environment != 'linux-ppc64le-binary-manywheel-ubi9'
+        if: inputs.build-environment != 'linux-s390x-binary-manywheel'
         with:
           cache_dir: .pytest_cache
           job_identifier: ${{ github.workflow }}_${{ inputs.build-environment }}
           s3_bucket: ${{ inputs.s3-bucket }}
 
       - name: Build
-        if: (steps.filter.outputs.is-test-matrix-empty == 'False' || inputs.test-matrix == '') && (inputs.build-environment != 'linux-ppc64le-binary-manywheel-ubi9')
+        if: steps.filter.outputs.is-test-matrix-empty == 'False' || inputs.test-matrix == ''
         id: build
         env:
           BUILD_ENVIRONMENT: ${{ inputs.build-environment }}
@@ -278,24 +275,14 @@ jobs:
           END_TIME=$(date +%s)
           echo "build_time=$((END_TIME - START_TIME))" >> "$GITHUB_OUTPUT"
 
-      - name: Execute Build and Tests inside ppc64le Docker Container
-        if: inputs.build-environment == 'linux-ppc64le-binary-manywheel-ubi9'
-        run: |
-            CONTAINER_NAME="temp_builder_${RUN_ID}"
-            docker run -d --name "$CONTAINER_NAME" pytorch-ppc64le:ubi9.3 /ppc64le-build.sh
-            docker wait "$CONTAINER_NAME"
-            docker logs "$CONTAINER_NAME"
-            docker cp "$CONTAINER_NAME":/workspace/pytorch/dist/. dist/
-            docker rm "$CONTAINER_NAME"
-
       - name: Archive artifacts into zip
         if: inputs.build-generates-artifacts && steps.build.outcome != 'skipped'
         run: |
           zip -1 -r artifacts.zip dist/ build/custom_test_artifacts build/lib build/bin .additional_ci_files
 
       - name: Store PyTorch Build Artifacts on S3
         uses: seemethere/upload-artifact-s3@v5
-        if: inputs.build-generates-artifacts && steps.build.outcome != 'skipped' && !inputs.use_split_build && inputs.build-environment != 'linux-s390x-binary-manywheel' && inputs.build-environment != 'linux-ppc64le-binary-manywheel-ubi9'
+        if: inputs.build-generates-artifacts && steps.build.outcome != 'skipped' && !inputs.use_split_build && inputs.build-environment != 'linux-s390x-binary-manywheel'
         with:
           name: ${{ inputs.build-environment }}
           retention-days: 14
@@ -305,7 +292,7 @@ jobs:
 
       - name: Store PyTorch Build Artifacts on S3 for split build
         uses: seemethere/upload-artifact-s3@v5
-        if: inputs.build-generates-artifacts && steps.build.outcome != 'skipped' && inputs.use_split_build && inputs.build-environment != 'linux-s390x-binary-manywheel' && inputs.build-environment != 'linux-ppc64le-binary-manywheel-ubi9'
+        if: inputs.build-generates-artifacts && steps.build.outcome != 'skipped' && inputs.use_split_build && inputs.build-environment != 'linux-s390x-binary-manywheel'
         with:
           name: ${{ inputs.build-environment }}-experimental-split-build
           retention-days: 14
@@ -331,35 +318,16 @@ jobs:
           if-no-files-found: error
           path: artifacts.zip
 
-      - name: Archive ppc64le artifacts into zip
-        if: inputs.build-environment == 'linux-ppc64le-binary-manywheel-ubi9'
-        run: |
-          zip -1 -r artifacts.zip dist/
-      
-  
-      - name: Store PyTorch Build Artifacts for ppc64le
-        uses: actions/upload-artifact@v4
-        if: inputs.build-environment == 'linux-ppc64le-binary-manywheel-ubi9'
-        with:
-          name: ${{ inputs.build-environment }}-ubi9
-          retention-days: 14
-          if-no-files-found: error
-          path: artifacts.zip
-
-      - name: Cleanup dangling Docker images for ppc64le
-        if: always() && inputs.build-environment == 'linux-ppc64le-binary-manywheel-ubi9'
-        run: docker image prune -f
-
       - name: Upload sccache stats
-        if: steps.build.outcome != 'skipped' && inputs.build-environment != 'linux-s390x-binary-manywheel' && inputs.build-environment != 'linux-ppc64le-binary-manywheel-ubi9'
+        if: steps.build.outcome != 'skipped' && inputs.build-environment != 'linux-s390x-binary-manywheel'
         uses: ./.github/actions/upload-sccache-stats
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           build-time: ${{ steps.build.outputs.build_time }}
 
       - name: Teardown Linux
         uses: pytorch/test-infra/.github/actions/teardown-linux@main
-        if: always() && (inputs.build-environment != 'linux-s390x-binary-manywheel' && inputs.build-environment != 'linux-ppc64le-binary-manywheel-ubi9')
+        if: always() && inputs.build-environment != 'linux-s390x-binary-manywheel'
 
       - name: Cleanup docker
         if: always() && inputs.build-environment == 'linux-s390x-binary-manywheel'