Fix logic in validate aarch64 and xpu linux binaries (#6072)

1. Aarch64: Skip docker builds same as here:
https://github.com/pytorch/test-infra/blob/main/.github/workflows/validate-linux-binaries.yml#L139
2. Aarch64: Fix script path
3. Aarch64: Add tests to nightly validation trigger
4. XPU: conda install path fix

Author: atalman

.github/workflows/validate-aarch64-linux-binaries.yml

@@ -113,6 +113,7 @@ jobs:
       job-name: ${{ matrix.build_name }}
       docker-image: ${{ matrix.container_image }}
       binary-matrix: ${{ toJSON(matrix) }}
+      docker-build-dir: "skip-docker-build"
       no-sudo: true
       script: |
         set -ex
@@ -154,4 +155,4 @@ jobs:
         unset LD_LIBRARY_PATH
 
         # Standard case: Validate binaries
-        source ../../.github/scripts/validate_binaries.sh
+        source ../../test-infra/.github/scripts/validate_binaries.sh

.github/workflows/validate-linux-binaries.yml

@@ -155,10 +155,6 @@ jobs:
 
         export USE_FORCE_REINSTALL="true"
         export TARGET_OS="linux"
-        # Due to xpu doesn't use pytorch/conda-builder image, need to install conda firstly
-        if [[ ${{ matrix.gpu_arch_type }} == 'xpu' ]]; then
-          source /.ci/docker/common/install_conda.sh
-        fi
         eval "$(conda shell.bash hook)"
         printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json
 

.github/workflows/validate-nightly-binaries.yml

@@ -15,13 +15,15 @@ on:
       - .github/workflows/validate-linux-binaries.yml
       - .github/workflows/validate-windows-binaries.yml
       - .github/workflows/validate-macos-arm64-binaries.yml
+      - .github/workflows/validate-aarch64-linux-binaries.yml
       - test/smoke_test/*
   pull_request:
     paths:
       - .github/workflows/validate-nightly-binaries.yml
       - .github/workflows/validate-linux-binaries.yml
       - .github/workflows/validate-windows-binaries.yml
       - .github/workflows/validate-macos-arm64-binaries.yml
+      - .github/workflows/validate-aarch64-linux-binaries.yml
       - .github/scripts/validate_binaries.sh
       - test/smoke_test/*