From a80efd7716e24a4d98cd086ecc5558c75f279150 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Tue, 26 Nov 2024 12:30:15 -0800
Subject: [PATCH 01/12] Use a template for Android test spec

---
 .github/workflows/android-perf.yml            | 39 +++++++----
 .../workflows/upload-android-test-specs.yml   | 66 +------------------
 ... android-llm-device-farm-test-spec.yml.j2} |  8 +++
 3 files changed, 37 insertions(+), 76 deletions(-)
 rename extension/benchmark/android/benchmark/{android-llm-device-farm-test-spec.yml => android-llm-device-farm-test-spec.yml.j2} (95%)

diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml
index 93ec4fe4e70..ccf1bd05e51 100644
--- a/.github/workflows/android-perf.yml
+++ b/.github/workflows/android-perf.yml
@@ -30,10 +30,6 @@ on:
         description: The list of configs used the benchmark
         required: false
         type: string
-      test_spec:
-        description: The test spec to drive the test on AWS devices
-        required: false
-        type: string
   workflow_call:
     inputs:
       models:
@@ -60,10 +56,6 @@ on:
         description: The list of configs used the benchmark
         required: false
         type: string
-      test_spec:
-        description: The test spec to drive the test on AWS devices
-        required: false
-        type: string
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
@@ -125,6 +117,32 @@ jobs:
           echo "devices=$(echo "$MAPPED_ARNS_JSON" | jq -c .)" >> $GITHUB_OUTPUT
           echo "delegates=$(echo $DELEGATES | jq -Rc 'split(",")')" >> $GITHUB_OUTPUT
 
+      - name: Prepare the spec
+        shell: bash
+        working-directory: extension/benchmark/android/benchmark
+        run: |
+          set -eux
+
+          # The model will be exported in the next step to this S3 path
+          MODEL_PATH="s3://gha-artifacts/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.delegate }}/model.zip"
+          # We could write a script to properly use jinja here, but there is only one variable,
+          # so let's just sed it
+          sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' android-llm-device-farm-test-spec.yml.j2
+          cp android-llm-device-farm-test-spec.yml.j2 android-llm-device-farm-test-spec.yml
+
+          # Just print the test spec for debugging
+          cat android-llm-device-farm-test-spec.yml
+
+      - name: Upload the spec
+        uses: seemethere/upload-artifact-s3@v5
+        with:
+          s3-bucket: gha-artifacts
+          s3-prefix: |
+            ${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.delegate }}
+          retention-days: 1
+          if-no-files-found: error
+          path: extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml
+
   export-models:
     name: export-models
     uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
@@ -231,10 +249,7 @@ jobs:
       device-pool-arn: ${{ matrix.device }}
       android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/minibench/app-debug.apk
       android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/minibench/app-debug-androidTest.apk
-      # NB: Need to set the default spec here so that it works for periodic too
-      test-spec: ${{ inputs.test_spec || 'https://ossci-android.s3.amazonaws.com/executorch/android-llm-device-farm-test-spec.yml' }}
-      # Uploaded to S3 from the previous job
-      extra-data: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.delegate }}/model.zip
+      test-spec: ${{ https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.delegate }}/android-llm-device-farm-test-spec.yml }}
 
   upload-benchmark-results:
     needs:
diff --git a/.github/workflows/upload-android-test-specs.yml b/.github/workflows/upload-android-test-specs.yml
index 4c0b37ab118..039cae8e4bf 100644
--- a/.github/workflows/upload-android-test-specs.yml
+++ b/.github/workflows/upload-android-test-specs.yml
@@ -4,13 +4,13 @@ on:
   pull_request:
     paths:
       - .github/workflows/upload-android-test-specs.yml
-      - extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml
+      - extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2
   push:
     branches:
       - main
     paths:
       - .github/workflows/upload-android-test-specs.yml
-      - extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml
+      - extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2
 
 concurrency:
   # NB: This concurency group needs to be different than the one used in android-perf, otherwise
@@ -19,21 +19,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  upload-android-test-spec-for-validation:
-    runs-on: linux.2xlarge
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Upload the spec as a GitHub artifact for validation
-        uses: seemethere/upload-artifact-s3@v5
-        with:
-          s3-bucket: gha-artifacts
-          s3-prefix: |
-            ${{ github.repository }}/${{ github.run_id }}/artifacts
-          retention-days: 1
-          if-no-files-found: error
-          path: extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml
-
   validate-android-test-spec:
     needs: upload-android-test-spec-for-validation
     uses: ./.github/workflows/android-perf.yml
@@ -45,50 +30,3 @@ jobs:
       models: stories110M
       devices: samsung_galaxy_s22
       delegates: xnnpack
-      test_spec: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/android-llm-device-farm-test-spec.yml
-
-  upload-android-test-spec:
-    needs: validate-android-test-spec
-    runs-on: ubuntu-22.04
-    timeout-minutes: 15
-    permissions:
-      id-token: write
-      contents: read
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
-          cache: pip
-
-      - name: configure aws credentials
-        uses: aws-actions/configure-aws-credentials@v1.7.0
-        with:
-          role-to-assume: arn:aws:iam::308535385114:role/gha_executorch_upload-frameworks-android
-          aws-region: us-east-1
-
-      - name: Only push to S3 when running the workflow manually from main branch
-        if: ${{ github.ref == 'refs/heads/main' }}
-        shell: bash
-        run: |
-          set -eux
-          echo "UPLOAD_ON_MAIN=1" >> "${GITHUB_ENV}"
-
-      - name: Upload the spec to S3 ossci-android bucket
-        shell: bash
-        working-directory: extension/benchmark/android/benchmark/
-        env:
-          SPEC_FILE: android-llm-device-farm-test-spec.yml
-        run: |
-          set -eux
-
-          pip install awscli==1.32.18
-
-          AWS_CMD="aws s3 cp --dryrun"
-          if [[ "${UPLOAD_ON_MAIN:-0}" == "1" ]]; then
-            AWS_CMD="aws s3 cp"
-          fi
-
-          shasum -a 256 "${SPEC_FILE}"
-          ${AWS_CMD} "${SPEC_FILE}" s3://ossci-android/executorch/ --acl public-read
diff --git a/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml b/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2
similarity index 95%
rename from extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml
rename to extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2
index ffb528767a5..3ede2bbf9ee 100644
--- a/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml
+++ b/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2
@@ -8,6 +8,14 @@ phases:
 
   pre_test:
     commands:
+      # Download the model from S3
+      - aws s3 cp {{ model_path }} model.zip
+      - unzip model.zip && ls -la
+
+      # Copy the extracted model to sdcard
+      - adb -s $DEVICEFARM_DEVICE_UDID push *.bin /sdcard
+      - adb -s $DEVICEFARM_DEVICE_UDID push *.pte /sdcard
+
       # Prepare the model and the tokenizer
       - adb -s $DEVICEFARM_DEVICE_UDID shell "ls -la /sdcard/"
       - adb -s $DEVICEFARM_DEVICE_UDID shell "mkdir -p /data/local/tmp/minibench/"

From dc205e1647f1ff940f0aec28b9f7c337aea7a943 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Tue, 26 Nov 2024 12:51:34 -0800
Subject: [PATCH 02/12] Minor workflow tweak

---
 .github/workflows/upload-android-test-specs.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/upload-android-test-specs.yml b/.github/workflows/upload-android-test-specs.yml
index 039cae8e4bf..fd4b03ea21d 100644
--- a/.github/workflows/upload-android-test-specs.yml
+++ b/.github/workflows/upload-android-test-specs.yml
@@ -20,7 +20,6 @@ concurrency:
 
 jobs:
   validate-android-test-spec:
-    needs: upload-android-test-spec-for-validation
     uses: ./.github/workflows/android-perf.yml
     permissions:
       id-token: write

From 90c43cbae547f7831d8a3df70774275f61d97ab1 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Tue, 26 Nov 2024 12:56:51 -0800
Subject: [PATCH 03/12] Fix wrong workflow format

---
 .github/workflows/android-perf.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml
index ccf1bd05e51..4192ef21240 100644
--- a/.github/workflows/android-perf.yml
+++ b/.github/workflows/android-perf.yml
@@ -249,7 +249,7 @@ jobs:
       device-pool-arn: ${{ matrix.device }}
       android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/minibench/app-debug.apk
       android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/minibench/app-debug-androidTest.apk
-      test-spec: ${{ https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.delegate }}/android-llm-device-farm-test-spec.yml }}
+      test-spec: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.delegate }}/android-llm-device-farm-test-spec.yml
 
   upload-benchmark-results:
     needs:

From 304452f437a820a255350abdf8de5d9c930600fa Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Tue, 26 Nov 2024 13:01:07 -0800
Subject: [PATCH 04/12] Add checkout step to get the spec

---
 .github/workflows/android-perf.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml
index 4192ef21240..e60641a641d 100644
--- a/.github/workflows/android-perf.yml
+++ b/.github/workflows/android-perf.yml
@@ -69,6 +69,8 @@ jobs:
       devices: ${{ steps.set-parameters.outputs.devices }}
       delegates: ${{ steps.set-parameters.outputs.delegates }}
     steps:
+      - uses: actions/checkout@v3
+
       - name: Set parameters
         id: set-parameters
         shell: bash

From c4104bc175cdfb02b168057f27bc401831534376 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Tue, 26 Nov 2024 13:12:53 -0800
Subject: [PATCH 05/12] Use linux_job_v2.yml

---
 .github/workflows/android-perf.yml | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml
index e60641a641d..b3a92ac3ade 100644
--- a/.github/workflows/android-perf.yml
+++ b/.github/workflows/android-perf.yml
@@ -69,8 +69,6 @@ jobs:
       devices: ${{ steps.set-parameters.outputs.devices }}
       delegates: ${{ steps.set-parameters.outputs.delegates }}
     steps:
-      - uses: actions/checkout@v3
-
       - name: Set parameters
         id: set-parameters
         shell: bash
@@ -119,6 +117,17 @@ jobs:
           echo "devices=$(echo "$MAPPED_ARNS_JSON" | jq -c .)" >> $GITHUB_OUTPUT
           echo "delegates=$(echo $DELEGATES | jq -Rc 'split(",")')" >> $GITHUB_OUTPUT
 
+  prepare-test-specs:
+    runs-on: linux.2xlarge
+    needs: set-parameters
+    strategy:
+      matrix:
+          model: ${{ fromJson(needs.set-parameters.outputs.models) }}
+          delegate: ${{ fromJson(needs.set-parameters.outputs.delegates) }}
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v3
+
       - name: Prepare the spec
         shell: bash
         working-directory: extension/benchmark/android/benchmark
@@ -147,7 +156,7 @@ jobs:
 
   export-models:
     name: export-models
-    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
+    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
     needs: set-parameters
     strategy:
       matrix:
@@ -198,7 +207,7 @@ jobs:
 
   build-benchmark-app:
     name: build-benchmark-app
-    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
+    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
     needs: set-parameters
     with:
       runner: linux.2xlarge
@@ -232,6 +241,7 @@ jobs:
     uses: pytorch/test-infra/.github/workflows/mobile_job.yml@main
     needs:
       - set-parameters
+      - prepare-test-specs
       - build-benchmark-app
       - export-models
     strategy:

From 157623baad616b396dc4244098358e8b32e2156f Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Tue, 26 Nov 2024 13:35:37 -0800
Subject: [PATCH 06/12] Fix test_llama invocation after
 https://github.com/pytorch/executorch/pull/6870

---
 .github/workflows/android-perf.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml
index b3a92ac3ade..38b4cace6b4 100644
--- a/.github/workflows/android-perf.yml
+++ b/.github/workflows/android-perf.yml
@@ -199,7 +199,12 @@ jobs:
                 echo "Unsupported delegate ${{ matrix.delegate }}"
                 exit 1
             fi
-            PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama.sh "${{ matrix.model }}" "${BUILD_MODE}" "${DTYPE}" "${DELEGATE_CONFIG}" "${ARTIFACTS_DIR_NAME}"
+            PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama.sh \
+              -model "${{ matrix.model }}" \
+              -build_tool "${BUILD_MODE}" \
+              -dtype "${DTYPE}" \
+              -mode "${DELEGATE_CONFIG}" \
+              -upload "${ARTIFACTS_DIR_NAME}"
         else
             PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh "${{ matrix.model }}" "${BUILD_MODE}" "${{ matrix.delegate }}" "${ARTIFACTS_DIR_NAME}"
         fi

From 79d532f501e21e07bf8ad1135cda416e892b4384 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Tue, 26 Nov 2024 15:48:49 -0800
Subject: [PATCH 07/12] Use curl instead

---
 .github/workflows/android-perf.yml                            | 2 +-
 .../benchmark/android-llm-device-farm-test-spec.yml.j2        | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml
index 38b4cace6b4..5aa12269ea3 100644
--- a/.github/workflows/android-perf.yml
+++ b/.github/workflows/android-perf.yml
@@ -135,7 +135,7 @@ jobs:
           set -eux
 
           # The model will be exported in the next step to this S3 path
-          MODEL_PATH="s3://gha-artifacts/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.delegate }}/model.zip"
+          MODEL_PATH="https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.delegate }}/model.zip"
           # We could write a script to properly use jinja here, but there is only one variable,
           # so let's just sed it
           sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' android-llm-device-farm-test-spec.yml.j2
diff --git a/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2 b/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2
index 3ede2bbf9ee..9b0f83a96f5 100644
--- a/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2
+++ b/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2
@@ -9,10 +9,10 @@ phases:
   pre_test:
     commands:
       # Download the model from S3
-      - aws s3 cp {{ model_path }} model.zip
+      - curl -s --fail '{{ model_path }}' -o model.zip
       - unzip model.zip && ls -la
 
-      # Copy the extracted model to sdcard
+      # Copy the model to sdcard
       - adb -s $DEVICEFARM_DEVICE_UDID push *.bin /sdcard
       - adb -s $DEVICEFARM_DEVICE_UDID push *.pte /sdcard
 

From e00e2e50bcd194f325ca12944f4dcb3921a74cca Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Tue, 26 Nov 2024 16:34:05 -0800
Subject: [PATCH 08/12] Format

---
 .github/workflows/android-perf.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml
index 5aa12269ea3..1a5c821f752 100644
--- a/.github/workflows/android-perf.yml
+++ b/.github/workflows/android-perf.yml
@@ -206,7 +206,11 @@ jobs:
               -mode "${DELEGATE_CONFIG}" \
               -upload "${ARTIFACTS_DIR_NAME}"
         else
-            PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh "${{ matrix.model }}" "${BUILD_MODE}" "${{ matrix.delegate }}" "${ARTIFACTS_DIR_NAME}"
+            PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh \
+              "${{ matrix.model }}" \
+              "${BUILD_MODE}" \
+              "${{ matrix.delegate }}" \
+              "${ARTIFACTS_DIR_NAME}"
         fi
         echo "::endgroup::"
 

From 5b09cd86d7cb9ade9b5b77dfde79c3900f52df18 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Wed, 27 Nov 2024 01:05:57 -0800
Subject: [PATCH 09/12] Copy file silently

---
 .../benchmark/android-llm-device-farm-test-spec.yml.j2        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2 b/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2
index 9b0f83a96f5..bcff21c08d1 100644
--- a/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2
+++ b/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2
@@ -13,8 +13,8 @@ phases:
       - unzip model.zip && ls -la
 
       # Copy the model to sdcard
-      - adb -s $DEVICEFARM_DEVICE_UDID push *.bin /sdcard
-      - adb -s $DEVICEFARM_DEVICE_UDID push *.pte /sdcard
+      - adb -s $DEVICEFARM_DEVICE_UDID push -q *.bin /sdcard
+      - adb -s $DEVICEFARM_DEVICE_UDID push -q *.pte /sdcard
 
       # Prepare the model and the tokenizer
       - adb -s $DEVICEFARM_DEVICE_UDID shell "ls -la /sdcard/"

From 63a47b30e42fd25a4c9705832cdaaac76c57dc5c Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Wed, 27 Nov 2024 07:57:23 -0800
Subject: [PATCH 10/12] Route adb push output to /dev/null

---
 .../benchmark/android-llm-device-farm-test-spec.yml.j2     | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2 b/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2
index bcff21c08d1..d3d6ea04f19 100644
--- a/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2
+++ b/extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2
@@ -12,9 +12,10 @@ phases:
       - curl -s --fail '{{ model_path }}' -o model.zip
       - unzip model.zip && ls -la
 
-      # Copy the model to sdcard
-      - adb -s $DEVICEFARM_DEVICE_UDID push -q *.bin /sdcard
-      - adb -s $DEVICEFARM_DEVICE_UDID push -q *.pte /sdcard
+      # Copy the model to sdcard. This prints too much progress info when the files
+      # are large, so it's better to just silent them
+      - adb -s $DEVICEFARM_DEVICE_UDID push *.bin /sdcard > /dev/null && echo OK
+      - adb -s $DEVICEFARM_DEVICE_UDID push *.pte /sdcard > /dev/null && echo OK
 
       # Prepare the model and the tokenizer
       - adb -s $DEVICEFARM_DEVICE_UDID shell "ls -la /sdcard/"

From a32ae242c88d5789900e53a413c32553eff4d2b6 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Mon, 2 Dec 2024 20:52:11 -0800
Subject: [PATCH 11/12] No need to use linux_job_v2 for now

---
 .github/workflows/android-perf.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml
index 1a5c821f752..8aab035f086 100644
--- a/.github/workflows/android-perf.yml
+++ b/.github/workflows/android-perf.yml
@@ -156,7 +156,7 @@ jobs:
 
   export-models:
     name: export-models
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
     needs: set-parameters
     strategy:
       matrix:
@@ -216,7 +216,7 @@ jobs:
 
   build-benchmark-app:
     name: build-benchmark-app
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
     needs: set-parameters
     with:
       runner: linux.2xlarge

From ac48848e8e979af01b44e053d3542cd1ac16bb24 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Tue, 3 Dec 2024 15:33:24 -0800
Subject: [PATCH 12/12] Clean up upload-android-test-specs workflow

---
 .github/workflows/android-perf.yml            | 16 ++++++++--
 .../workflows/upload-android-test-specs.yml   | 31 -------------------
 2 files changed, 13 insertions(+), 34 deletions(-)
 delete mode 100644 .github/workflows/upload-android-test-specs.yml

diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml
index 8aab035f086..66e031bea72 100644
--- a/.github/workflows/android-perf.yml
+++ b/.github/workflows/android-perf.yml
@@ -3,6 +3,16 @@ name: android-perf
 on:
   schedule:
     - cron: 0 0 * * *
+  pull_request:
+    paths:
+      - .github/workflows/android-perf.yml
+      - extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2
+  push:
+    branches:
+      - main
+    paths:
+      - .github/workflows/android-perf.yml
+      - extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2
   # Note: GitHub has an upper limit of 10 inputs
   workflow_dispatch:
     inputs:
@@ -76,9 +86,9 @@ jobs:
           # Separate default values from the workflow dispatch. To ensure defaults are accessible
           # during scheduled runs and to provide flexibility for different defaults between
           # on-demand and periodic benchmarking.
-          CRON_DEFAULT_MODELS: "stories110M,dl3,mv3,mv2,ic4,ic3,vit"
-          CRON_DEFAULT_DEVICES: "samsung_galaxy_s22"
-          CRON_DEFAULT_DELEGATES: "xnnpack,qnn"
+          CRON_DEFAULT_MODELS: ${{ github.event_name == 'schedule' && 'stories110M,dl3,mv3,mv2,ic4,ic3,vit' || 'stories110M' }}
+          CRON_DEFAULT_DEVICES: samsung_galaxy_s22
+          CRON_DEFAULT_DELEGATES: ${{ github.event_name == 'schedule' && 'xnnpack,qnn' || 'xnnpack' }}
         run: |
           set -ex
           MODELS="${{ inputs.models }}"
diff --git a/.github/workflows/upload-android-test-specs.yml b/.github/workflows/upload-android-test-specs.yml
deleted file mode 100644
index fd4b03ea21d..00000000000
--- a/.github/workflows/upload-android-test-specs.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-name: Upload AWS Device Farm Android test specs
-
-on:
-  pull_request:
-    paths:
-      - .github/workflows/upload-android-test-specs.yml
-      - extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2
-  push:
-    branches:
-      - main
-    paths:
-      - .github/workflows/upload-android-test-specs.yml
-      - extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2
-
-concurrency:
-  # NB: This concurency group needs to be different than the one used in android-perf, otherwise
-  # GH complains about concurrency deadlock
-  group: android-spec-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
-  cancel-in-progress: true
-
-jobs:
-  validate-android-test-spec:
-    uses: ./.github/workflows/android-perf.yml
-    permissions:
-      id-token: write
-      contents: read
-    with:
-      # Just use a small model here with a minimal amount of configuration to test the spec
-      models: stories110M
-      devices: samsung_galaxy_s22
-      delegates: xnnpack