Trim macos jobs and do as much as possible in aarch64 instances

Author: mergennachin

.github/workflows/pull.yml

@@ -50,6 +50,10 @@ jobs:
         backend: [portable, xnnpack-quantization-delegation]
         build-tool: [cmake, buck2]
         runner: [linux.2xlarge]
+        docker-image: [executorch-ubuntu-22.04-clang12]
+        include:
+          - runner: linux.arm64.2xlarge 
+            docker-image: executorch-ubuntu-22.04-gcc11-aarch64
       fail-fast: false
     with:
       runner: ${{ matrix.runner }}

.github/workflows/trunk.yml

@@ -23,8 +23,8 @@ jobs:
     uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
     strategy:
       matrix:
-        model: [add, add_mul, emformer_join, emformer_transcribe, ic3, ic4, linear, llama2, mobilebert, mv2, mv3, resnet18, resnet50, vit, w2l]
-        backend: [portable, xnnpack-quantization-delegation]
+        model: [emformer_join, ic4, llama2, mobilebert, mv3, resnet50, vit, w2l]
+        backend: [xnnpack-quantization-delegation]
         build-tool: [cmake]
         runner: [macos-m1-stable]
         include:
@@ -34,16 +34,12 @@ jobs:
             backend: portable
           - model: llama3_2_vision_encoder
             backend: portable
-          - model: lstm
-            backend: portable
-          - model: mul
+          - model: mv3
             backend: portable
           - model: phi-4-mini
             backend: portable
           - model: qwen2_5
             backend: portable
-          - model: softmax
-            backend: portable
       fail-fast: false
     with:
       runner: ${{ matrix.runner }}
@@ -74,6 +70,13 @@ jobs:
         model: [linear, add, add_mul, ic3, ic4, mv2, mv3, resnet18, resnet50, vit, w2l, mobilebert, emformer_join, emformer_transcribe]
         backend: [portable, xnnpack-quantization-delegation]
         runner: [linux.arm64.2xlarge]
+        include:
+          - model: lstm
+            backend: portable
+          - model: mul
+            backend: portable
+          - model: softmax
+            backend: portable
       fail-fast: false
     with:
       runner: ${{ matrix.runner }}