diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml
index 58decca0d30..795272688bd 100644
--- a/.github/workflows/pull.yml
+++ b/.github/workflows/pull.yml
@@ -481,11 +481,18 @@ jobs:
       build-tool: buck2
       docker-image: executorch-ubuntu-22.04-clang12
 
-  unittest-arm:
+  unittest-arm-backend-with-no-fvp:
+    name: unittest-arm-backend-with-no-fvp
     uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
     permissions:
       id-token: write
       contents: read
+    strategy:
+      matrix:
+        include:
+          - test_arm_baremetal: test_pytest_ops
+          - test_arm_baremetal: test_pytest_models
+      fail-fast: false
     with:
       runner: linux.2xlarge
       docker-image: executorch-ubuntu-22.04-arm-sdk
@@ -493,25 +500,19 @@ jobs:
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
-        set -eux
-
         # The generic Linux job chooses to use base env, not the one setup by the image
         CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
         conda activate "${CONDA_ENV}"
 
-        BUILD_TOOL="cmake"
-
-        # Setup MacOS dependencies as there is no Docker support on MacOS atm
-        PYTHON_EXECUTABLE=python \
-        CMAKE_ARGS="-DEXECUTORCH_BUILD_PYBIND=ON" \
-        EXECUTORCH_BUILD_ARM_BAREMETAL=ON \
-        .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}"
+        source .ci/scripts/utils.sh
+        install_executorch "--use-pt-pinned-commit"
 
-        # Install Arm dependencies
         .ci/scripts/setup-arm-baremetal-tools.sh
 
-        # Run pytest without simulator
-        backends/arm/test/test_arm_baremetal.sh test_pytest
+        ARM_TEST=${{ matrix.test_arm_baremetal }}
+
+        # Test test_arm_baremetal.sh with test
+        backends/arm/test/test_arm_baremetal.sh "${ARM_TEST}"
 
   test-llama-runner-qnn-linux:
     name: test-llama-runner-qnn-linux
diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml
index cbcb93f82f2..f393e52aa1d 100644
--- a/.github/workflows/trunk.yml
+++ b/.github/workflows/trunk.yml
@@ -176,12 +176,22 @@ jobs:
         # Test selective build
         PYTHON_EXECUTABLE=python bash examples/portable/scripts/test_demo_backend_delegation.sh "${BUILD_TOOL}"
 
-  test-arm-backend-delegation:
-    name: test-arm-backend-delegation
+  test-arm-backend:
+    name: test-arm-backend
     uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
     permissions:
       id-token: write
       contents: read
+    strategy:
+      matrix:
+        include:
+          - test_arm_baremetal: test_pytest_ops_ethosu_fvp
+          - test_arm_baremetal: test_pytest_models_ethosu_fvp
+          - test_arm_baremetal: test_run_ethosu_fvp
+          - test_arm_baremetal: test_models_tosa
+          - test_arm_baremetal: test_models_ethos-u55
+          - test_arm_baremetal: test_models_ethos-u85
+      fail-fast: false
     with:
       runner: linux.2xlarge.memory
       docker-image: executorch-ubuntu-22.04-arm-sdk
@@ -202,34 +212,10 @@ jobs:
         # Hopefully this is high enough for this setup.
         sudo sysctl fs.inotify.max_user_watches=1048576 # 1024 * 1024
 
-        # Test ethos-u delegate examples with run.sh
-        backends/arm/test/test_arm_baremetal.sh test_full_ethosu_fvp
-
-
-  test-arm-reference-delegation:
-    name: test-arm-reference-delegation
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
-    permissions:
-      id-token: write
-      contents: read
-    with:
-      runner: linux.2xlarge.memory
-      docker-image: executorch-ubuntu-22.04-arm-sdk
-      submodules: 'recursive'
-      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-      timeout: 90
-      script: |
-        # The generic Linux job chooses to use base env, not the one setup by the image
-        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
-        conda activate "${CONDA_ENV}"
-
-        source .ci/scripts/utils.sh
-        install_executorch "--use-pt-pinned-commit"
-
-        .ci/scripts/setup-arm-baremetal-tools.sh
+        ARM_TEST=${{ matrix.test_arm_baremetal }}
 
-        # Run arm unit tests using the simulator
-        backends/arm/test/test_arm_baremetal.sh test_pytest_ethosu_fvp
+        # Test test_arm_baremetal.sh with test
+        backends/arm/test/test_arm_baremetal.sh "${ARM_TEST}"
 
   test-arm-cortex-m-size-test:
     name: test-arm-cortex-m-size-test
diff --git a/backends/arm/scripts/run_fvp.sh b/backends/arm/scripts/run_fvp.sh
index 52247e08cab..e4dfa1d7339 100755
--- a/backends/arm/scripts/run_fvp.sh
+++ b/backends/arm/scripts/run_fvp.sh
@@ -65,7 +65,7 @@ hash ${fvp_model} \
 num_macs=$(echo ${target} | cut -d - -f 3)
 
 echo "--------------------------------------------------------------------------------"
-echo "Running ${elf_file} for ${target} run with FVP:${fvp_model} num_macs:${num_macs}"
+echo "Running ${elf_file} for ${target} run with FVP:${fvp_model} num_macs:${num_macs} timeout:${timeout}"
 echo "WARNING: Corstone FVP is not cycle accurate and should NOT be used to determine valid runtime"
 echo "--------------------------------------------------------------------------------"
 
diff --git a/backends/arm/test/test_arm_baremetal.sh b/backends/arm/test/test_arm_baremetal.sh
index f2e1f663501..48cee9acd95 100755
--- a/backends/arm/test/test_arm_baremetal.sh
+++ b/backends/arm/test/test_arm_baremetal.sh
@@ -69,27 +69,71 @@ all() { # Run all tests
     echo "${TEST_SUITE_NAME}: PASS"
 }
 
-test_pytest() { # Test ops and other things
+test_pytest_ops() { # Test ops and other things
     echo "${TEST_SUITE_NAME}: Run pytest"
 
-    ./examples/models/llama3_2_vision/install_requirements.sh
+    # Prepare for pytest
+    backends/arm/scripts/build_executorch.sh
 
     # Run arm baremetal pytest tests without FVP
-    pytest  --verbose --color=yes --numprocesses=auto backends/arm/test/
+    pytest  --verbose --color=yes --numprocesses=auto backends/arm/test/ --ignore=backends/arm/test/models
     echo "${TEST_SUITE_NAME}: PASS"
 }
 
-test_pytest_ethosu_fvp() { # Same as test_pytest but also sometime verify using Corstone FVP
+test_pytest_models() { # Test ops and other things
+    echo "${TEST_SUITE_NAME}: Run pytest"
+
+    examples/models/llama3_2_vision/install_requirements.sh
+
+    # Prepare for pytest
+    backends/arm/scripts/build_executorch.sh
+
+    # Run arm baremetal pytest tests without FVP
+    pytest  --verbose --color=yes --numprocesses=auto backends/arm/test/models
+    echo "${TEST_SUITE_NAME}: PASS"
+}
+
+test_pytest() { # Test ops and other things
+    echo "${TEST_SUITE_NAME}: Run pytest"
+    test_pytest_ops
+    test_pytest_models
+    echo "${TEST_SUITE_NAME}: PASS"
+}
+
+test_pytest_ops_ethosu_fvp() { # Same as test_pytest but also sometime verify using Corstone FVP
     echo "${TEST_SUITE_NAME}: Run pytest with fvp"
 
-    ./examples/models/llama3_2_vision/install_requirements.sh
+    # Prepare Corstone-3x0 FVP for pytest
+    backends/arm/scripts/build_executorch.sh
+    backends/arm/scripts/build_portable_kernels.sh
+    # Build semihosting version of the runner used by pytest testing when using --arm_run_corstoneFVP
+    backends/arm/test/setup_testing.sh
+
+    # Run arm baremetal pytest tests with FVP
+    pytest  --verbose --color=yes --numprocesses=auto backends/arm/test/ --ignore=backends/arm/test/models --arm_run_corstoneFVP
+    echo "${TEST_SUITE_NAME}: PASS"
+}
+
+test_pytest_models_ethosu_fvp() { # Same as test_pytest but also sometime verify using Corstone FVP
+    echo "${TEST_SUITE_NAME}: Run pytest with fvp"
+
+    examples/models/llama3_2_vision/install_requirements.sh
 
     # Prepare Corstone-3x0 FVP for pytest
-    examples/arm/run.sh --model_name=add --build_only
+    backends/arm/scripts/build_executorch.sh
+    backends/arm/scripts/build_portable_kernels.sh
+    # Build semihosting version of the runner used by pytest testing when using --arm_run_corstoneFVP
     backends/arm/test/setup_testing.sh
 
     # Run arm baremetal pytest tests with FVP
-    pytest  --verbose --color=yes --numprocesses=auto backends/arm/test/ --arm_run_corstoneFVP
+    pytest  --verbose --color=yes --numprocesses=auto backends/arm/test/models --arm_run_corstoneFVP
+    echo "${TEST_SUITE_NAME}: PASS"
+}
+
+test_pytest_ethosu_fvp() { # Same as test_pytest but also sometime verify using Corstone FVP
+    echo "${TEST_SUITE_NAME}: Run pytest with fvp"
+    test_pytest_ops_ethosu_fvp
+    test_pytest_models_ethosu_fvp
     echo "${TEST_SUITE_NAME}: PASS"
 }
 
@@ -113,8 +157,8 @@ test_run_ethosu_fvp() { # End to End model tests using run.sh
     echo "${TEST_SUITE_NAME}: PASS"
     }
 
-test_models_ethosu_fvp() { # End to End model tests using model_test.py
-    echo "${TEST_SUITE_NAME}: Test ethos-u delegate models with test_model.py"
+test_models_tosa() { # End to End model tests using model_test.py
+    echo "${TEST_SUITE_NAME}: Test TOSA delegated models with test_model.py"
 
     # Build common libs once
     python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --build_libs
@@ -125,6 +169,22 @@ test_models_ethosu_fvp() { # End to End model tests using model_test.py
     python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=mv3
     python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=lstm
     python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=edsr
+    # python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=emformer_transcribe # Takes long time to run
+    # python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=emformer_join       # Takes long time to run
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=w2l
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=ic3
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=ic4
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=resnet18
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=resnet50
+
+    echo "${TEST_SUITE_NAME}: PASS"
+    }
+
+test_models_ethos-u55() { # End to End model tests using model_test.py
+    echo "${TEST_SUITE_NAME}: Test Ethos-U55 delegated models with test_model.py"
+
+    # Build common libs once
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --build_libs
 
     # Ethos-U55
     echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U55"
@@ -132,19 +192,34 @@ test_models_ethosu_fvp() { # End to End model tests using model_test.py
     python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u55-64  --model=mv3  --extra_flags="-DET_ATOL=5.00 -DET_RTOL=5.00"
     python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u55-256 --model=lstm --extra_flags="-DET_ATOL=0.03 -DET_RTOL=0.03"
 
+    echo "${TEST_SUITE_NAME}: PASS"
+    }
+
+test_models_ethos-u85() { # End to End model tests using model_test.py
+    echo "${TEST_SUITE_NAME}: Test Ethos-U85 delegated models with test_model.py"
+
+    # Build common libs once
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --build_libs
+
     # Ethos-U85
     echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U85"
-    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u85-256  --model=mv2  --extra_flags="-DET_ATOL=2.00 -DET_RTOL=2.00"
-    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u85-1024 --model=mv3  --extra_flags="-DET_ATOL=5.00 -DET_RTOL=5.00"
-    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u85-128  --model=lstm --extra_flags="-DET_ATOL=0.03 -DET_RTOL=0.03"
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u85-256 --model=mv2  --extra_flags="-DET_ATOL=2.00 -DET_RTOL=2.00"
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u85-512 --model=mv3  --extra_flags="-DET_ATOL=5.00 -DET_RTOL=5.00"
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u85-128 --model=lstm --extra_flags="-DET_ATOL=0.03 -DET_RTOL=0.03"
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u85-128 --model=w2l  --extra_flags="-DET_ATOL=0.01 -DET_RTOL=0.01"
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u85-256 --model=ic4  --extra_flags="-DET_ATOL=0.8 -DET_RTOL=0.8" --timeout=2400
+
     echo "${TEST_SUITE_NAME}: PASS"
     }
 
+
 test_full_ethosu_fvp() { # All End to End model tests
     echo "${TEST_SUITE_NAME}: Test ethos-u delegate models and examples on fvp"
 
-    test_models_ethosu_fvp
     test_run_ethosu_fvp
+    test_models_tosa
+    test_models_ethos-u55
+    test_models_ethos-u85
     echo "${TEST_SUITE_NAME}: PASS"
     }
 
diff --git a/backends/arm/test/test_model.py b/backends/arm/test/test_model.py
index da8319ed2ec..b0fd2f2a381 100755
--- a/backends/arm/test/test_model.py
+++ b/backends/arm/test/test_model.py
@@ -61,6 +61,12 @@ def get_args():
         default=None,
         help="Extra cmake flags to pass the when building the executor_runner",
     )
+    parser.add_argument(
+        "--timeout",
+        required=False,
+        default=60 * 10,
+        help="Timeout in seconds used when running the model",
+    )
     args = parser.parse_args()
 
     if args.model and "ethos-u" in args.target and args.system_config is None:
@@ -185,13 +191,14 @@ def build_ethosu_runtime(
     return elf_file
 
 
-def run_elf_with_fvp(script_path: str, elf_file: str, target: str):
+def run_elf_with_fvp(script_path: str, elf_file: str, target: str, timeout: int):
     run_external_cmd(
         [
             "bash",
             os.path.join(script_path, "run_fvp.sh"),
             f"--elf={elf_file}",
             f"--target={target}",
+            f"--timeout={timeout}",
         ]
     )
 
@@ -243,5 +250,5 @@ def run_elf_with_fvp(script_path: str, elf_file: str, target: str):
             )
             print(f"ELF file created: {elf_file} ")
 
-            run_elf_with_fvp(script_path, elf_file, args.target)
+            run_elf_with_fvp(script_path, elf_file, args.target, args.timeout)
         print(f"Model: {model_name} on {args.target} -> PASS")