Arm backend: Enable VGF backend testing in github (#15847)

Reapply "Arm backend: Enable pytest VKML op unit testing in
ciflow/trunk" (#15841)

This reverts commit 0c0cee5.

Reason why the first commit was reverted was because the all op test
timed out. Therefore on top of the "reapply" test_pytest_ops_vkml is
split out into its own job, test-arm-backend-vkml. This means the
original all ops test will run like before.

cc @freddan80 @per @zingo @oscarandersson8218 @digantdesai

Author: mansnils

.ci/scripts/test_backend.sh

@@ -57,15 +57,25 @@ if [[ "$FLOW" == *vulkan* ]]; then
 fi
 
 if [[ "$FLOW" == *arm* ]]; then
+
     # Setup ARM deps.
-    .ci/scripts/setup-arm-baremetal-tools.sh
+    if [[ "$FLOW" == *vgf* ]]; then
+        .ci/scripts/setup-arm-baremetal-tools.sh --enable-mlsdk-deps --install-mlsdk-deps-with-pip
+    else
+        .ci/scripts/setup-arm-baremetal-tools.sh
+    fi
     source examples/arm/ethos-u-scratch/setup_path.sh
 
     if [[ "$FLOW" == *ethos_u* ]]; then
         # Prepare a test runner binary that can run on the Corstone-3x0 FVPs
         backends/arm/scripts/build_executorch.sh
         backends/arm/test/setup_testing.sh
     fi
+
+    if [[ "$FLOW" == *vgf* ]]; then
+        # Prepare a test runner binary for VKML runtime
+        backends/arm/test/setup_testing_vkml.sh
+    fi
 fi
 
 if [[ $IS_MACOS -eq 1 ]]; then

.github/workflows/test-backend-arm.yml

@@ -26,7 +26,7 @@ jobs:
     uses: ./.github/workflows/_test_backend.yml
     with:
       backend: arm
-      flows: '["arm_tosa_fp", "arm_tosa_int", "arm_ethos_u55", "arm_ethos_u85"]'
+      flows: '["arm_tosa_fp", "arm_tosa_int", "arm_ethos_u55", "arm_ethos_u85", "arm_vgf_fp", "arm_vgf_int"]'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 120
       run-linux: true

.github/workflows/trunk.yml

@@ -317,6 +317,40 @@ jobs:
         # Test test_arm_baremetal.sh with test
         backends/arm/test/test_arm_baremetal.sh "${ARM_TEST}"
 
+  test-arm-backend-vkml:
+    name: test-arm-backend-vkml
+    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      matrix:
+        include:
+          - test_arm_baremetal: test_pytest_ops_vkml
+      fail-fast: false
+    with:
+      runner: linux.2xlarge.memory
+      docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk
+      submodules: 'recursive'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 120
+      script: |
+        # The generic Linux job chooses to use base env, not the one setup by the image
+        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+        conda activate "${CONDA_ENV}"
+        source .ci/scripts/utils.sh
+        install_executorch "--use-pt-pinned-commit"
+
+        .ci/scripts/setup-arm-baremetal-tools.sh --disable-ethos-u-deps --enable-mlsdk-deps --install-mlsdk-deps-with-pip
+
+        # Increase number of files user can monitor to bypass buck failures.
+        # Hopefully this is high enough for this setup.
+        sudo sysctl fs.inotify.max_user_watches=1048576 # 1024 * 1024
+
+        ARM_TEST=${{ matrix.test_arm_baremetal }}
+
+        backends/arm/test/test_arm_baremetal.sh "${ARM_TEST}"
+
   test-arm-cortex-m-size-test:
     name: test-arm-cortex-m-size-test
     uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main

backends/arm/test/test_arm_baremetal.sh

@@ -157,7 +157,6 @@ test_pytest_ethosu_fvp() { # Same as test_pytest but also sometime verify using
 test_pytest_ops_vkml() { # Same as test_pytest but also sometime verify using VKML runtime
     echo "${TEST_SUITE_NAME}: Run pytest operator tests with VKML runtime"
 
-    backends/arm/scripts/build_executorch.sh
     backends/arm/test/setup_testing_vkml.sh
 
     pytest  --verbose --color=yes --numprocesses=auto --durations=10  backends/arm/test/ \

backends/test/suite/flow.py

@@ -132,13 +132,17 @@ def all_flows() -> dict[str, TestFlow]:
             ARM_ETHOS_U85_FLOW,
             ARM_TOSA_FP_FLOW,
             ARM_TOSA_INT_FLOW,
+            ARM_VGF_FP_FLOW,
+            ARM_VGF_INT_FLOW,
         )
 
         flows += [
             ARM_TOSA_FP_FLOW,
             ARM_TOSA_INT_FLOW,
             ARM_ETHOS_U55_FLOW,
             ARM_ETHOS_U85_FLOW,
+            ARM_VGF_FP_FLOW,
+            ARM_VGF_INT_FLOW,
         ]
     except Exception as e:
         logger.info(f"Skipping ARM flow registration: {e}")

backends/test/suite/flows/arm.py

@@ -78,3 +78,15 @@ def create_quantize_stage() -> Quantize:
     lambda: common.get_u85_compile_spec(),
     quantize=True,
 )
+ARM_VGF_FP_FLOW = _create_arm_flow(
+    "arm_vgf_fp",
+    lambda: common.get_vgf_compile_spec(tosa_spec="TOSA-1.0+FP"),
+    quantize=False,
+    use_portable_ops=False,
+)
+ARM_VGF_INT_FLOW = _create_arm_flow(
+    "arm_vgf_int",
+    lambda: common.get_vgf_compile_spec(tosa_spec="TOSA-1.0+INT"),
+    quantize=True,
+    use_portable_ops=False,
+)