Arm backend: ArmTester support testing with portable ops

backends/arm/test/runner_utils.py

@@ -707,30 +707,35 @@ def assert_elf_path_exists(elf_path):
         )
 
 
-def get_elf_path(target_board):
+def get_elf_path(target_board: str, use_portable_ops: bool = False):
     if target_board not in VALID_TARGET:
         raise ValueError(f"Unsupported target: {target_board}")
 
+    if use_portable_ops:
+        portable_ops_str = "portable-ops_"
+    else:
+        portable_ops_str = ""
+
     if target_board in ("corstone-300", "corstone-320"):
         elf_path = os.path.join(
             "arm_test",
-            f"arm_semihosting_executor_runner_{target_board}",
+            f"arm_semihosting_executor_runner_{portable_ops_str}{target_board}",
             "arm_executor_runner",
         )
         assert_elf_path_exists(elf_path)
     elif target_board == "vkml_emulation_layer":
         elf_path = os.path.join(
-            "arm_test/arm_executor_runner_vkml",
+            f"arm_test/arm_executor_runner_{portable_ops_str}vkml",
             "executor_runner",
         )
         assert_elf_path_exists(elf_path)
 
     return elf_path
 
 
-def arm_executor_runner_exists(target_board):
+def arm_executor_runner_exists(target_board: str, use_portable_ops: bool = False):
     try:
-        get_elf_path(target_board)
+        get_elf_path(target_board, use_portable_ops=use_portable_ops)
     except:
         return False
     else:

backends/arm/test/setup_testing.sh

@@ -10,6 +10,21 @@ script_dir=$(realpath "$(dirname "${BASH_SOURCE[0]}")")
 et_root_dir=$(realpath "${script_dir}/../../..")
 build_executor_runner=${et_root_dir}/backends/arm/scripts/build_executor_runner.sh
 build_root_test_dir=${et_root_dir}/arm_test/arm_semihosting_executor_runner
+extraflags="-DET_ARM_BAREMETAL_METHOD_ALLOCATOR_POOL_SIZE=83886080"
 
-${build_executor_runner} --pte=semihosting --target=ethos-u55-128 --output="${build_root_test_dir}_corstone-300"
-${build_executor_runner} --pte=semihosting --target=ethos-u85-128 --output="${build_root_test_dir}_corstone-320"
+# By default tests with an elf without any portable_ops
+# If you supply use_portable_ops=True when creating the ArmTester()
+# you will instead test with some portable ops compiled in, see list below.
+
+${build_executor_runner} --pte=semihosting --target=ethos-u55-128 --output="${build_root_test_dir}_corstone-300" --extra_build_flags=${extraflags}
+${build_executor_runner} --pte=semihosting --target=ethos-u85-128 --output="${build_root_test_dir}_corstone-320" --extra_build_flags=${extraflags}
+
+# List of portable ops used by testing, this is mainly used to test models in the flow
+# test setup to make sure models that are not fully delegated can still be tested and run OK
+# To use this you can set use_portable_ops=True when creating ArmTester()
+
+portable_ops_list_u55="aten::permute_copy.out,aten::convolution.out,aten::relu.out,aten::_native_batch_norm_legit_no_training.out,aten::as_strided_copy.out,aten::mean.out,aten::squeeze_copy.dims,dim_order_ops::_clone_dim_order.out"
+portable_ops_list_u85="aten::permute_copy.out,aten::convolution.out,aten::relu.out,aten::_native_batch_norm_legit_no_training.out,aten::as_strided_copy.out,aten::mean.out,aten::full_like.out,aten::bmm.out,aten::scalar_tensor.out,aten::index.Tensor_out,aten::where.self_out" 
+
+${build_executor_runner} --pte=semihosting --target=ethos-u55-128 --select_ops_list="${portable_ops_list_u55}" --output="${build_root_test_dir}_portable-ops_corstone-300" --extra_build_flags=${extraflags}
+${build_executor_runner} --pte=semihosting --target=ethos-u85-128 --select_ops_list="${portable_ops_list_u85}" --output="${build_root_test_dir}_portable-ops_corstone-320" --extra_build_flags=${extraflags}

backends/arm/test/tester/arm_tester.py

@@ -249,6 +249,8 @@ def __init__(
         transform_passes: Optional[
             Union[Sequence[PassType], Dict[str, Sequence[PassType]]]
         ] = None,
+        use_portable_ops: bool = False,
+        timeout: int = 480,
     ):
         """
         Args:
@@ -269,6 +271,8 @@ def __init__(
         # Initial model needs to be set as a *possible* but not yet added Stage, therefore add None entry.
         self.stages[StageType.INITIAL_MODEL] = None
         self._run_stage(InitialModel(self.original_module))
+        self.use_portable_ops = use_portable_ops
+        self.timeout = timeout
 
     def quantize(
         self,
@@ -346,13 +350,15 @@ def to_executorch(self, to_executorch_stage: Optional[ToExecutorch] | None = Non
         return super().to_executorch(to_executorch_stage)
 
     def serialize(
-        self, serialize_stage: Optional[Serialize] = None, timeout: int = 480
+        self,
+        serialize_stage: Optional[Serialize] = None,
     ):
         if serialize_stage is None:
             serialize_stage = Serialize(
                 compile_spec=self.compile_spec,
                 module=self.original_module,
-                timeout=timeout,
+                use_portable_ops=self.use_portable_ops,
+                timeout=self.timeout,
             )
         assert (
             self.compile_spec.get_intermediate_path() is not None

backends/arm/test/tester/serialize.py

@@ -31,19 +31,22 @@ def __init__(
         self,
         compile_spec: ArmCompileSpec,
         module: Optional[torch.nn.Module],
+        use_portable_ops: bool = False,
         timeout: int = 120,
     ):
         """
         Args:
             compile_spec: CompileSpecs to be used for serialization.
             module: Original Module to be used for serialization. Optional - can be used for reference output generation.
+            portable_ops: If True tests with compiled in portable ops, default is to test without this to get error if not fully delegated
             timeout: Timeout for fvp. Default is 120 seconds.
         """
         super().__init__()
         self.module = module
         self.timeout = timeout
         self.executorch_program_manager: ExecutorchProgramManager | None
         self.compile_spec = compile_spec
+        self.use_portable_ops = use_portable_ops
 
     def run(self, artifact: ExecutorchProgramManager, inputs=None) -> None:
         super().run(artifact, inputs)
@@ -58,7 +61,7 @@ def run_artifact(self, inputs):
         inputs_flattened, _ = tree_flatten(inputs)
         intermediate_path = self.compile_spec.get_intermediate_path()
         target_board = get_target_board(self.compile_spec)
-        elf_path = get_elf_path(target_board)
+        elf_path = get_elf_path(target_board, self.use_portable_ops)
 
         if not os.path.exists(elf_path):
             raise FileNotFoundError(

backends/test/suite/flows/arm.py

@@ -20,11 +20,15 @@ def _create_arm_flow(
     compile_spec: ArmCompileSpec,
     symmetric_io_quantization: bool = False,
     per_channel_quantization: bool = True,
+    use_portable_ops: bool = True,
+    timeout: int = 600,
 ) -> TestFlow:
 
     def _create_arm_tester(*args, **kwargs) -> ArmTester:
         kwargs["compile_spec"] = compile_spec
-        return ArmTester(*args, **kwargs)
+        return ArmTester(
+            *args, **kwargs, use_portable_ops=use_portable_ops, timeout=timeout
+        )
 
     support_serialize = not isinstance(compile_spec, TosaCompileSpec)
     quantize = compile_spec.tosa_spec.support_integer()