Allow TOSA tests to not have quant info

Quantized models might have output without quantization parameters
attached to them. The assert for parameters not being None are removed
and handled in order to allow for that case.
numpy transpose is removed in favor of torch.permute to keep the type
of the output after the operation.

Signed-off-by: Per Åstrand <[email protected]>
Change-Id: I0e404062154cefa39f18b5706d72d19cac0e6d73

Author: per

backends/arm/test/runner_utils.py

@@ -127,7 +127,7 @@ def _get_output_node(program: ExportedProgram) -> Node:
 
 def _get_output_quantization_params(
     program: ExportedProgram, output_node: Node
-) -> QuantizationParams:
+) -> Optional[QuantizationParams]:
     """
     Get output QuantizationParams from a program.
     Args:
@@ -153,8 +153,6 @@ def _get_output_quantization_params(
                 dtype=node.args[5],
             )
             break  # break early, there's only one output node
-    if quant_params is None:
-        raise RuntimeError("No Quantization parameters not found in exported model.")
     return quant_params
 
 
@@ -485,13 +483,17 @@ def run_tosa_ref_model(
                 if tosa_ref_output.dtype == np.int8:
                     tosa_ref_output = tosa_ref_output.astype(np.int32)
                 quant_param = self.qp_output
-                assert (
-                    quant_param is not None
-                ), "There are no quantization parameters, check output parameters"
-                tosa_ref_output = (tosa_ref_output - quant_param.zp) * quant_param.scale
+                if quant_param is not None:
+                    # I.e. bool output is possible for quantized models
+                    tosa_ref_output = (
+                        tosa_ref_output - quant_param.zp
+                    ) * quant_param.scale
 
             if tosa_ref_output.dtype == np.double:
                 tosa_ref_output = tosa_ref_output.astype("float32")
+            elif tosa_ref_output.dtype == bool:
+                # retain the bool output though for boolean related comparisons
+                tosa_ref_output = tosa_ref_output.astype("bool")
 
             # tosa_output is a numpy array, convert to torch tensor for comparison
             tosa_ref_outputs.append(torch.from_numpy(tosa_ref_output))

backends/arm/test/tester/arm_tester.py

@@ -12,7 +12,6 @@
 
 import executorch.backends.xnnpack.test.tester.tester as tester
 
-import numpy as np
 import serializer.tosa_serializer as ts
 
 import torch.fx
@@ -319,12 +318,15 @@ def run_method_and_compare_outputs(
             target_board,
         )
 
+        quantization_scale = None
         if is_quantized:
             reference_stage = self.stages[self.stage_name(tester.Quantize)]
-            quantization_scale = self.runner_util.qp_output.scale
+            # bool output is quantized with none quantized output so allow
+            # self.runner_util.qp_output to be none
+            if self.runner_util.qp_output is not None:
+                quantization_scale = self.runner_util.qp_output.scale
         else:
             reference_stage = self.stages[self.stage_name(InitialModel)]
-            quantization_scale = None
 
         logger.info(
             f"Comparing Stage '{self.stage_name(test_stage)}' with Stage '{self.stage_name(reference_stage)}'"
@@ -504,7 +506,7 @@ def transpose_data_format(
         inputs_transposed = list(data)
         for i in range(len(data)):
             if hasattr(data[i], "shape") and len(data[i].shape) == 4:
-                inputs_transposed[i] = np.transpose(data[i], dim_order)
+                inputs_transposed[i] = torch.permute(data[i], dim_order)
         return tuple(inputs_transposed)
 
     def _compare_outputs(