[Backend Tester] Add tensor error statistic reporting

backends/qualcomm/tests/TARGETS

@@ -37,3 +37,13 @@ python_library(
         "//executorch/backends/qualcomm/debugger:utils",
     ],
 )
+
+python_library(
+    name = "tester",
+    srcs = [
+        "tester.py",
+    ],
+    deps = [
+        ":test_qnn_delegate"
+    ]
+)

backends/qualcomm/tests/tester.py

@@ -0,0 +1,88 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+from typing import Any, List, Optional, Tuple
+
+import executorch
+import executorch.backends.test.harness.stages as BaseStages
+
+import torch
+from executorch.backends.qualcomm._passes.qnn_pass_manager import QnnPassManager
+from executorch.backends.qualcomm.partition.qnn_partitioner import QnnPartitioner
+from executorch.backends.qualcomm.utils.utils import (
+    generate_htp_compiler_spec,
+    generate_qnn_executorch_compiler_spec,
+    get_soc_to_chipset_map,
+)
+from executorch.backends.test.harness import Tester as TesterBase
+from executorch.backends.test.harness.stages import StageType
+from executorch.exir import EdgeCompileConfig, to_edge_transform_and_lower
+from executorch.exir.backend.partitioner import Partitioner
+from torch.export import ExportedProgram
+
+
+class Partition(BaseStages.Partition):
+    def __init__(self, partitioner: Optional[Partitioner] = None):
+        super().__init__(
+            partitioner=partitioner or QnnPartitioner,
+        )
+
+
+class ToEdgeTransformAndLower(BaseStages.ToEdgeTransformAndLower):
+    def __init__(
+        self,
+        partitioners: Optional[List[Partitioner]] = None,
+        edge_compile_config: Optional[EdgeCompileConfig] = None,
+        soc_model: str = "SM8650",
+    ):
+        backend_options = generate_htp_compiler_spec(use_fp16=True)
+        self.chipset = get_soc_to_chipset_map()[soc_model]
+        self.compiler_specs = generate_qnn_executorch_compiler_spec(
+            soc_model=self.chipset,
+            backend_options=backend_options,
+        )
+
+        super().__init__(
+            partitioners=partitioners or [QnnPartitioner(self.compiler_specs)],
+            edge_compile_config=edge_compile_config
+            or EdgeCompileConfig(_check_ir_validity=False),
+            default_partitioner_cls=QnnPartitioner,
+        )
+
+    def run(self, artifact: ExportedProgram, inputs=None) -> None:
+        ep = QnnPassManager().transform_for_export_pipeline(artifact)
+        transform_passes = QnnPassManager().get_to_edge_transform_passes(ep)
+
+        self.edge_dialect_program = to_edge_transform_and_lower(
+            ep,
+            transform_passes=transform_passes,
+            partitioner=self.partitioners,
+            compile_config=self.edge_compile_conf,
+        )
+
+
+class QualcommTester(TesterBase):
+    def __init__(
+        self,
+        module: torch.nn.Module,
+        example_inputs: Tuple[torch.Tensor],
+        dynamic_shapes: Optional[Tuple[Any]] = None,
+    ):
+        # Specialize for Qualcomm
+        stage_classes = (
+            executorch.backends.test.harness.Tester.default_stage_classes()
+            | {
+                StageType.PARTITION: Partition,
+                StageType.TO_EDGE_TRANSFORM_AND_LOWER: ToEdgeTransformAndLower,
+            }
+        )
+
+        super().__init__(
+            module=module,
+            stage_classes=stage_classes,
+            example_inputs=example_inputs,
+            dynamic_shapes=dynamic_shapes,
+        )

backends/test/harness/error_statistics.py

@@ -0,0 +1,99 @@
+from dataclasses import dataclass
+
+import torch
+from torch.ao.ns.fx.utils import compute_sqnr
+
+
+@dataclass
+class TensorStatistics:
+    """Contains summary statistics for a tensor."""
+
+    shape: torch.Size
+    """ The shape of the tensor. """
+
+    numel: int
+    """ The number of elements in the tensor. """
+
+    median: float
+    """ The median of the tensor. """
+
+    mean: float
+    """ The mean of the tensor. """
+
+    max: torch.types.Number
+    """ The maximum element of the tensor. """
+
+    min: torch.types.Number
+    """ The minimum element of the tensor. """
+
+    @classmethod
+    def from_tensor(cls, tensor: torch.Tensor) -> "TensorStatistics":
+        """Creates a TensorStatistics object from a tensor."""
+        flattened = torch.flatten(tensor)
+        return cls(
+            shape=tensor.shape,
+            numel=tensor.numel(),
+            median=torch.quantile(flattened, q=0.5).item(),
+            mean=flattened.mean().item(),
+            max=flattened.max().item(),
+            min=flattened.min().item(),
+        )
+
+
+@dataclass
+class ErrorStatistics:
+    """Contains statistics derived from the difference of two tensors."""
+
+    reference_stats: TensorStatistics
+    """ Statistics for the reference tensor. """
+
+    actual_stats: TensorStatistics
+    """ Statistics for the actual tensor. """
+
+    error_l2_norm: float | None
+    """ The L2 norm of the error between the actual and reference tensor. """
+
+    error_mae: float | None
+    """ The mean absolute error between the actual and reference tensor. """
+
+    error_max: float | None
+    """ The maximum absolute elementwise error between the actual and reference tensor. """
+
+    error_msd: float | None
+    """ The mean signed deviation between the actual and reference tensor. """
+
+    sqnr: float | None
+    """ The signal-to-quantization-noise ratio between the actual and reference tensor. """
+
+    @classmethod
+    def from_tensors(
+        cls, actual: torch.Tensor, reference: torch.Tensor
+    ) -> "ErrorStatistics":
+        """Creates an ErrorStatistics object from two tensors."""
+        actual = actual.to(torch.float64)
+        reference = reference.to(torch.float64)
+
+        if actual.shape != reference.shape:
+            return cls(
+                reference_stats=TensorStatistics.from_tensor(reference),
+                actual_stats=TensorStatistics.from_tensor(actual),
+                error_l2_norm=None,
+                error_mae=None,
+                error_max=None,
+                error_msd=None,
+                sqnr=None,
+            )
+
+        error = actual - reference
+        flat_error = torch.flatten(error)
+
+        return cls(
+            reference_stats=TensorStatistics.from_tensor(reference),
+            actual_stats=TensorStatistics.from_tensor(actual),
+            error_l2_norm=torch.linalg.norm(flat_error).item(),
+            error_mae=torch.mean(torch.abs(flat_error)).item(),
+            error_max=torch.max(torch.abs(flat_error)).item(),
+            error_msd=torch.mean(flat_error).item(),
+            # Torch sqnr implementation requires float32 due to decorator logic
+            sqnr=compute_sqnr(actual.to(torch.float), reference.to(torch.float)).item(),
+        )

backends/test/harness/tester.py

@@ -4,6 +4,7 @@
 
 import torch
 
+from executorch.backends.test.harness.error_statistics import ErrorStatistics
 from executorch.backends.test.harness.stages import (
     Export,
     Partition,
@@ -302,20 +303,15 @@ def run_method_and_compare_outputs(
         atol=1e-03,
         rtol=1e-03,
         qtol=0,
+        statistics_callback: Callable[[ErrorStatistics], None] | None = None,
     ):
         number_of_runs = 1 if inputs is not None else num_runs
         reference_stage = self.stages[StageType.EXPORT]
 
         stage = stage or self.cur
 
-        print(f"Comparing Stage {stage} with Stage {reference_stage}")
-        for run_iteration in range(number_of_runs):
+        for _ in range(number_of_runs):
             inputs_to_run = inputs if inputs else next(self.generate_random_inputs())
-            input_shapes = [
-                generated_input.shape if hasattr(generated_input, "shape") else None
-                for generated_input in inputs_to_run
-            ]
-            print(f"Run {run_iteration} with input shapes: {input_shapes}")
 
             # Reference output (and quantization scale)
             (
@@ -328,13 +324,25 @@ def run_method_and_compare_outputs(
             # Output from running artifact at stage
             stage_output = self.stages[stage].run_artifact(inputs_to_run)
             self._compare_outputs(
-                reference_output, stage_output, quantization_scale, atol, rtol, qtol
+                reference_output,
+                stage_output,
+                quantization_scale,
+                atol,
+                rtol,
+                qtol,
+                statistics_callback,
             )
 
         return self
 
     @staticmethod
-    def _assert_outputs_equal(model_output, ref_output, atol=1e-03, rtol=1e-03):
+    def _assert_outputs_equal(
+        model_output,
+        ref_output,
+        atol=1e-03,
+        rtol=1e-03,
+        statistics_callback: Callable[[ErrorStatistics], None] | None = None,
+    ):
         """
         Helper testing function that asserts that the model output and the reference output
         are equal with some tolerance. Due to numerical differences between eager mode and
@@ -349,6 +357,11 @@ def _assert_outputs_equal(model_output, ref_output, atol=1e-03, rtol=1e-03):
         for i in range(len(model_output)):
             model = model_output[i]
             ref = ref_output[i]
+
+            error_stats = ErrorStatistics.from_tensors(model, ref)
+            if statistics_callback is not None:
+                statistics_callback(error_stats)
+
             assert (
                 ref.shape == model.shape
             ), f"Output {i} shape {model.shape} does not match reference output shape {ref.shape}"
@@ -386,6 +399,7 @@ def _compare_outputs(
         atol=1e-03,
         rtol=1e-03,
         qtol=0,
+        statistics_callback: Callable[[ErrorStatistics], None] | None = None,
     ):
         """
         Compares the original of the original nn module with the output of the generated artifact.
@@ -408,6 +422,7 @@ def _compare_outputs(
             reference_output,
             atol=atol,
             rtol=rtol,
+            statistics_callback=statistics_callback,
         )
 
     @staticmethod

backends/test/harness/tests/test_error_statistics.py

@@ -0,0 +1,65 @@
+import unittest
+
+import torch
+from executorch.backends.test.harness.error_statistics import ErrorStatistics
+
+
+class ErrorStatisticsTests(unittest.TestCase):
+    def test_error_stats_simple(self):
+        tensor1 = torch.tensor([1, 2, 3, 4])
+        tensor2 = torch.tensor([2, 2, 2, 5])
+
+        error_stats = ErrorStatistics.from_tensors(tensor1, tensor2)
+
+        # Check actual tensor statistics
+        self.assertEqual(error_stats.actual_stats.shape, torch.Size([4]))
+        self.assertEqual(error_stats.actual_stats.numel, 4)
+        self.assertEqual(error_stats.actual_stats.median, 2.5)
+        self.assertEqual(error_stats.actual_stats.mean, 2.5)
+        self.assertEqual(error_stats.actual_stats.max, 4)
+        self.assertEqual(error_stats.actual_stats.min, 1)
+
+        # Check reference tensor statistics
+        self.assertEqual(error_stats.reference_stats.shape, torch.Size([4]))
+        self.assertEqual(error_stats.reference_stats.numel, 4)
+        self.assertEqual(error_stats.reference_stats.median, 2.0)
+        self.assertEqual(error_stats.reference_stats.mean, 2.75)
+        self.assertEqual(error_stats.reference_stats.max, 5)
+        self.assertEqual(error_stats.reference_stats.min, 2)
+
+        # Check error statistics
+        self.assertAlmostEqual(error_stats.error_l2_norm, 1.732, places=3)
+        self.assertEqual(error_stats.error_mae, 0.75)
+        self.assertEqual(error_stats.error_max, 1.0)
+        self.assertEqual(error_stats.error_msd, -0.25)
+        self.assertAlmostEqual(error_stats.sqnr, 10.0, places=3)
+
+    def test_error_stats_different_shapes(self):
+        # Create tensors with different shapes
+        tensor1 = torch.tensor([1, 2, 3, 4])
+        tensor2 = torch.tensor([[2, 3], [4, 5]])
+
+        error_stats = ErrorStatistics.from_tensors(tensor1, tensor2)
+
+        # Check actual tensor statistics
+        self.assertEqual(error_stats.actual_stats.shape, torch.Size([4]))
+        self.assertEqual(error_stats.actual_stats.numel, 4)
+        self.assertEqual(error_stats.actual_stats.median, 2.5)
+        self.assertEqual(error_stats.actual_stats.mean, 2.5)
+        self.assertEqual(error_stats.actual_stats.max, 4)
+        self.assertEqual(error_stats.actual_stats.min, 1)
+
+        # Check reference tensor statistics
+        self.assertEqual(error_stats.reference_stats.shape, torch.Size([2, 2]))
+        self.assertEqual(error_stats.reference_stats.numel, 4)
+        self.assertEqual(error_stats.reference_stats.median, 3.5)
+        self.assertEqual(error_stats.reference_stats.mean, 3.5)
+        self.assertEqual(error_stats.reference_stats.max, 5)
+        self.assertEqual(error_stats.reference_stats.min, 2)
+
+        # Check that all error values are None when shapes differ
+        self.assertIsNone(error_stats.error_l2_norm)
+        self.assertIsNone(error_stats.error_mae)
+        self.assertIsNone(error_stats.error_max)
+        self.assertIsNone(error_stats.error_msd)
+        self.assertIsNone(error_stats.sqnr)

backends/test/suite/context.py

@@ -1,8 +1,11 @@
 # Test run context management. This is used to determine the test context for reporting
 # purposes.
 class TestContext:
-    def __init__(self, test_name: str, flow_name: str, params: dict | None):
+    def __init__(
+        self, test_name: str, test_base_name: str, flow_name: str, params: dict | None
+    ):
         self.test_name = test_name
+        self.test_base_name = test_base_name
         self.flow_name = flow_name
         self.params = params
 

backends/test/suite/flow.py

@@ -71,4 +71,13 @@ def all_flows() -> dict[str, TestFlow]:
     except Exception as e:
         logger.info(f"Skipping Vulkan flow registration: {e}")
 
+    try:
+        from executorch.backends.test.suite.flows.qualcomm import QUALCOMM_TEST_FLOW
+
+        flows += [
+            QUALCOMM_TEST_FLOW,
+        ]
+    except Exception as e:
+        logger.info(f"Skipping Qualcomm flow registration: {e}")
+
     return {f.name: f for f in flows if f is not None}