[Backend Tester] Seed based on test name

CMakeLists.txt

@@ -757,6 +757,10 @@ if(EXECUTORCH_BUILD_PYBIND)
     list(APPEND _dep_libs openvino_backend)
   endif()
 
+  if(EXECUTORCH_BUILD_VULKAN)
+    list(APPEND _dep_libs vulkan_backend)
+  endif()
+
   if(EXECUTORCH_BUILD_XNNPACK)
     # need to explicitly specify XNNPACK and xnnpack-microkernels-prod here
     # otherwise uses XNNPACK and microkernel-prod symbols from libtorch_cpu

backends/qualcomm/tests/TARGETS

@@ -37,3 +37,13 @@ python_library(
         "//executorch/backends/qualcomm/debugger:utils",
     ],
 )
+
+python_library(
+    name = "tester",
+    srcs = [
+        "tester.py",
+    ],
+    deps = [
+        ":test_qnn_delegate"
+    ]
+)

backends/qualcomm/tests/tester.py

@@ -0,0 +1,91 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+from typing import Any, List, Optional, Tuple
+
+import executorch
+import executorch.backends.test.harness.stages as BaseStages
+
+import torch
+from executorch.backends.qualcomm._passes.qnn_pass_manager import QnnPassManager
+from executorch.backends.qualcomm.partition.qnn_partitioner import QnnPartitioner
+from executorch.backends.qualcomm.utils.utils import (
+    generate_htp_compiler_spec,
+    generate_qnn_executorch_compiler_spec,
+    get_soc_to_chipset_map,
+)
+from executorch.backends.test.harness import Tester as TesterBase
+from executorch.backends.test.harness.stages import StageType
+from executorch.exir import EdgeCompileConfig, to_edge_transform_and_lower
+from executorch.exir.backend.partitioner import Partitioner
+from torch.export import ExportedProgram
+
+
+class Partition(BaseStages.Partition):
+    def __init__(self, partitioner: Optional[Partitioner] = None):
+        super().__init__(
+            partitioner=partitioner or QnnPartitioner,
+        )
+
+
+class ToEdgeTransformAndLower(BaseStages.ToEdgeTransformAndLower):
+    def __init__(
+        self,
+        partitioners: Optional[List[Partitioner]] = None,
+        edge_compile_config: Optional[EdgeCompileConfig] = None,
+        soc_model: str = "SM8650",
+    ):
+        backend_options = generate_htp_compiler_spec(use_fp16=True)
+        self.chipset = get_soc_to_chipset_map()[soc_model]
+        self.compiler_specs = generate_qnn_executorch_compiler_spec(
+            soc_model=self.chipset,
+            backend_options=backend_options,
+        )
+
+        super().__init__(
+            partitioners=partitioners or [QnnPartitioner(self.compiler_specs)],
+            edge_compile_config=edge_compile_config
+            or EdgeCompileConfig(_check_ir_validity=False),
+            default_partitioner_cls=QnnPartitioner,
+        )
+
+    def run(
+        self, artifact: ExportedProgram, inputs=None, generate_etrecord: bool = False
+    ) -> None:
+        ep = QnnPassManager().transform_for_export_pipeline(artifact)
+        transform_passes = QnnPassManager().get_to_edge_transform_passes(ep)
+
+        self.edge_dialect_program = to_edge_transform_and_lower(
+            ep,
+            transform_passes=transform_passes,
+            partitioner=self.partitioners,
+            compile_config=self.edge_compile_conf,
+            generate_etrecord=generate_etrecord,
+        )
+
+
+class QualcommTester(TesterBase):
+    def __init__(
+        self,
+        module: torch.nn.Module,
+        example_inputs: Tuple[torch.Tensor],
+        dynamic_shapes: Optional[Tuple[Any]] = None,
+    ):
+        # Specialize for Qualcomm
+        stage_classes = (
+            executorch.backends.test.harness.Tester.default_stage_classes()
+            | {
+                StageType.PARTITION: Partition,
+                StageType.TO_EDGE_TRANSFORM_AND_LOWER: ToEdgeTransformAndLower,
+            }
+        )
+
+        super().__init__(
+            module=module,
+            stage_classes=stage_classes,
+            example_inputs=example_inputs,
+            dynamic_shapes=dynamic_shapes,
+        )

backends/test/harness/error_statistics.py

@@ -0,0 +1,99 @@
+from dataclasses import dataclass
+
+import torch
+from torch.ao.ns.fx.utils import compute_sqnr
+
+
+@dataclass
+class TensorStatistics:
+    """Contains summary statistics for a tensor."""
+
+    shape: torch.Size
+    """ The shape of the tensor. """
+
+    numel: int
+    """ The number of elements in the tensor. """
+
+    median: float
+    """ The median of the tensor. """
+
+    mean: float
+    """ The mean of the tensor. """
+
+    max: torch.types.Number
+    """ The maximum element of the tensor. """
+
+    min: torch.types.Number
+    """ The minimum element of the tensor. """
+
+    @classmethod
+    def from_tensor(cls, tensor: torch.Tensor) -> "TensorStatistics":
+        """Creates a TensorStatistics object from a tensor."""
+        flattened = torch.flatten(tensor)
+        return cls(
+            shape=tensor.shape,
+            numel=tensor.numel(),
+            median=torch.quantile(flattened, q=0.5).item(),
+            mean=flattened.mean().item(),
+            max=flattened.max().item(),
+            min=flattened.min().item(),
+        )
+
+
+@dataclass
+class ErrorStatistics:
+    """Contains statistics derived from the difference of two tensors."""
+
+    reference_stats: TensorStatistics
+    """ Statistics for the reference tensor. """
+
+    actual_stats: TensorStatistics
+    """ Statistics for the actual tensor. """
+
+    error_l2_norm: float | None
+    """ The L2 norm of the error between the actual and reference tensor. """
+
+    error_mae: float | None
+    """ The mean absolute error between the actual and reference tensor. """
+
+    error_max: float | None
+    """ The maximum absolute elementwise error between the actual and reference tensor. """
+
+    error_msd: float | None
+    """ The mean signed deviation between the actual and reference tensor. """
+
+    sqnr: float | None
+    """ The signal-to-quantization-noise ratio between the actual and reference tensor. """
+
+    @classmethod
+    def from_tensors(
+        cls, actual: torch.Tensor, reference: torch.Tensor
+    ) -> "ErrorStatistics":
+        """Creates an ErrorStatistics object from two tensors."""
+        actual = actual.to(torch.float64)
+        reference = reference.to(torch.float64)
+
+        if actual.shape != reference.shape:
+            return cls(
+                reference_stats=TensorStatistics.from_tensor(reference),
+                actual_stats=TensorStatistics.from_tensor(actual),
+                error_l2_norm=None,
+                error_mae=None,
+                error_max=None,
+                error_msd=None,
+                sqnr=None,
+            )
+
+        error = actual - reference
+        flat_error = torch.flatten(error)
+
+        return cls(
+            reference_stats=TensorStatistics.from_tensor(reference),
+            actual_stats=TensorStatistics.from_tensor(actual),
+            error_l2_norm=torch.linalg.norm(flat_error).item(),
+            error_mae=torch.mean(torch.abs(flat_error)).item(),
+            error_max=torch.max(torch.abs(flat_error)).item(),
+            error_msd=torch.mean(flat_error).item(),
+            # Torch sqnr implementation requires float32 due to decorator logic
+            sqnr=compute_sqnr(actual.to(torch.float), reference.to(torch.float)).item(),
+        )

backends/test/harness/stages/serialize.py

@@ -13,6 +13,7 @@
 try:
     from executorch.extension.pybindings.portable_lib import (  # @manual
         _load_for_executorch_from_buffer,
+        Verification,
     )
 except ImportError as e:
     logger.warning(f"{e=}")
@@ -39,7 +40,9 @@ def graph_module(self) -> None:
 
     def run_artifact(self, inputs):
         inputs_flattened, _ = tree_flatten(inputs)
-        executorch_module = _load_for_executorch_from_buffer(self.buffer)
+        executorch_module = _load_for_executorch_from_buffer(
+            self.buffer, program_verification=Verification.Minimal
+        )
         executorch_output = copy.deepcopy(
             executorch_module.run_method("forward", tuple(inputs_flattened))
         )

backends/test/harness/stages/to_edge_transform_and_lower.py

@@ -7,28 +7,38 @@
     to_edge_transform_and_lower,
 )
 from executorch.exir.backend.partitioner import Partitioner
+
 from torch.export import ExportedProgram
 
 
 class ToEdgeTransformAndLower(Stage):
     def __init__(
         self,
-        default_partitioner_cls: Type,
+        default_partitioner_cls: Type | None = None,
         partitioners: Optional[List[Partitioner]] = None,
         edge_compile_config: Optional[EdgeCompileConfig] = None,
     ):
-        self.partitioners = partitioners or [default_partitioner_cls()]
-        self.edge_compile_conf = edge_compile_config or EdgeCompileConfig()
+        self.partitioners = (
+            partitioners or [default_partitioner_cls()]
+            if default_partitioner_cls is not None
+            else []
+        )
+        self.edge_compile_conf = edge_compile_config or EdgeCompileConfig(
+            _check_ir_validity=False
+        )
         self.edge_dialect_program = None
 
     def stage_type(self) -> StageType:
         return StageType.TO_EDGE_TRANSFORM_AND_LOWER
 
-    def run(self, artifact: ExportedProgram, inputs=None) -> None:
+    def run(
+        self, artifact: ExportedProgram, inputs=None, generate_etrecord: bool = False
+    ) -> None:
         self.edge_dialect_program = to_edge_transform_and_lower(
             artifact,
             compile_config=self.edge_compile_conf,
             partitioner=self.partitioners,
+            generate_etrecord=generate_etrecord,
         )
 
     @property