[Backend Tester] Add CSV report generation

backends/apple/coreml/test/tester.py

@@ -4,23 +4,64 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-from typing import Any, List, Optional, Tuple
+from typing import Any, List, Optional, Sequence, Tuple
 
+import coremltools as ct
 import executorch
 import executorch.backends.test.harness.stages as BaseStages
-
+import functools
 import torch
+
+from executorch.backends.apple.coreml.compiler import CoreMLBackend
 from executorch.backends.apple.coreml.partition import CoreMLPartitioner
+from executorch.backends.apple.coreml.quantizer import CoreMLQuantizer
 from executorch.backends.test.harness import Tester as TesterBase
 from executorch.backends.test.harness.stages import StageType
 from executorch.exir import EdgeCompileConfig
 from executorch.exir.backend.partitioner import Partitioner
 
 
+def _get_static_int8_qconfig():
+    return ct.optimize.torch.quantization.LinearQuantizerConfig(
+        global_config=ct.optimize.torch.quantization.ModuleLinearQuantizerConfig(
+            quantization_scheme="symmetric",
+            activation_dtype=torch.quint8,
+            weight_dtype=torch.qint8,
+            weight_per_channel=True,
+        )
+    )
+
+
+class Quantize(BaseStages.Quantize):
+    def __init__(
+        self,
+        quantizer: Optional[CoreMLQuantizer] = None,
+        quantization_config: Optional[Any] = None,
+        calibrate: bool = True,
+        calibration_samples: Optional[Sequence[Any]] = None,
+        is_qat: Optional[bool] = False,
+    ):
+        super().__init__(
+            quantizer=quantizer or CoreMLQuantizer(quantization_config or _get_static_int8_qconfig()),
+            calibrate=calibrate,
+            calibration_samples=calibration_samples,
+            is_qat=is_qat,
+        )
+
+
+
 class Partition(BaseStages.Partition):
-    def __init__(self, partitioner: Optional[Partitioner] = None):
+    def __init__(
+        self, 
+        partitioner: Optional[Partitioner] = None,
+        minimum_deployment_target: Optional[Any] = ct.target.iOS15,
+    ):
         super().__init__(
-            partitioner=partitioner or CoreMLPartitioner,
+            partitioner=partitioner or CoreMLPartitioner(
+                compile_specs=CoreMLBackend.generate_compile_specs(
+                    minimum_deployment_target=minimum_deployment_target
+                )
+            ),
         )
 
 
@@ -29,9 +70,14 @@ def __init__(
         self,
         partitioners: Optional[List[Partitioner]] = None,
         edge_compile_config: Optional[EdgeCompileConfig] = None,
+        minimum_deployment_target: Optional[Any] = ct.target.iOS15,
     ):
         super().__init__(
-            default_partitioner_cls=CoreMLPartitioner,
+            default_partitioner_cls=lambda: CoreMLPartitioner(
+               compile_specs=CoreMLBackend.generate_compile_specs(
+                    minimum_deployment_target=minimum_deployment_target
+                ) 
+            ),
             partitioners=partitioners,
             edge_compile_config=edge_compile_config,
         )
@@ -43,13 +89,15 @@ def __init__(
         module: torch.nn.Module,
         example_inputs: Tuple[torch.Tensor],
         dynamic_shapes: Optional[Tuple[Any]] = None,
+        minimum_deployment_target: Optional[Any] = ct.target.iOS15,
     ):
         # Specialize for XNNPACK
         stage_classes = (
             executorch.backends.test.harness.Tester.default_stage_classes()
             | {
-                StageType.PARTITION: Partition,
-                StageType.TO_EDGE_TRANSFORM_AND_LOWER: ToEdgeTransformAndLower,
+                StageType.QUANTIZE: Quantize,
+                StageType.PARTITION: functools.partial(Partition, minimum_deployment_target=minimum_deployment_target),
+                StageType.TO_EDGE_TRANSFORM_AND_LOWER: functools.partial(ToEdgeTransformAndLower, minimum_deployment_target=minimum_deployment_target),
             }
         )
 

backends/qualcomm/tests/TARGETS

@@ -37,3 +37,13 @@ python_library(
         "//executorch/backends/qualcomm/debugger:utils",
     ],
 )
+
+python_library(
+    name = "tester",
+    srcs = [
+        "tester.py",
+    ],
+    deps = [
+        ":test_qnn_delegate"
+    ]
+)

backends/qualcomm/tests/tester.py

@@ -0,0 +1,87 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+from typing import Any, List, Optional, Tuple
+
+import executorch
+import executorch.backends.test.harness.stages as BaseStages
+
+import torch
+from executorch.backends.qualcomm._passes.qnn_pass_manager import QnnPassManager
+from executorch.backends.qualcomm.partition.qnn_partitioner import QnnPartitioner
+from executorch.backends.qualcomm.utils.utils import (
+    generate_qnn_executorch_compiler_spec,
+    generate_htp_compiler_spec,
+    get_soc_to_chipset_map,
+)
+from executorch.backends.test.harness import Tester as TesterBase
+from executorch.backends.test.harness.stages import StageType
+from executorch.exir import EdgeCompileConfig, to_edge_transform_and_lower
+from executorch.exir.backend.partitioner import Partitioner
+from torch.export import ExportedProgram
+
+
+class Partition(BaseStages.Partition):
+    def __init__(self, partitioner: Optional[Partitioner] = None):
+        super().__init__(
+            partitioner=partitioner or QnnPartitioner,
+        )
+
+
+class ToEdgeTransformAndLower(BaseStages.ToEdgeTransformAndLower):
+    def __init__(
+        self,
+        partitioners: Optional[List[Partitioner]] = None,
+        edge_compile_config: Optional[EdgeCompileConfig] = None,
+        soc_model: str = "SM8650"
+    ):
+        backend_options = generate_htp_compiler_spec(use_fp16=True)
+        self.chipset = get_soc_to_chipset_map()[soc_model]
+        self.compiler_specs = generate_qnn_executorch_compiler_spec(
+            soc_model=self.chipset,
+            backend_options=backend_options,
+        )
+
+        super().__init__(
+            partitioners=partitioners or [QnnPartitioner(self.compiler_specs)],
+            edge_compile_config=edge_compile_config or EdgeCompileConfig(_check_ir_validity=False),
+            default_partitioner_cls=QnnPartitioner,
+        )
+
+    def run(self, artifact: ExportedProgram, inputs=None) -> None:
+        ep = QnnPassManager().transform_for_export_pipeline(artifact)
+        transform_passes = QnnPassManager().get_to_edge_transform_passes(ep)
+
+        self.edge_dialect_program = to_edge_transform_and_lower(
+            ep,
+            transform_passes=transform_passes,
+            partitioner=self.partitioners,
+            compile_config=self.edge_compile_conf,
+        )
+
+
+class QualcommTester(TesterBase):
+    def __init__(
+        self,
+        module: torch.nn.Module,
+        example_inputs: Tuple[torch.Tensor],
+        dynamic_shapes: Optional[Tuple[Any]] = None,
+    ): 
+        # Specialize for Qualcomm
+        stage_classes = (
+            executorch.backends.test.harness.Tester.default_stage_classes()
+            | {
+                StageType.PARTITION: Partition,
+                StageType.TO_EDGE_TRANSFORM_AND_LOWER: ToEdgeTransformAndLower,
+            }
+        )
+
+        super().__init__(
+            module=module,
+            stage_classes=stage_classes,
+            example_inputs=example_inputs,
+            dynamic_shapes=dynamic_shapes,
+        )

backends/test/harness/stages/quantize.py

@@ -31,7 +31,8 @@ def __init__(
         self.calibrate = calibrate
         self.calibration_samples = calibration_samples
 
-        self.quantizer.set_global(self.quantization_config)
+        if self.quantization_config is not None:
+            self.quantizer.set_global(self.quantization_config)
 
         self.converted_graph = None
         self.is_qat = is_qat

backends/test/harness/tester.py

@@ -1,6 +1,6 @@
 import random
 from collections import Counter, OrderedDict
-from typing import Any, Dict, List, Optional, Tuple, Type
+from typing import Any, Callable, Dict, List, Optional, Tuple
 
 import torch
 
@@ -33,7 +33,7 @@ def __init__(
         self,
         module: torch.nn.Module,
         example_inputs: Tuple[torch.Tensor],
-        stage_classes: Dict[StageType, Type],
+        stage_classes: Dict[StageType, Callable],
         dynamic_shapes: Optional[Tuple[Any]] = None,
     ):
         module.eval()
@@ -81,7 +81,7 @@ def __init__(
         self.stage_output = None
 
     @staticmethod
-    def default_stage_classes() -> Dict[StageType, Type]:
+    def default_stage_classes() -> Dict[StageType, Callable]:
         """
         Returns a map of StageType to default Stage implementation.
         """