Update

[ghstack-poisoned]

Author: GregoryComer

backends/qualcomm/tests/tester.py

@@ -13,8 +13,8 @@
 from executorch.backends.qualcomm._passes.qnn_pass_manager import QnnPassManager
 from executorch.backends.qualcomm.partition.qnn_partitioner import QnnPartitioner
 from executorch.backends.qualcomm.utils.utils import (
-    generate_qnn_executorch_compiler_spec,
     generate_htp_compiler_spec,
+    generate_qnn_executorch_compiler_spec,
     get_soc_to_chipset_map,
 )
 from executorch.backends.test.harness import Tester as TesterBase
@@ -36,7 +36,7 @@ def __init__(
         self,
         partitioners: Optional[List[Partitioner]] = None,
         edge_compile_config: Optional[EdgeCompileConfig] = None,
-        soc_model: str = "SM8650"
+        soc_model: str = "SM8650",
     ):
         backend_options = generate_htp_compiler_spec(use_fp16=True)
         self.chipset = get_soc_to_chipset_map()[soc_model]
@@ -47,7 +47,8 @@ def __init__(
 
         super().__init__(
             partitioners=partitioners or [QnnPartitioner(self.compiler_specs)],
-            edge_compile_config=edge_compile_config or EdgeCompileConfig(_check_ir_validity=False),
+            edge_compile_config=edge_compile_config
+            or EdgeCompileConfig(_check_ir_validity=False),
             default_partitioner_cls=QnnPartitioner,
         )
 
@@ -69,7 +70,7 @@ def __init__(
         module: torch.nn.Module,
         example_inputs: Tuple[torch.Tensor],
         dynamic_shapes: Optional[Tuple[Any]] = None,
-    ): 
+    ):
         # Specialize for Qualcomm
         stage_classes = (
             executorch.backends.test.harness.Tester.default_stage_classes()

backends/test/harness/error_statistics.py

@@ -1,15 +1,16 @@
 from dataclasses import dataclass
-from torch.ao.ns.fx.utils import compute_sqnr
 
 import torch
+from torch.ao.ns.fx.utils import compute_sqnr
+
 
 @dataclass
 class TensorStatistics:
-    """ Contains summary statistics for a tensor. """
+    """Contains summary statistics for a tensor."""
 
     shape: torch.Size
     """ The shape of the tensor. """
-    
+
     numel: int
     """ The number of elements in the tensor. """
 
@@ -24,10 +25,10 @@ class TensorStatistics:
 
     min: torch.types.Number
     """ The minimum element of the tensor. """
-    
+
     @classmethod
     def from_tensor(cls, tensor: torch.Tensor) -> "TensorStatistics":
-        """ Creates a TensorStatistics object from a tensor. """
+        """Creates a TensorStatistics object from a tensor."""
         flattened = torch.flatten(tensor)
         return cls(
             shape=tensor.shape,
@@ -38,41 +39,44 @@ def from_tensor(cls, tensor: torch.Tensor) -> "TensorStatistics":
             min=flattened.min().item(),
         )
 
+
 @dataclass
 class ErrorStatistics:
-    """ Contains statistics derived from the difference of two tensors. """
+    """Contains statistics derived from the difference of two tensors."""
 
-    reference_stats: TensorStatistics   
+    reference_stats: TensorStatistics
     """ Statistics for the reference tensor. """
 
     actual_stats: TensorStatistics
     """ Statistics for the actual tensor. """
-    
+
     error_l2_norm: float | None
     """ The L2 norm of the error between the actual and reference tensor. """
-    
+
     error_mae: float | None
     """ The mean absolute error between the actual and reference tensor. """
-    
+
     error_max: float | None
     """ The maximum absolute elementwise error between the actual and reference tensor. """
-    
+
     error_msd: float | None
     """ The mean signed deviation between the actual and reference tensor. """
-    
+
     sqnr: float | None
     """ The signal-to-quantization-noise ratio between the actual and reference tensor. """
 
     @classmethod
-    def from_tensors(cls, actual: torch.Tensor, reference: torch.Tensor) -> "ErrorStatistics":
-        """ Creates an ErrorStatistics object from two tensors. """
+    def from_tensors(
+        cls, actual: torch.Tensor, reference: torch.Tensor
+    ) -> "ErrorStatistics":
+        """Creates an ErrorStatistics object from two tensors."""
         if actual.shape != reference.shape:
             return cls(
                 reference_stats=TensorStatistics.from_tensor(reference),
                 actual_stats=TensorStatistics.from_tensor(actual),
                 error_l2_norm=None,
                 error_mae=None,
-                error_max = None,
+                error_max=None,
                 error_msd=None,
                 sqnr=None,
             )
@@ -87,5 +91,5 @@ def from_tensors(cls, actual: torch.Tensor, reference: torch.Tensor) -> "ErrorSt
             error_mae=torch.mean(torch.abs(flat_error)).item(),
             error_max=torch.max(torch.abs(flat_error)).item(),
             error_msd=torch.mean(flat_error).item(),
-            sqnr=compute_sqnr(actual, reference).item()
+            sqnr=compute_sqnr(actual, reference).item(),
         )

backends/test/harness/tester.py

@@ -324,16 +324,22 @@ def run_method_and_compare_outputs(
             # Output from running artifact at stage
             stage_output = self.stages[stage].run_artifact(inputs_to_run)
             self._compare_outputs(
-                reference_output, stage_output, quantization_scale, atol, rtol, qtol, statistics_callback
+                reference_output,
+                stage_output,
+                quantization_scale,
+                atol,
+                rtol,
+                qtol,
+                statistics_callback,
             )
 
         return self
 
     @staticmethod
     def _assert_outputs_equal(
-        model_output, 
-        ref_output, 
-        atol=1e-03, 
+        model_output,
+        ref_output,
+        atol=1e-03,
         rtol=1e-03,
         statistics_callback: Callable[[ErrorStatistics], None] | None = None,
     ):
@@ -351,7 +357,7 @@ def _assert_outputs_equal(
         for i in range(len(model_output)):
             model = model_output[i]
             ref = ref_output[i]
-            
+
             error_stats = ErrorStatistics.from_tensors(model, ref)
             if statistics_callback is not None:
                 statistics_callback(error_stats)
@@ -410,7 +416,7 @@ def _compare_outputs(
         # atol by qtol quant units.
         if quantization_scale is not None:
             atol += quantization_scale * qtol
-            
+
         Tester._assert_outputs_equal(
             stage_output,
             reference_output,

backends/test/suite/context.py

@@ -1,7 +1,9 @@
 # Test run context management. This is used to determine the test context for reporting
 # purposes.
 class TestContext:
-    def __init__(self, test_name: str, test_base_name: str, flow_name: str, params: dict | None):
+    def __init__(
+        self, test_name: str, test_base_name: str, flow_name: str, params: dict | None
+    ):
         self.test_name = test_name
         self.test_base_name = test_base_name
         self.flow_name = flow_name

backends/test/suite/flow.py

@@ -64,6 +64,7 @@ def all_flows() -> dict[str, TestFlow]:
 
     try:
         from executorch.backends.test.suite.flows.vulkan import VULKAN_TEST_FLOW
+
         flows += [
             VULKAN_TEST_FLOW,
         ]
@@ -72,6 +73,7 @@ def all_flows() -> dict[str, TestFlow]:
 
     try:
         from executorch.backends.test.suite.flows.qualcomm import QUALCOMM_TEST_FLOW
+
         flows += [
             QUALCOMM_TEST_FLOW,
         ]

backends/test/suite/flows/qualcomm.py

@@ -1,9 +1,10 @@
 from executorch.backends.qualcomm.tests.tester import QualcommTester
 from executorch.backends.test.suite.flow import TestFlow
 
+
 def _create_qualcomm_flow(
-    name: str, 
-    quantize: bool = False, 
+    name: str,
+    quantize: bool = False,
 ) -> TestFlow:
     return TestFlow(
         name,
@@ -12,4 +13,5 @@ def _create_qualcomm_flow(
         quantize=quantize,
     )
 
+
 QUALCOMM_TEST_FLOW = _create_qualcomm_flow("qualcomm")

backends/test/suite/flows/vulkan.py

@@ -1,9 +1,10 @@
-from executorch.backends.vulkan.test.tester import VulkanTester
 from executorch.backends.test.suite.flow import TestFlow
+from executorch.backends.vulkan.test.tester import VulkanTester
+
 
 def _create_vulkan_flow(
-    name: str, 
-    quantize: bool = False, 
+    name: str,
+    quantize: bool = False,
 ) -> TestFlow:
     return TestFlow(
         name,
@@ -12,4 +13,5 @@ def _create_vulkan_flow(
         quantize=quantize,
     )
 
+
 VULKAN_TEST_FLOW = _create_vulkan_flow("vulkan")

backends/test/suite/operators/__init__.py

@@ -117,7 +117,9 @@ def _create_test_for_backend(
 
     if test_type == TestType.STANDARD:
         test_name = f"{test_func.__name__}_{flow.name}"
-        wrapped_test = _make_wrapped_test(test_func, test_name, test_func.__name__, flow)
+        wrapped_test = _make_wrapped_test(
+            test_func, test_name, test_func.__name__, flow
+        )
         setattr(cls, test_name, wrapped_test)
     elif test_type == TestType.DTYPE:
         for dtype in DTYPES:

backends/test/suite/reporting.py

@@ -1,13 +1,13 @@
+import csv
 from collections import Counter
 from dataclasses import dataclass
 from enum import IntEnum
 from functools import reduce
 from typing import TextIO
 
-import csv
-
 from executorch.backends.test.harness.error_statistics import ErrorStatistics
 
+
 class TestResult(IntEnum):
     """Represents the result of a test case run, indicating success or a specific failure reason."""
 
@@ -80,13 +80,13 @@ class TestCaseSummary:
     """
     Contains summary results for the execution of a single test case.
     """
-    
+
     backend: str
     """ The name of the target backend. """
 
     base_name: str
     """ The base name of the test, not including flow or parameter suffixes. """
-    
+
     flow: str
     """ The backend-specific flow name. Corresponds to flows registered in backends/test/suite/__init__.py. """
 
@@ -101,7 +101,7 @@ class TestCaseSummary:
 
     error: Exception | None
     """ The Python exception object, if any. """
-    
+
     tensor_error_statistics: list[ErrorStatistics]
     """ 
     Statistics about the error between the backend and reference outputs. Each element of this list corresponds to
@@ -180,8 +180,9 @@ def complete_test_session() -> RunSummary:
 
     return summary
 
+
 def generate_csv_report(summary: RunSummary, output: TextIO):
-    """ Write a run summary report to a file in CSV format. """
+    """Write a run summary report to a file in CSV format."""
 
     field_names = [
         "Test ID",
@@ -190,30 +191,38 @@ def generate_csv_report(summary: RunSummary, output: TextIO):
         "Flow",
         "Result",
     ]
-    
+
     # Tests can have custom parameters. We'll want to report them here, so we need
     # a list of all unique parameter names.
     param_names = reduce(
         lambda a, b: a.union(b),
-        (set(s.params.keys()) for s in summary.test_case_summaries if s.params is not None),
-        set()
+        (
+            set(s.params.keys())
+            for s in summary.test_case_summaries
+            if s.params is not None
+        ),
+        set(),
     )
     field_names += (s.capitalize() for s in param_names)
 
     # Add tensor error statistic field names for each output index.
-    max_outputs = max(len(s.tensor_error_statistics) for s in summary.test_case_summaries)
+    max_outputs = max(
+        len(s.tensor_error_statistics) for s in summary.test_case_summaries
+    )
     for i in range(max_outputs):
-        field_names.extend([
-            f"Output {i} Error Max",
-            f"Output {i} Error MAE",
-            f"Output {i} Error MSD",
-            f"Output {i} Error L2",
-            f"Output {i} SQNR",
-        ])
+        field_names.extend(
+            [
+                f"Output {i} Error Max",
+                f"Output {i} Error MAE",
+                f"Output {i} Error MSD",
+                f"Output {i} Error L2",
+                f"Output {i} SQNR",
+            ]
+        )
 
     writer = csv.DictWriter(output, field_names)
     writer.writeheader()
-    
+
     for record in summary.test_case_summaries:
         row = {
             "Test ID": record.name,
@@ -223,10 +232,8 @@ def generate_csv_report(summary: RunSummary, output: TextIO):
             "Result": record.result.display_name(),
         }
         if record.params is not None:
-            row.update({
-                k.capitalize(): v for k, v in record.params.items()
-            })
-            
+            row.update({k.capitalize(): v for k, v in record.params.items()})
+
         for output_idx, error_stats in enumerate(record.tensor_error_statistics):
             row[f"Output {output_idx} Error Max"] = error_stats.error_max
             row[f"Output {output_idx} Error MAE"] = error_stats.error_mae