|
| 1 | +# Copyright (c) Meta Platforms, Inc. and affiliates. |
| 2 | +# All rights reserved. |
| 3 | +# |
| 4 | +# This source code is licensed under the BSD-style license found in the |
| 5 | +# LICENSE file in the root directory of this source tree. |
| 6 | + |
| 7 | +# pyre-unsafe |
| 8 | + |
| 9 | +# |
| 10 | +# This file contains logic to run generated operator tests using the FACTO |
| 11 | +# library (https://github.com/pytorch-labs/FACTO). To run the tests, first |
| 12 | +# clone and install FACTO by running pip install . from the FACTO source |
| 13 | +# directory. Then, from the executorch root directory, run the following: |
| 14 | +# |
| 15 | +# python -m unittest backends.test.operators.test_facto.FactoTestsXNNPACK |
| 16 | +# |
| 17 | + |
| 18 | +import copy |
| 19 | +import functools |
| 20 | +import traceback |
| 21 | +import unittest |
| 22 | +from typing import Any, Callable, Sequence |
| 23 | + |
| 24 | +import torch |
| 25 | +from executorch.backends.test.harness.tester import Tester as TesterBase |
| 26 | +from executorch.backends.xnnpack.test.tester.tester import Tester as XnnpackTester |
| 27 | +from facto.inputgen.argtuple.gen import ArgumentTupleGenerator |
| 28 | +from facto.inputgen.specs.model import ConstraintProducer as cp, Spec |
| 29 | +from facto.inputgen.utils.random_manager import random_manager |
| 30 | +from facto.specdb.db import SpecDictDB |
| 31 | +from torch._ops import OpOverload |
| 32 | + |
| 33 | +from .facto_specs import ExtraSpecDB |
| 34 | + |
| 35 | +CombinedSpecDB = SpecDictDB | ExtraSpecDB |
| 36 | + |
| 37 | +COMMON_TENSOR_CONSTRAINTS = [ |
| 38 | + cp.Rank.Ge(lambda deps: 1), # Avoid zero and high rank tensors. |
| 39 | + cp.Rank.Le(lambda deps: 4), |
| 40 | + cp.Size.Ge(lambda deps, r, d: 1), # Keep sizes reasonable. |
| 41 | + cp.Size.Le(lambda deps, r, d: 2**9), |
| 42 | +] |
| 43 | + |
| 44 | +COMMON_SCALAR_CONSTRAINS = [ |
| 45 | + cp.Value.Ge(lambda deps, dtype: -1000), |
| 46 | + cp.Value.Le(lambda deps, dtype: 1000), |
| 47 | +] |
| 48 | + |
| 49 | +# Operator args are treated as runtime graph inputs if the argument name is |
| 50 | +# in this list. |
| 51 | +RUNTIME_INPUT_NAMES = { |
| 52 | + "self", |
| 53 | + "tensor", |
| 54 | + "other", |
| 55 | +} |
| 56 | + |
| 57 | + |
| 58 | +def _patch_spec(spec: Spec) -> Spec: |
| 59 | + spec = copy.deepcopy(spec) |
| 60 | + for inspec in spec.inspec: |
| 61 | + if inspec.type.is_tensor(): |
| 62 | + inspec.constraints.extend(COMMON_TENSOR_CONSTRAINTS) |
| 63 | + elif inspec.type.is_scalar(): |
| 64 | + inspec.constraints.extend(COMMON_SCALAR_CONSTRAINS) |
| 65 | + return spec |
| 66 | + |
| 67 | + |
| 68 | +class OpModel(torch.nn.Module): |
| 69 | + """ |
| 70 | + Wraps a single torch operator in an nn.Module. |
| 71 | + """ |
| 72 | + |
| 73 | + def __init__( |
| 74 | + self, |
| 75 | + op: OpOverload, |
| 76 | + runtime_input_count: int, |
| 77 | + fixed_args: Sequence[Any], |
| 78 | + fixed_kwargs: dict[str, Any], |
| 79 | + ): |
| 80 | + super().__init__() |
| 81 | + self.op = op |
| 82 | + self.runtime_input_count = runtime_input_count |
| 83 | + self.fixed_kwargs = fixed_kwargs |
| 84 | + |
| 85 | + # Register parameters for fixed tensors. Some things will choke on |
| 86 | + # constant tensor weights, for example. |
| 87 | + new_args = [] |
| 88 | + for i, arg in enumerate(fixed_args): |
| 89 | + if isinstance(arg, torch.Tensor): |
| 90 | + param = torch.nn.Parameter(arg, requires_grad=False) |
| 91 | + param_name = f"arg_{i}_param" |
| 92 | + setattr(self, param_name, param) |
| 93 | + self.register_parameter(param_name, param) |
| 94 | + new_args.append(param) |
| 95 | + else: |
| 96 | + new_args.append(arg) |
| 97 | + self.fixed_args = tuple(new_args) |
| 98 | + |
| 99 | + def forward(self, *args, **kwargs): |
| 100 | + return self.op(*(args + self.fixed_args), **(kwargs | self.fixed_kwargs)) |
| 101 | + |
| 102 | + |
| 103 | +class ConvModel(OpModel): |
| 104 | + def forward(self, *args, **kwargs): |
| 105 | + weight, bias, stride, padding, dilation, transposed, output_padding, groups = ( |
| 106 | + self.fixed_args |
| 107 | + ) |
| 108 | + |
| 109 | + if not transposed: |
| 110 | + if len(weight.shape) == 3: |
| 111 | + op = torch.nn.functional.conv1d |
| 112 | + elif len(weight.shape) == 4: |
| 113 | + op = torch.nn.functional.conv2d |
| 114 | + elif len(weight.shape) == 5: |
| 115 | + op = torch.nn.functional.conv3d |
| 116 | + |
| 117 | + return op(args[0], weight, bias, stride, padding, dilation, groups) |
| 118 | + else: |
| 119 | + if len(weight.shape) == 3: |
| 120 | + op = torch.nn.functional.conv_transpose1d |
| 121 | + elif len(weight.shape) == 4: |
| 122 | + op = torch.nn.functional.conv_transpose2d |
| 123 | + elif len(weight.shape) == 5: |
| 124 | + op = torch.nn.functional.conv_transpose3d |
| 125 | + |
| 126 | + return op( |
| 127 | + args[0], weight, bias, stride, padding, output_padding, groups, dilation |
| 128 | + ) |
| 129 | + |
| 130 | + |
| 131 | +def get_module_for_op(op: OpOverload): |
| 132 | + if op == torch.ops.aten.convolution.default: |
| 133 | + return ConvModel |
| 134 | + else: |
| 135 | + return OpModel |
| 136 | + |
| 137 | + |
| 138 | +class FactoTestsBase(unittest.TestCase): |
| 139 | + def __init__(self, tester_factory: Callable[[], TesterBase], *args, **kwargs): |
| 140 | + super().__init__(*args, **kwargs) |
| 141 | + self._tester_factory = tester_factory |
| 142 | + |
| 143 | + @staticmethod |
| 144 | + def _generate_test(op_name: str) -> None: |
| 145 | + # Find the torch op with the given name. |
| 146 | + sections = op_name.split(".") |
| 147 | + torch_op = functools.reduce(getattr, sections, torch.ops.aten) |
| 148 | + |
| 149 | + test_name = "test_" + op_name.replace(".", "_") |
| 150 | + |
| 151 | + def test_body(self): |
| 152 | + self._test_op(torch_op) |
| 153 | + |
| 154 | + setattr(FactoTestsBase, test_name, test_body) |
| 155 | + |
| 156 | + @staticmethod |
| 157 | + def get_runtime_input_count(spec: Spec): |
| 158 | + # Determine which inputs are fixed at tracing time (weights, for example), |
| 159 | + # vs inputs to the runtime graph. We currently assume that the runtime graph |
| 160 | + # inputs start at the beginning of the arg list and are contiguous. |
| 161 | + # |
| 162 | + # Args are consider to be runtime inputs if they are positional and are named |
| 163 | + # one of RUNTIME_INPUT_NAMES. If none match, we assume only the first arg is a |
| 164 | + # runtime input. |
| 165 | + runtime_input_count = 0 |
| 166 | + for inspec in spec.inspec: |
| 167 | + is_runtime_input = ( |
| 168 | + inspec.type.is_tensor() and inspec.name.lower() in RUNTIME_INPUT_NAMES |
| 169 | + ) |
| 170 | + if is_runtime_input: |
| 171 | + runtime_input_count += 1 |
| 172 | + else: |
| 173 | + break |
| 174 | + |
| 175 | + return max(1, runtime_input_count) |
| 176 | + |
| 177 | + def setUp(self): |
| 178 | + torch.set_printoptions(threshold=3) |
| 179 | + |
| 180 | + def _test_op(self, op: OpOverload) -> None: # noqa: C901 |
| 181 | + random_manager.seed(0) |
| 182 | + |
| 183 | + # Strip namespace |
| 184 | + op_name = op.name().split("::")[-1] |
| 185 | + |
| 186 | + # Default to .default overload |
| 187 | + if "." not in op_name: |
| 188 | + op_name += ".default" |
| 189 | + |
| 190 | + # Find and patch op spec |
| 191 | + if op_name not in CombinedSpecDB: |
| 192 | + raise ValueError(f"Operator {op_name} not found in SpecDictDB.") |
| 193 | + spec = _patch_spec(CombinedSpecDB[op_name]) |
| 194 | + |
| 195 | + runtime_input_count = FactoTestsBase.get_runtime_input_count(spec) |
| 196 | + |
| 197 | + print(f"Op: {op_name}, {runtime_input_count} runtime inputs") |
| 198 | + |
| 199 | + # Run test cases |
| 200 | + success_count_delegated = 0 |
| 201 | + success_count_undelegated = 0 |
| 202 | + fail_count = 0 |
| 203 | + |
| 204 | + i = 0 |
| 205 | + for posargs, inkwargs, _ in ArgumentTupleGenerator(spec).gen(): |
| 206 | + i += 1 |
| 207 | + |
| 208 | + try: |
| 209 | + if isinstance(posargs[0], torch.Tensor): |
| 210 | + # Temporary for getting around XNN crashes |
| 211 | + if posargs[0].dtype not in {torch.float32, torch.float16}: |
| 212 | + print("SKIPPING NON FLOAT CASE") |
| 213 | + continue |
| 214 | + |
| 215 | + module_cls = get_module_for_op(op) |
| 216 | + model = module_cls( |
| 217 | + op, runtime_input_count, posargs[runtime_input_count:], inkwargs |
| 218 | + ) |
| 219 | + |
| 220 | + # Sanity check to make sure it runs in eager. This can present nicer error |
| 221 | + # messages sometimes compared to tracing. |
| 222 | + try: |
| 223 | + model(*posargs[:runtime_input_count]) |
| 224 | + except Exception as e: |
| 225 | + print(f"Eager execution failed: {e}") |
| 226 | + continue |
| 227 | + |
| 228 | + tester = ( |
| 229 | + self._tester_factory(model, tuple(posargs[:runtime_input_count])) |
| 230 | + .export() |
| 231 | + .dump_artifact() |
| 232 | + .to_edge_transform_and_lower() |
| 233 | + ) |
| 234 | + |
| 235 | + is_delegated = any( |
| 236 | + n.target == torch._higher_order_ops.executorch_call_delegate |
| 237 | + for n in tester.stages[tester.cur].graph_module.graph.nodes |
| 238 | + if n.op == "call_function" |
| 239 | + ) |
| 240 | + |
| 241 | + # Only run the runtime test if the op was delegated. |
| 242 | + if is_delegated: |
| 243 | + ( |
| 244 | + tester.to_executorch() |
| 245 | + .serialize() |
| 246 | + .run_method_and_compare_outputs() |
| 247 | + ) |
| 248 | + |
| 249 | + if is_delegated: |
| 250 | + success_count_delegated += 1 |
| 251 | + else: |
| 252 | + success_count_undelegated += 1 |
| 253 | + except Exception as e: |
| 254 | + fail_count += 1 |
| 255 | + print(f"Error: {e}") |
| 256 | + print("Args:") |
| 257 | + for arg in posargs: |
| 258 | + if isinstance(arg, torch.Tensor): |
| 259 | + print(f" {arg.dtype} {arg.shape}") |
| 260 | + else: |
| 261 | + print(f" {arg}") |
| 262 | + |
| 263 | + traceback.print_exc() |
| 264 | + |
| 265 | + print( |
| 266 | + f"{success_count_delegated + success_count_undelegated} PASS, {fail_count} FAIL" |
| 267 | + ) |
| 268 | + print( |
| 269 | + f" {success_count_delegated} DELEGATED, {success_count_undelegated} UNDELEGATED" |
| 270 | + ) |
| 271 | + |
| 272 | + |
| 273 | +# Programatically generate tests for each operator. |
| 274 | +for op_name in CombinedSpecDB.keys(): |
| 275 | + FactoTestsBase._generate_test(op_name) |
| 276 | + |
| 277 | + |
| 278 | +# TODO Figure out where to put these |
| 279 | +class FactoTestsXNNPACK(FactoTestsBase): |
| 280 | + def __init__(self, *args, **kwargs): |
| 281 | + super().__init__(XnnpackTester, *args, **kwargs) |
0 commit comments