From 98ab7f5606d71b12a41f7ee6d41f096ed0f6eb45 Mon Sep 17 00:00:00 2001 From: Abhinay Kukkadapu Date: Thu, 18 Sep 2025 13:43:48 -0700 Subject: [PATCH] Add android target recipes and extensive model tests using ios and android recipes (#14290) Summary: Changes in this diff: 1. Adds android target recipes using QNN backend. 2. Add extensive model lowering and accuracy checks using torchvision models. ``` from executorch.export import export qnn_android_recipe = get_android_recipe() # lower a model to pte session = export(model, recipe, example_inputs) out = session.run_method("forward", *example_inputs) ``` Differential Revision: D82284987 --- .ci/scripts/test_wheel_package_qnn.sh | 1 + backends/qualcomm/_passes/TARGETS | 1 + export/TARGETS | 10 + export/target_recipes.py | 120 ++++-- export/tests/TARGETS | 13 + export/tests/test_target_recipes.py | 513 ++++++++++++++++++++++---- export/utils.py | 51 +++ 7 files changed, 607 insertions(+), 102 deletions(-) create mode 100644 export/utils.py diff --git a/.ci/scripts/test_wheel_package_qnn.sh b/.ci/scripts/test_wheel_package_qnn.sh index 39c52a4a396..4a50b8e2c36 100644 --- a/.ci/scripts/test_wheel_package_qnn.sh +++ b/.ci/scripts/test_wheel_package_qnn.sh @@ -145,6 +145,7 @@ run_core_tests () { echo "=== [$LABEL] Import smoke tests ===" "$PYBIN" -c "import executorch; print('executorch imported successfully')" "$PYBIN" -c "import executorch.backends.qualcomm; print('executorch.backends.qualcomm imported successfully')" + "$PYBIN" -c "from executorch.export.target_recipes import get_android_recipe; recipe = get_android_recipe('android-arm64-snapdragon-fp16'); print(f'executorch.export.target_recipes imported successfully: {recipe}')" echo "=== [$LABEL] List installed executorch/backends/qualcomm/python ===" local SITE_DIR diff --git a/backends/qualcomm/_passes/TARGETS b/backends/qualcomm/_passes/TARGETS index 62a0fc43a78..876b51d3863 100644 --- a/backends/qualcomm/_passes/TARGETS +++ b/backends/qualcomm/_passes/TARGETS @@ -15,5 +15,6 @@ runtime.python_library( "//executorch/backends/transforms:decompose_sdpa", "//executorch/exir/backend:backend_details", "//executorch/exir/backend:compile_spec_schema", + "//executorch/backends/qualcomm/quantizer:quantizer", ], ) diff --git a/export/TARGETS b/export/TARGETS index ae41393d883..50afa6db6ed 100644 --- a/export/TARGETS +++ b/export/TARGETS @@ -117,9 +117,19 @@ runtime.python_library( "target_recipes.py", ], deps = [ + ":export_utils", "fbsource//third-party/pypi/coremltools:coremltools", "//executorch/export:recipe", "//executorch/backends/xnnpack/recipes:xnnpack_recipes", "//executorch/backends/apple/coreml:coreml_recipes", + "//executorch/backends/qualcomm/recipes:qnn_recipes", + ] +) + +runtime.python_library( + name = "export_utils", + srcs = ["utils.py"], + deps = [ + "//caffe2:torch", ] ) diff --git a/export/target_recipes.py b/export/target_recipes.py index 0a5ae9ce754..2d2eba46b0a 100644 --- a/export/target_recipes.py +++ b/export/target_recipes.py @@ -11,31 +11,14 @@ selection and combine multiple backends optimally for target hardware. """ -import sys +import os from typing import Dict, List -if sys.platform != "win32": - import coremltools as ct - from executorch.backends.apple.coreml.recipes import CoreMLRecipeType - -# pyre-ignore from executorch.backends.xnnpack.recipes import XNNPackRecipeType from executorch.export.recipe import ExportRecipe, RecipeType - - -## IOS Target configs -# The following list of recipes are not exhaustive for CoreML; refer to CoreMLRecipeType for more detailed recipes. -IOS_CONFIGS: Dict[str, List[RecipeType]] = ( - { - # pyre-ignore - "ios-arm64-coreml-fp32": [CoreMLRecipeType.FP32, XNNPackRecipeType.FP32], - # pyre-ignore - "ios-arm64-coreml-fp16": [CoreMLRecipeType.FP16], - # pyre-ignore - "ios-arm64-coreml-int8": [CoreMLRecipeType.PT2E_INT8_STATIC], - } - if sys.platform != "win32" - else {} +from executorch.export.utils import ( + is_supported_platform_for_coreml_lowering, + is_supported_platform_for_qnn_lowering, ) @@ -46,7 +29,7 @@ def _create_target_recipe( Create a combined recipe for a target. Args: - target: Human-readable hardware configuration name + target_config: Human-readable hardware configuration name recipes: List of backend recipe types to combine **kwargs: Additional parameters - each backend will use what it needs @@ -67,7 +50,6 @@ def _create_target_recipe( f"Failed to create {recipe_type.value} recipe for {target_config}: {e}" ) from e - # Combine into single recipe if len(backend_recipes) == 1: return backend_recipes[0] @@ -100,8 +82,24 @@ def get_ios_recipe( recipe = get_ios_recipe('ios-arm64-coreml-int8') session = export(model, recipe, example_inputs) """ - if target_config not in IOS_CONFIGS: - supported = list(IOS_CONFIGS.keys()) + + if not is_supported_platform_for_coreml_lowering(): + raise ValueError("CoreML is not supported on this platform") + + import coremltools as ct + from executorch.backends.apple.coreml.recipes import CoreMLRecipeType + + ios_configs: Dict[str, List[RecipeType]] = { + # pyre-ignore + "ios-arm64-coreml-fp32": [CoreMLRecipeType.FP32, XNNPackRecipeType.FP32], + # pyre-ignore + "ios-arm64-coreml-fp16": [CoreMLRecipeType.FP16], + # pyre-ignore + "ios-arm64-coreml-int8": [CoreMLRecipeType.PT2E_INT8_STATIC], + } + + if target_config not in ios_configs: + supported = list(ios_configs.keys()) raise ValueError( f"Unsupported iOS configuration: '{target_config}'. " f"Supported: {supported}" @@ -113,5 +111,75 @@ def get_ios_recipe( if "minimum_deployment_target" not in kwargs: kwargs["minimum_deployment_target"] = ct.target.iOS17 - backend_recipes = IOS_CONFIGS[target_config] + backend_recipes = ios_configs[target_config] + return _create_target_recipe(target_config, backend_recipes, **kwargs) + + +# Android Recipe +def get_android_recipe( + target_config: str = "android-arm64-snapdragon-fp16", **kwargs +) -> ExportRecipe: + """ + Get Android-optimized recipe for specified hardware configuration. + + Supported configurations: + - 'android-arm64-snapdragon-fp16': QNN fp16 recipe + + Args: + target_config: Android configuration string + **kwargs: Additional parameters for backend recipes + + Returns: + ExportRecipe configured for Android deployment + + Raises: + ValueError: If target configuration is not supported + + Example: + recipe = get_android_recipe('android-arm64-snapdragon-fp16') + session = export(model, recipe, example_inputs) + """ + + if not is_supported_platform_for_qnn_lowering(): + raise ValueError( + "QNN is not supported or not properly configured on this platform" + ) + + try: + # Qualcomm QNN backend runs QNN sdk download on first use + # with a pip install, so wrap it in a try/except + # pyre-ignore + from executorch.backends.qualcomm.recipes import QNNRecipeType + + # (1) if this is called from a pip install, the QNN SDK will be available + # (2) if this is called from a source build, check if qnn is available otherwise, had to run build.sh + if os.getenv("QNN_SDK_ROOT", None) is None: + raise ValueError( + "QNN SDK not found, cannot use QNN recipes. First run `./backends/qualcomm/scripts/build.sh`, if building from source" + ) + except Exception as e: + raise ValueError( + "QNN backend is not available. Please ensure the Qualcomm backend " + "is properly installed and configured, " + ) from e + + android_configs: Dict[str, List[RecipeType]] = { + # pyre-ignore + "android-arm64-snapdragon-fp16": [QNNRecipeType.FP16], + } + + if target_config not in android_configs: + supported = list(android_configs.keys()) + raise ValueError( + f"Unsupported Android configuration: '{target_config}'. " + f"Supported: {supported}" + ) + + kwargs = kwargs or {} + + if target_config == "android-arm64-snapdragon-fp16": + if "soc_model" not in kwargs: + kwargs["soc_model"] = "SM8650" + + backend_recipes = android_configs[target_config] return _create_target_recipe(target_config, backend_recipes, **kwargs) diff --git a/export/tests/TARGETS b/export/tests/TARGETS index 71f28b64df7..7b1578ce508 100644 --- a/export/tests/TARGETS +++ b/export/tests/TARGETS @@ -1,4 +1,5 @@ load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") +load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version") oncall("executorch") @@ -37,11 +38,23 @@ runtime.python_test( srcs = [ "test_target_recipes.py", ], + env = { + "LD_LIBRARY_PATH": "$(location fbsource//third-party/qualcomm/qnn/qnn-{0}:qnn_offline_compile_libs)".format(get_qnn_library_version()), + "QNN_SDK_ROOT": "$(location fbsource//third-party/qualcomm/qnn/qnn-{0}:__dir__)".format(get_qnn_library_version()), + "HTTP_PROXY": "http://fwdproxy:8080", + "HTTPS_PROXY": "http://fwdproxy:8080", + }, + labels = ["long_running"], deps = [ "//executorch/export:lib", "//executorch/export:target_recipes", + "//executorch/export:export_utils", "//executorch/runtime:runtime", "//executorch/backends/xnnpack/recipes:xnnpack_recipes", "//executorch/backends/apple/coreml:coreml_recipes", + "//executorch/backends/qualcomm/recipes:qnn_recipes", + "//executorch/examples/models:models", + "//executorch/backends/xnnpack/test/tester:tester", + "fbsource//third-party/pypi/coremltools:coremltools" ] ) diff --git a/export/tests/test_target_recipes.py b/export/tests/test_target_recipes.py index 7a2a7c87342..61725e58f3a 100644 --- a/export/tests/test_target_recipes.py +++ b/export/tests/test_target_recipes.py @@ -7,54 +7,182 @@ # pyre-strict import logging -import sys +import os import unittest +from typing import Any, Dict, List, Optional, Tuple import torch from executorch.backends.xnnpack.recipes.xnnpack_recipe_provider import ( XNNPACKRecipeProvider, ) -from executorch.export import export, recipe_registry -from executorch.export.target_recipes import get_ios_recipe +from executorch.backends.xnnpack.test.tester import Tester +from executorch.examples.models import MODEL_NAME_TO_MODEL +from executorch.examples.models.model_factory import EagerModelFactory +from executorch.exir.schema import DelegateCall, Program +from executorch.export import ( + export, + ExportRecipe, + ExportSession, + recipe_registry, + StageType, +) +from executorch.export.utils import ( + is_fbcode, + is_supported_platform_for_coreml_lowering, + is_supported_platform_for_qnn_lowering, +) from executorch.runtime import Runtime - -if sys.platform != "win32": - from executorch.backends.apple.coreml.recipes import ( # pyre-ignore - CoreMLRecipeProvider, - ) +from torch import nn, Tensor +from torch.testing import FileCheck +from torchao.quantization.utils import compute_error class TestTargetRecipes(unittest.TestCase): """Test target recipes.""" + class Model(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.linear1 = torch.nn.Linear(4, 4) + self.linear2 = torch.nn.Linear(4, 2) + + def forward(self, x: Tensor, y: Tensor) -> Tensor: + a = self.linear1(x) + b = a + y + c = b - x + result = self.linear2(c) + return result + def setUp(self) -> None: torch._dynamo.reset() super().setUp() recipe_registry.register_backend_recipe_provider(XNNPACKRecipeProvider()) - if sys.platform != "win32": + if is_supported_platform_for_coreml_lowering(): + from executorch.backends.apple.coreml.recipes import ( # pyre-ignore + CoreMLRecipeProvider, + ) + # pyre-ignore recipe_registry.register_backend_recipe_provider(CoreMLRecipeProvider()) + if is_fbcode() and is_supported_platform_for_qnn_lowering(): + from executorch.backends.qualcomm.recipes import ( # pyre-ignore + QNNRecipeProvider, + ) + + # pyre-ignore + recipe_registry.register_backend_recipe_provider(QNNRecipeProvider()) + self.model = TestTargetRecipes.Model() + def tearDown(self) -> None: super().tearDown() - @unittest.skipIf(sys.platform == "win32", "Core ML is not available on Windows.") + def check_delegated( + self, program: Program, expected_backends: Optional[List[str]] = None + ) -> None: + """Check if the program has been delegated to expected backends.""" + instructions = program.execution_plan[0].chains[0].instructions + assert instructions is not None + + if expected_backends is None: + # Just check that there's at least one delegate call + self.assertGreater(len(instructions), 0) + for instruction in instructions: + self.assertIsInstance(instruction.instr_args, DelegateCall) + else: + # Check for specific backends + delegates = program.execution_plan[0].delegates + delegate_ids = [delegate.id for delegate in delegates] + for expected_backend in expected_backends: + self.assertIn( + expected_backend, + delegate_ids, + f"Expected backend {expected_backend} not found in delegates: {delegate_ids}", + ) + + def check_num_partitions( + self, executorch_program: Program, expected_num_partitions: int + ) -> None: + """Check if the program has the expected number of partitions.""" + self.assertEqual( + len(executorch_program.execution_plan[0].delegates), + expected_num_partitions, + ) + + def _check_lowering_error( + self, + # pyre-ignore[11] + session: ExportSession, + example_inputs: List[Tuple[Tensor]], + model_name: str, + recipe_key: str, + atol: float = 1e-3, + rtol: float = 1e-3, + ) -> None: + """Compare original model output with session output using tolerance.""" + quantized_model = session.get_stage_artifacts()[StageType.QUANTIZE].data[ + "forward" + ] + lowered_output = session.run_method("forward", *example_inputs)[0] + quantized_output = quantized_model(*example_inputs[0]) + + try: + Tester._assert_outputs_equal( + lowered_output, quantized_output, atol=atol, rtol=rtol + ) + logging.info( + f"Tolerance check passed for {model_name} with atol={atol}, rtol={rtol}" + ) + except AssertionError as e: + raise AssertionError( + f"Model '{model_name}' Recipe: {recipe_key}, tolerance check failed" + ) from e + + def _check_quantization_error( + self, + session: ExportSession, + eager_model: nn.Module, + example_inputs: List[Tuple[Tensor]], + model_name: str, + recipe_key: str, + sqnr_threshold: float = 20.0, + ) -> None: + """Compare original model output with session output using SQNR.""" + eager_output = eager_model(*example_inputs[0]) + + # get quantized model from session + all_artifacts = session.get_stage_artifacts() + quantized_model = all_artifacts[StageType.QUANTIZE].data["forward"] + quantized_output = quantized_model(*example_inputs[0]) + + error = compute_error(eager_output, quantized_output) + logging.info(f"SQNR for {model_name}: {error} dB") + self.assertTrue( + error > sqnr_threshold, + f"Model {model_name}, recipe: {recipe_key} SQNR check failed. Expected > {sqnr_threshold}, got {error}", + ) + + def _check_delegation_with_filecheck(self, session: ExportSession) -> None: + """Check that the lowered module contains expected delegate calls.""" + all_artifacts = session.get_stage_artifacts() + edge_program_manager = all_artifacts[StageType.TO_EDGE_TRANSFORM_AND_LOWER].data + lowered_module = edge_program_manager.exported_program().module() + + # Check if model got lowered + FileCheck().check("torch.ops.higher_order.executorch_call_delegate").run( + lowered_module.code + ) + + # pyre-ignore + @unittest.skipIf( + not is_supported_platform_for_coreml_lowering(), + "Skip test, coreml lowering not supported", + ) def test_ios_fp32_recipe_with_xnnpack_fallback(self) -> None: + from executorch.export.target_recipes import get_ios_recipe + # Linear ops skipped by coreml but handled by xnnpack - class Model(torch.nn.Module): - def __init__(self): - super().__init__() - self.linear1 = torch.nn.Linear(4, 4) - self.linear2 = torch.nn.Linear(4, 2) - - def forward(self, x, y): - a = self.linear1(x) - b = a + y - c = b - x - result = self.linear2(c) - return result - - model = Model() + model = self.model model.eval() example_inputs = [(torch.randn(2, 4), torch.randn(2, 4))] @@ -114,65 +242,298 @@ def forward(self, x, y): et_output = session.run_method("forward", example_inputs[0]) logging.info(f"et output {et_output}") - @unittest.skipIf(sys.platform == "win32", "Core ML is not available on Windows.") - def test_ios_quant_recipes(self) -> None: - class Model(torch.nn.Module): - def __init__(self): - super().__init__() - self.linear1 = torch.nn.Linear(4, 4) - self.linear2 = torch.nn.Linear(4, 2) - - def forward(self, x, y): - a = self.linear1(x) - b = a + y - c = b - x - result = self.linear2(c) - return result - - model = Model() - model.eval() + def _test_model_with_target_recipes( + self, + model_name: str, + recipe: ExportRecipe, + expected_backend_name: str, + eager_model: nn.Module, + example_inputs: Tuple[Tensor], + recipe_key: str, + dynamic_shapes: Optional[Dict[str, Tuple[int, ...]]], + atol: Optional[float] = 1e-1, + rtol: Optional[float] = 1e-1, + sqnr_threshold: Optional[int] = 20, + ) -> None: + """Test a model with a specific target recipe and expected backend.""" + logging.info(f"Testing model {model_name} with {expected_backend_name} backend") + + # Export with the provided recipe + session = export( + model=eager_model, + example_inputs=[example_inputs], + export_recipe=recipe, + dynamic_shapes=dynamic_shapes, + ) + logging.info(f"Exporting done for {model_name}-{recipe_key}") - example_inputs = [(torch.randn(2, 4), torch.randn(2, 4))] + executorch_program = session.get_executorch_program() + self.assertIsNotNone( + executorch_program, + f"ExecuTorch program should not be None for {expected_backend_name}", + ) - for recipe in [ - get_ios_recipe("ios-arm64-coreml-fp16"), - get_ios_recipe("ios-arm64-coreml-int8"), - ]: - # Export the model - session = export( - model=model, example_inputs=example_inputs, export_recipe=recipe - ) + # Check delegation for the expected backend + self.check_delegated(executorch_program, [expected_backend_name]) - # Verify we can create executable - executorch_program = session.get_executorch_program() - session.print_delegation_info() + # Check number of partitions created + self.check_num_partitions(executorch_program, 1) - self.assertIsNotNone( - executorch_program, "ExecuTorch program should not be None" - ) + # Run the model if the backend is available + et_runtime: Runtime = Runtime.get() + backend_registry = et_runtime.backend_registry - # Assert there is an execution plan - self.assertTrue(len(executorch_program.execution_plan) == 1) + logging.info( + f"backends registered: {et_runtime.backend_registry.registered_backend_names}" + ) - # Check number of partitions created - self.assertTrue(len(executorch_program.execution_plan[0].delegates) == 1) + if backend_registry.is_available(expected_backend_name): + logging.info(f"Running with {expected_backend_name} backend") + if atol is not None and rtol is not None: + self._check_lowering_error( + session, + [example_inputs], + model_name, + recipe_key, + atol=atol, + rtol=rtol, + ) + logging.info( + f"Accuracy checks passed for {model_name} with {expected_backend_name} with atol={atol}, rtol={rtol}" + ) + + # Test SQNR if specified + if sqnr_threshold is not None: + self._check_quantization_error( + session, + eager_model, + [example_inputs], + model_name, + recipe_key, + sqnr_threshold=sqnr_threshold, + ) + + logging.info( + f"SQNR check passed for {model_name} with {expected_backend_name} with sqnr={sqnr_threshold}" + ) + + @classmethod + def _get_model_test_configs( + cls, + ) -> Dict[str, Dict[str, Tuple[Optional[float], Optional[float], Optional[int]]]]: + """Get model-specific test configurations for different recipes.""" + # Format: {model_name: {target_recipe_name: (atol, rtol, sqnr_threshold)}} + # If a model/recipe combination is present in this config, the model will be lowered for that recipe. + # A value of `None` for any of atol, rtol, or sqnr_threshold means the corresponding accuracy check will be skipped after lowering. + return { + "linear": { + "ios-arm64-coreml-fp16": (1e-3, 1e-3, 20), + "ios-arm64-coreml-int8": (1e-2, 1e-2, 20), + "android-arm64-snapdragon-fp16": (1e-3, 1e-3, None), + }, + "add": { + "ios-arm64-coreml-fp16": (1e-3, 1e-3, 20), + "ios-arm64-coreml-int8": (1e-3, 1e-3, 20), + "android-arm64-snapdragon-fp16": (1e-3, 1e-3, None), + }, + "add_mul": { + "ios-arm64-coreml-fp16": (1e-3, 1e-3, 20), + "ios-arm64-coreml-int8": (1e-3, 1e-3, 20), + "android-arm64-snapdragon-fp16": (1e-3, 1e-3, None), + }, + "ic3": { + "ios-arm64-coreml-fp16": (1e-1, 1.0, 20), + "ios-arm64-coreml-int8": (None, None, None), + "android-arm64-snapdragon-fp16": (5e-1, 1e-1, None), + }, + "ic4": { + "ios-arm64-coreml-fp16": (1e-1, 1e-1, 20), + "ios-arm64-coreml-int8": (None, None, None), + "android-arm64-snapdragon-fp16": (None, None, None), + }, + "mv2": { + "ios-arm64-coreml-fp16": (5e-2, 5e-2, 20), + "ios-arm64-coreml-int8": (2e-1, 2e-1, 20), + "android-arm64-snapdragon-fp16": (1e-2, 5e-2, None), + }, + "mv3": { + "ios-arm64-coreml-fp16": (2e-1, 2e-1, 20), + "ios-arm64-coreml-int8": (None, None, None), + "android-arm64-snapdragon-fp16": (None, None, None), + }, + "resnet18": { + "ios-arm64-coreml-fp16": (1e-1, 1e-1, 20), + "ios-arm64-coreml-int8": (None, None, None), + "android-arm64-snapdragon-fp16": (2e-1, 2e-1, None), + }, + "resnet50": { + "ios-arm64-coreml-fp16": (1e-2, 1e-2, 20), + "ios-arm64-coreml-int8": (None, None, None), + "android-arm64-snapdragon-fp16": (5e-1, 2e-1, None), + }, + "vit": { + "ios-arm64-coreml-fp16": (None, None, None), # only lower + "ios-arm64-coreml-int8": (None, None, None), # only lower + # Couldn't lower it to qnn + # "android-arm64-snapdragon-fp16": (None, None, None), + }, + "w2l": { + "ios-arm64-coreml-fp16": (1e-2, 1e-2, 20), + "ios-arm64-coreml-int8": (1e-1, 1e-1, 20), + "android-arm64-snapdragon-fp16": (1e-2, 1e-2, None), + }, + } + + @classmethod + def _get_recipes(cls) -> Dict[str, Tuple[ExportRecipe, str]]: + """Get available recipes with their configurations based on platform.""" + all_recipes = {} + + # Add iOS recipes + if is_supported_platform_for_coreml_lowering(): + from executorch.export.target_recipes import get_ios_recipe + + all_recipes = { + "ios-arm64-coreml-fp16": (get_ios_recipe(), "CoreMLBackend"), + "ios-arm64-coreml-int8": ( + get_ios_recipe("ios-arm64-coreml-int8"), + "CoreMLBackend", + ), + } + + # Add android recipes + if is_fbcode() and is_supported_platform_for_qnn_lowering(): + from executorch.export.target_recipes import get_android_recipe + + all_recipes["android-arm64-snapdragon-fp16"] = ( + get_android_recipe(), + "QnnBackend", + ) - # Delegate backend is CoreML - self.assertEqual( - executorch_program.execution_plan[0].delegates[0].id, - "CoreMLBackend", + return all_recipes + + def _run_model_with_recipe( + self, + model_name: str, + recipe_key: str, + eager_model: nn.Module, + example_inputs: Tuple[Tensor], + # pyre-ignore + dynamic_shapes: Any, + ) -> None: + model_configs = self._get_model_test_configs() + recipes = self._get_recipes() + + if model_name not in model_configs: + raise ValueError(f"Model {model_name} not found in test configurations") + + if recipe_key not in recipes: + raise ValueError(f"Recipe {recipe_key} not found in recipe configurations") + + recipe_tolerances = model_configs[model_name] + + if recipe_key not in recipe_tolerances: + raise ValueError(f"Model {model_name} does not support recipe {recipe_key}") + + atol, rtol, sqnr_threshold = recipe_tolerances[recipe_key] + recipe, expected_backend = recipes[recipe_key] + + with torch.no_grad(): + logging.info(f"Running model {model_name} with recipe {recipe_key}") + self._test_model_with_target_recipes( + model_name=model_name, + recipe=recipe, + expected_backend_name=expected_backend, + eager_model=eager_model, + example_inputs=example_inputs, + dynamic_shapes=dynamic_shapes, + recipe_key=recipe_key, + atol=atol, + rtol=rtol, + sqnr_threshold=sqnr_threshold, ) - # Check number of instructions - instructions = executorch_program.execution_plan[0].chains[0].instructions - self.assertIsNotNone(instructions) - self.assertEqual(len(instructions), 1) + def _run_model_with_all_recipes(self, model_name: str) -> None: + if model_name not in MODEL_NAME_TO_MODEL: + self.skipTest(f"Model {model_name} not found in MODEL_NAME_TO_MODEL") + return - et_runtime: Runtime = Runtime.get() - backend_registry = et_runtime.backend_registry - logging.info( - f"backends registered: {et_runtime.backend_registry.registered_backend_names}" - ) - if backend_registry.is_available("CoreMLBackend"): - et_output = session.run_method("forward", example_inputs[0]) - logging.info(f"et output {et_output}") + eager_model, example_inputs, _example_kwarg_inputs, dynamic_shapes = ( + EagerModelFactory.create_model(*MODEL_NAME_TO_MODEL[model_name]) + ) + eager_model = eager_model.eval() + + recipes = self._get_recipes() + model_configs = self._get_model_test_configs() + + try: + # Pre-filter recipes to only those supported by the model + supported_recipes = [] + for recipe_key in recipes.keys(): + if ( + model_name in model_configs + and recipe_key in model_configs[model_name] + ): + supported_recipes.append(recipe_key) + + if not supported_recipes: + self.skipTest(f"Model {model_name} has no supported recipes") + return + + for recipe_key in supported_recipes: + with self.subTest(recipe=recipe_key): + self._run_model_with_recipe( + model_name, + recipe_key, + eager_model, + example_inputs, + dynamic_shapes, + ) + finally: + # Clean up dog.jpg file if it exists + if os.path.exists("dog.jpg"): + os.remove("dog.jpg") + + def test_linear_model(self) -> None: + """Test linear model with all applicable recipes.""" + self._run_model_with_all_recipes("linear") + + def test_add_model(self) -> None: + """Test add model with all applicable recipes.""" + self._run_model_with_all_recipes("add") + + def test_add_mul_model(self) -> None: + """Test add_mul model with all applicable recipes.""" + self._run_model_with_all_recipes("add_mul") + + def test_ic3_model(self) -> None: + """Test ic3 model with all applicable recipes.""" + self._run_model_with_all_recipes("ic3") + + def test_ic4_model(self) -> None: + """Test ic4 model with all applicable recipes.""" + self._run_model_with_all_recipes("ic4") + + def test_mv2_model(self) -> None: + """Test mv2 model with all applicable recipes.""" + self._run_model_with_all_recipes("mv2") + + def test_mv3_model(self) -> None: + """Test mv3 model with all applicable recipes.""" + self._run_model_with_all_recipes("mv3") + + def test_resnet18_model(self) -> None: + """Test resnet18 model with all applicable recipes.""" + self._run_model_with_all_recipes("resnet18") + + def test_resnet50_model(self) -> None: + """Test resnet50 model with all applicable recipes.""" + self._run_model_with_all_recipes("resnet50") + + def test_vit_model(self) -> None: + """Test vit model with all applicable recipes.""" + self._run_model_with_all_recipes("vit") + + def test_w2l_model(self) -> None: + """Test w2l model with all applicable recipes.""" + self._run_model_with_all_recipes("w2l") diff --git a/export/utils.py b/export/utils.py new file mode 100644 index 00000000000..da2c30443c4 --- /dev/null +++ b/export/utils.py @@ -0,0 +1,51 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-strict +import logging +import platform + +import torch + + +def is_fbcode() -> bool: + return not hasattr(torch.version, "git_version") + + +# Check if lowering for CoreML is supported on the current platform +def is_supported_platform_for_coreml_lowering() -> bool: + system = platform.system() + machine = platform.machine().lower() + + # Check for Linux x86_64 + if system == "Linux" and machine == "x86_64": + return True + + # Check for macOS aarch64 + if system == "Darwin" and machine in ("arm64", "aarch64"): + return True + + logging.info(f"Unsupported platform: {system} {machine}") + + return False + + +# Check if lowering for QNN is supported on the current platform +def is_supported_platform_for_qnn_lowering() -> bool: + system = platform.system() + machine = platform.machine().lower() + + # Check for Linux x86_64 + if platform.system().lower() == "linux" and platform.machine().lower() in ( + "x86_64", + "amd64", + "i386", + "i686", + ): + return True + + logging.error(f"Unsupported platform for QNN lowering: {system} {machine}") + return False