Update on "[ET-VK] 8/n Split dispatches between multiple command buffers. This diff adds a config to limit the maximum number of command buffers created when splitting execution between multiple command buffers."

This diff introduces a new configuration option, `execute_max_cmds`, to limit the maximum number of command buffers created when splitting execution between multiple command buffers. This feature allows for more efficient management of command buffers, particularly in scenarios where the number of nodes in the graph is large.

Differential Revision: [D79575908](https://our.internmc.facebook.com/intern/diff/D79575908/)

[ghstack-poisoned]

Author: trivedivivek

.ci/scripts/test_huggingface_optimum_model.py

@@ -262,14 +262,20 @@ def test_vit(model_id, model_dir, recipe, *, quantize=False, run_only=False):
 
     assert torch.allclose(
         eager_output.logits, et_output, atol=1e-02, rtol=1e-02
-    ), "CoreML output does not match eager"
+    ), "Model output does not match eager"
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--model", type=str, required=True)
     parser.add_argument("--recipe", type=str, required=True)
     parser.add_argument("--quantize", action="store_true", help="Enable quantization")
+    parser.add_argument(
+        "--model_dir",
+        type=str,
+        required=False,
+        help="When provided, write the pte file to this directory. Otherwise, a temporary directory is created for the test.",
+    )
     args = parser.parse_args()
 
     model_to_model_id_and_test_function = {
@@ -294,11 +300,11 @@ def test_vit(model_id, model_dir, recipe, *, quantize=False, run_only=False):
             f"Unknown model name: {args.model}. Available models: {model_to_model_id_and_test_function.keys()}"
         )
 
+    model_id, test_fn = model_to_model_id_and_test_function[args.model]
     with tempfile.TemporaryDirectory() as tmp_dir:
-        model_id, test_fn = model_to_model_id_and_test_function[args.model]
         test_fn(
             model_id=model_id,
-            model_dir=tmp_dir,
+            model_dir=tmp_dir if args.model_dir is None else args.model_dir,
             recipe=args.recipe,
             quantize=args.quantize,
         )

.ci/scripts/test_model.sh

@@ -166,34 +166,49 @@ test_model_with_qnn() {
   export PYTHONPATH=$EXECUTORCH_ROOT/..
 
   EXTRA_FLAGS=""
+  # Ordered by the folder name, then alphabetically by the model name
+  # Following models are inside examples/qualcomm/scripts folder
   if [[ "${MODEL_NAME}" == "dl3" ]]; then
     EXPORT_SCRIPT=deeplab_v3
-  elif [[ "${MODEL_NAME}" == "mv3" ]]; then
-    EXPORT_SCRIPT=mobilenet_v3
-  elif [[ "${MODEL_NAME}" == "mv2" ]]; then
-    EXPORT_SCRIPT=mobilenet_v2
-  elif [[ "${MODEL_NAME}" == "ic4" ]]; then
-    EXPORT_SCRIPT=inception_v4
+  elif [[ "${MODEL_NAME}" == "edsr" ]]; then
+    EXPORT_SCRIPT=edsr
+    # Additional deps for edsr
+    pip install piq
   elif [[ "${MODEL_NAME}" == "ic3" ]]; then
     EXPORT_SCRIPT=inception_v3
-  elif [[ "${MODEL_NAME}" == "vit" ]]; then
-    EXPORT_SCRIPT=torchvision_vit
+  elif [[ "${MODEL_NAME}" == "ic4" ]]; then
+    EXPORT_SCRIPT=inception_v4
   elif [[ "${MODEL_NAME}" == "mb" ]]; then
     EXPORT_SCRIPT=mobilebert_fine_tune
     EXTRA_FLAGS="--num_epochs 1"
     pip install scikit-learn
+  elif [[ "${MODEL_NAME}" == "mv2" ]]; then
+    EXPORT_SCRIPT=mobilenet_v2
+  elif [[ "${MODEL_NAME}" == "mv3" ]]; then
+    EXPORT_SCRIPT=mobilenet_v3
+  elif [[ "${MODEL_NAME}" == "vit" ]]; then
+    EXPORT_SCRIPT=torchvision_vit
   elif [[ "${MODEL_NAME}" == "w2l" ]]; then
     EXPORT_SCRIPT=wav2letter
   elif [[ "${MODEL_NAME}" == "edsr" ]]; then
     EXPORT_SCRIPT=edsr
     # Additional deps for edsr
     pip install piq
+  # Following models are inside examples/qualcomm/oss_scripts folder
+  elif [[ "${MODEL_NAME}" == "albert" ]]; then
+    EXPORT_SCRIPT=albert
+  elif [[ "${MODEL_NAME}" == "bert" ]]; then
+    EXPORT_SCRIPT=bert
   elif [[ "${MODEL_NAME}" == "cvt" ]]; then
     EXPORT_SCRIPT=cvt
+  elif [[ "${MODEL_NAME}" == "distilbert" ]]; then
+    EXPORT_SCRIPT=distilbert
   elif [[ "${MODEL_NAME}" == "dit" ]]; then
     EXPORT_SCRIPT=dit
   elif [[ "${MODEL_NAME}" == "efficientnet" ]]; then
     EXPORT_SCRIPT=efficientnet
+  elif [[ "${MODEL_NAME}" == "eurobert" ]]; then
+    EXPORT_SCRIPT=eurobert
   elif [[ "${MODEL_NAME}" == "focalnet" ]]; then
     EXPORT_SCRIPT=focalnet
   elif [[ "${MODEL_NAME}" == "mobilevit_v1" ]]; then
@@ -202,18 +217,10 @@ test_model_with_qnn() {
     EXPORT_SCRIPT=mobilevit_v2
   elif [[ "${MODEL_NAME}" == "pvt" ]]; then
     EXPORT_SCRIPT=pvt
-  elif [[ "${MODEL_NAME}" == "swin" ]]; then
-    EXPORT_SCRIPT=swin_transformer
-  elif [[ "${MODEL_NAME}" == "albert" ]]; then
-    EXPORT_SCRIPT=albert
-  elif [[ "${MODEL_NAME}" == "bert" ]]; then
-    EXPORT_SCRIPT=bert
-  elif [[ "${MODEL_NAME}" == "distilbert" ]]; then
-    EXPORT_SCRIPT=distilbert
-  elif [[ "${MODEL_NAME}" == "eurobert" ]]; then
-    EXPORT_SCRIPT=eurobert
   elif [[ "${MODEL_NAME}" == "roberta" ]]; then
     EXPORT_SCRIPT=roberta
+  elif [[ "${MODEL_NAME}" == "swin" ]]; then
+    EXPORT_SCRIPT=swin_transformer
   else
     echo "Unsupported model $MODEL_NAME"
     exit 1

.github/workflows/trunk.yml

@@ -285,12 +285,12 @@ jobs:
         setup_script_args=""
         if [[ ${{ matrix.os}} == "bare_metal" ]]; then
           toolchain_prefix=arm-none-eabi-
-          threshold="109000"
+          threshold="110592" # 108 KiB
           toolchain_cmake=examples/arm/ethos-u-setup/arm-none-eabi-gcc.cmake
         elif [[ ${{ matrix.os}} == "zephyr-preset" ]]; then
           setup_script_args="--target-toolchain zephyr"
           toolchain_prefix=arm-zephyr-eabi-
-          threshold="135000"
+          threshold="135168" # 132 KiB
           toolchain_cmake=examples/zephyr/x86_64-linux-arm-zephyr-eabi-gcc.cmake
         else
           echo "Fail unsupport OS selection ${{ matrix.os }}"
@@ -815,6 +815,9 @@ jobs:
           smollm|coreml_fp32_gpu|--quantize,
           llama3|coreml_fp32_gpu|--quantize,
           olmo|coreml_fp32_gpu|--quantize,
+          # roberta|coreml_fp32_gpu|--quantize, roberta requires special HF access
+          bert|coreml_fp32_gpu|--quantize,
+          distilbert|coreml_fp32_gpu|--quantize,
         ]
       fail-fast: false
     with:

backends/apple/coreml/compiler/coreml_preprocess.py

@@ -126,15 +126,18 @@ def model_compute_precision_from_compile_specs(
 
     @staticmethod
     def generate_minimum_deployment_target_compile_spec(
-        min_deployment_target: ct.target,
+        min_deployment_target: Optional[ct.target],
     ) -> CompileSpec:
         """
         Returns the compile spec representing the minimum deployment target on which the model can run,
         for additional details please refer to the documentation for ``coremltools.target``.
         """
+        value = str("").encode("utf-8")
+        if min_deployment_target is not None:
+            value = str(min_deployment_target.value).encode("utf-8")
         return CompileSpec(
             COMPILE_SPEC_KEYS.MIN_DEPLOYMENT_TARGET.value,
-            str(min_deployment_target.value).encode("utf-8"),
+            value,
         )
 
     @staticmethod
@@ -146,10 +149,13 @@ def min_deployment_target_from_compile_specs(
         """
         for compile_spec in compile_specs:
             if compile_spec.key == COMPILE_SPEC_KEYS.MIN_DEPLOYMENT_TARGET.value:
-                compile_spec_value: int = int(compile_spec.value.decode("utf-8"))
+                value = compile_spec.value.decode("utf-8")
+                if value == "":
+                    return None
+                compile_spec_value: int = int(value)
                 return ct.target(compile_spec_value)
 
-        return ct.target.iOS15
+        return None
 
     @staticmethod
     def compute_unit_from_compile_specs(
@@ -211,7 +217,7 @@ def op_linear_quantizer_config_from_compile_specs(
     @staticmethod
     def generate_compile_specs(
         compute_unit: ct.ComputeUnit = ct.ComputeUnit.ALL,
-        minimum_deployment_target: ct.target = ct.target.iOS15,
+        minimum_deployment_target: Optional[ct.target] = None,
         compute_precision: ct.precision = ct.precision.FLOAT16,
         model_type: MODEL_TYPE = MODEL_TYPE.MODEL,
         op_linear_quantizer_config: Optional[Dict] = None,
@@ -248,6 +254,13 @@ def model_metadata_from_spec(
         input_names: List[str] = [input.name for input in model_spec.description.input]
         output_names = [output.name for output in model_spec.description.output]
 
+        if len(output_names) == 0:
+            raise ValueError("Cannot lower a model with no outputs in CoreML.")
+        if len(input_names) == 0:
+            assert (
+                model_spec.specificationVersion >= 9
+            ), "Deploying a model with no inputs in CoreML requires you set minimum_deployment_target to iOS18 or later in the CoreMLPartitioner."
+
         return ModelMetadata(
             inputNames=input_names, outputNames=output_names, identifier=identifier
         )
@@ -352,6 +365,12 @@ def preprocess_model(
         dir_path: Path = Path("tmp") / identifier
         model_dir_path: Path = dir_path / "lowered_module"
         model_spec: ct.proto.Model_pb2 = mlmodel.get_spec()
+        logger.warning(
+            f"The model with identifier {identifier} was exported with CoreML specification version {model_spec.specificationVersion}, and it will not run on all version of iOS/macOS."
+            " See https://apple.github.io/coremltools/mlmodel/Format/Model.html#model for information on what OS versions are compatible with this specifcation version."
+            " If you want to control the deployment target, please set the minimum_deployment_target compile spec in the CoreMLPartitioner."
+        )
+
         model_metadata: ModelMetadata = CoreMLBackend.model_metadata_from_spec(
             model_spec=model_spec,
             identifier=identifier,
@@ -418,7 +437,7 @@ def preprocess(
         model_compute_precision: ct.precision = (
             CoreMLBackend.model_compute_precision_from_compile_specs(compile_specs)
         )
-        minimum_deployment_target: ct.target = (
+        minimum_deployment_target: Optional[ct.target] = (
             CoreMLBackend.min_deployment_target_from_compile_specs(compile_specs)
         )
         compute_units: ct.ComputeUnit = CoreMLBackend.compute_unit_from_compile_specs(

backends/apple/coreml/runtime/delegate/executorch_operations.h

@@ -0,0 +1,5 @@
+#pragma once
+
+namespace executorch::core_ml_backend_delegate {
+void register_backend_coreml();
+} // namespace executorch::core_ml_backend_delegate

backends/apple/coreml/runtime/delegate/executorch_operations.mm

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "executorch_operations.h"
+#import <coreml_backend/delegate.h>
+#import "ETCoreMLStrings.h"
+#import "backend_delegate.h"
+
+#import <executorch/runtime/core/evalue.h>
+#import <executorch/runtime/platform/log.h>
+#import <executorch/runtime/backend/interface.h>
+
+#include <array>
+#import <memory>
+
+namespace executorch::core_ml_backend_delegate {
+  using executorch::runtime::get_backend_class;
+
+static std::unique_ptr<executorch::backends::coreml::CoreMLBackendDelegate> backendInterfaceLazy_;
+
+void register_backend_coreml() {
+    auto backendInterface = executorch::runtime::get_backend_class(ETCoreMLStrings.delegateIdentifier.UTF8String);
+    if (backendInterface == nullptr) {
+      backendInterfaceLazy_ = std::make_unique<executorch::backends::coreml::CoreMLBackendDelegate>();
+      executorch::runtime::Backend backend{ETCoreMLStrings.delegateIdentifier.UTF8String, backendInterfaceLazy_.get()};
+      std::ignore = register_backend(backend);
+    }
+  }
+
+} // namespace executorch::core_ml_backend_delegate

backends/apple/coreml/runtime/delegate/model_metadata.h

@@ -29,9 +29,7 @@ struct ModelMetadata {
     inline ModelMetadata() noexcept { }
 
     /// Returns `true` if the metadata is valid otherwise `false`.
-    inline bool is_valid() const noexcept {
-        return !identifier.empty() && !input_names.empty() && !output_names.empty();
-    }
+    inline bool is_valid() const noexcept { return !identifier.empty() && !output_names.empty(); }
 
     inline std::string to_json_string() const noexcept { return executorchcoreml::serde::json::to_json_string(*this); }
 

backends/arm/CMakeLists.txt

@@ -14,7 +14,9 @@ endif()
 
 include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake)
 
-set(_common_include_directories ${EXECUTORCH_ROOT}/.. ${EXECUTORCH_ROOT}/runtime/core/portable_type/c10)
+set(_common_include_directories
+    ${EXECUTORCH_ROOT}/.. ${EXECUTORCH_ROOT}/runtime/core/portable_type/c10
+)
 add_compile_definitions(C10_USING_CUSTOM_GENERATED_MACROS)
 
 
@@ -34,13 +36,12 @@ set(_arm_baremetal_sources backends/arm/runtime/EthosUBackend.cpp
 list(TRANSFORM _arm_baremetal_sources PREPEND "${EXECUTORCH_ROOT}/")
 
 add_library(executorch_delegate_ethos_u STATIC ${_arm_baremetal_sources})
-target_include_directories(
-  executorch_delegate_ethos_u PUBLIC ${_common_include_directories}
-)
-target_include_directories(
-  executorch_delegate_ethos_u PUBLIC ${DRIVER_ETHOSU_INCLUDE_DIR}
+target_link_libraries(
+  executorch_delegate_ethos_u PUBLIC executorch_core ethosu_core_driver
 )
 
+install(TARGETS executorch_delegate_ethos_u EXPORT ExecuTorchTargets)
+
 # end config for bare metal builds
 endif()
 

backends/arm/_passes/annotate_decomposed_matmul.py

@@ -12,7 +12,7 @@
 import torch
 from executorch.backends.arm._passes.arm_pass_utils import create_node
 
-from executorch.backends.arm.tosa_quant_utils import dq_ops, q_ops
+from executorch.backends.arm.constants import DQ_OPS, Q_OPS
 from executorch.exir.dialects._ops import ops as exir_ops
 from executorch.exir.dialects.edge._ops import EdgeOpOverload
 from executorch.exir.pass_base import ExportPass, PassResult
@@ -62,7 +62,7 @@ def call(self, graph_module: GraphModule) -> PassResult:
         }
         for partition in matmul_partitions:
             quantized_input = all(
-                input_node.target in dq_ops for input_node in partition.input_nodes
+                input_node.target in DQ_OPS for input_node in partition.input_nodes
             )
             matmul_node = [
                 node for node in partition.nodes if node.target in matmul_targets
@@ -93,7 +93,7 @@ def call(self, graph_module: GraphModule) -> PassResult:
                     graph_module.graph.erase_node(partition_input)
 
             partition_output = list(partition.output_nodes[0].users)[0]
-            quantized_output = partition_output.target in q_ops
+            quantized_output = partition_output.target in Q_OPS
             if quantized_output:
                 with graph_module.graph.inserting_after(matmul_node):
                     # Create q-node after matmul

backends/arm/_passes/fold_qdq_with_annotated_qparams_pass.py

@@ -15,8 +15,9 @@
     get_param_tensor,
     is_param_node,
 )
+from executorch.backends.arm.constants import DQ_OPS, Q_OPS
 
-from executorch.backends.arm.tosa_quant_utils import dq_ops, q_ops, QuantArgs
+from executorch.backends.arm.tosa_quant_utils import QuantArgs
 
 from executorch.exir.dialects._ops import ops as exir_ops
 from executorch.exir.dialects.edge._ops import EdgeOpOverload
@@ -109,7 +110,7 @@ def fold_and_annotate_arg(
                 return
 
             arg_quant_params = None
-            if arg.target in dq_ops:
+            if arg.target in DQ_OPS:
                 args = arg.args
                 scales = args[1]
                 if (
@@ -137,9 +138,9 @@ def fold_and_annotate_arg(
         if input_qparams is not None:
             node.meta["input_qparams"][i] = input_qparams
             for n in nodes_to_remove:
-                if n.target not in dq_ops:
+                if n.target not in DQ_OPS:
                     raise RuntimeError(
-                        f"Expected one of {dq_ops} dq_op, got {n.target}"
+                        f"Expected one of {DQ_OPS} dq_op, got {n.target}"
                     )
 
                 node.replace_input_with(n, cast(Node, n.args[0]))
@@ -154,7 +155,7 @@ def call(self, graph_module: GraphModule) -> PassResult:
             if n.op != "call_function":
                 continue
             # Don't fold chains of quant-ops into each other.
-            if n.target in (*q_ops, *dq_ops):
+            if n.target in (*Q_OPS, *DQ_OPS):
                 continue
 
             # Make sure we haven't already set qparams meta information on the node
@@ -184,7 +185,7 @@ def call(self, graph_module: GraphModule) -> PassResult:
             # Copy the users, since we are modifying it.
             users_copy = copy.copy(n.users)
             for i, user in enumerate(users_copy):
-                if user.target not in q_ops:
+                if user.target not in Q_OPS:
                     continue
 
                 # quantization node found here, store the quantization parameters in meta value
@@ -221,7 +222,7 @@ def call(self, graph_module: GraphModule) -> PassResult:
 
             # Make sure we have a quantized operator
             user = list(n.users)[0]
-            if user.target not in q_ops:
+            if user.target not in Q_OPS:
                 continue
 
             qargs = QuantArgs.from_operator(user.target, user.args)