Update on "[ET-VK][ez][Ops] registering Q/DQ/CQP ops and specifying optimal storage"

# Context

Certain quantization operators need scales and zeros to be set with a storage layout as buffers. Since the existing op_registry does not allow specifying how input parameters are set with their memory or storage layout, we need to specify that the optimal storage type is buffer so that is conversion pass is added to ensure that the inputs are also buffers.

# Changes

This moves the quantized_decomposed operators in their own registration, while also specifying that buffer is preferred.

Differential Revision: [D77746131](https://our.internmc.facebook.com/intern/diff/D77746131/)

[ghstack-poisoned]

Author: morelos

.ci/scripts/test_yolo12.sh

@@ -0,0 +1,197 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -ex
+# shellcheck source=/dev/null
+source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -model)
+      MODEL_NAME="$2" # stories110M
+      shift 2
+      ;;
+    -mode)
+      MODE="$2" # portable or xnnpack+custom or xnnpack+custom+qe
+      shift 2
+      ;;
+    -pt2e_quantize)
+      PT2E_QUANTIZE="$2"
+      shift 2
+      ;;
+    -upload)
+      UPLOAD_DIR="$2"
+      shift 2
+      ;;
+    -video_path)
+      VIDEO_PATH="$2" # portable or xnnpack+custom or xnnpack+custom+qe
+      shift 2
+      ;;
+    *)
+      echo "Unknown option: $1"
+      usage
+      ;;
+  esac
+done
+
+# Default mode to xnnpack+custom if not set
+MODE=${MODE:-"openvino"}
+
+# Default UPLOAD_DIR to empty string if not set
+UPLOAD_DIR="${UPLOAD_DIR:-}"
+
+# Default PT2E_QUANTIZE to empty string if not set
+PT2E_QUANTIZE="${PT2E_QUANTIZE:-}"
+
+# Default CMake Build Type to release mode
+CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release}
+
+if [[ $# -lt 5 ]]; then # Assuming 4 mandatory args
+    echo "Expecting atleast 5 positional arguments"
+    echo "Usage: [...]"
+fi
+if [[ -z "${MODEL_NAME:-}" ]]; then
+  echo "Missing model name, exiting..."
+  exit 1
+fi
+
+
+if [[ -z "${MODE:-}" ]]; then
+  echo "Missing mode, choose openvino or xnnpack, exiting..."
+  exit 1
+fi
+
+if [[ -z "${PYTHON_EXECUTABLE:-}" ]]; then
+  PYTHON_EXECUTABLE=python3
+fi
+
+TARGET_LIBS=""
+
+if [[ "${MODE}" =~ .*openvino.* ]]; then
+  OPENVINO=ON
+  TARGET_LIBS="$TARGET_LIBS openvino_backend "
+
+  git clone https://github.com/openvinotoolkit/openvino.git
+  cd openvino && git b16b776ac119dafda51f69a80f1e6b7376d02c3b
+  git submodule update --init --recursive
+  sudo ./install_build_dependencies.sh
+  mkdir build && cd build
+  cmake .. -DCMAKE_BUILD_TYPE=Release -DENABLE_PYTHON=ON
+  make -j$(nproc)
+
+  cd ..
+  cmake --install build --prefix dist
+
+  source dist/setupvars.sh
+  cd ../backends/openvino
+  pip install -r requirements.txt
+  cd ../../
+else
+  OPENVINO=OFF
+fi
+
+if [[ "${MODE}" =~ .*xnnpack.* ]]; then
+  XNNPACK=ON
+  TARGET_LIBS="$TARGET_LIBS xnnpack_backend "
+else
+  XNNPACK=OFF
+fi
+
+which "${PYTHON_EXECUTABLE}"
+
+
+DIR="examples/models/yolo12"
+$PYTHON_EXECUTABLE -m pip install -r ${DIR}/requirements.txt
+
+cmake_install_executorch_libraries() {
+    rm -rf cmake-out
+    build_dir=cmake-out
+    mkdir $build_dir
+
+
+    retry cmake -DCMAKE_INSTALL_PREFIX="${build_dir}" \
+          -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \
+          -DEXECUTORCH_BUILD_OPENVINO="$OPENVINO" \
+          -DEXECUTORCH_BUILD_XNNPACK="$XNNPACK" \
+          -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
+          -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
+          -DEXECUTORCH_BUILD_EXTENSION_RUNNER_UTIL=ON \
+          -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
+          -B"${build_dir}"
+
+    # Build the project
+    cmake --build ${build_dir} --target install --config ${CMAKE_BUILD_TYPE} -j$(nproc)
+
+    export CMAKE_ARGS="
+                       -DEXECUTORCH_BUILD_OPENVINO="$OPENVINO" \
+                       -DEXECUTORCH_BUILD_XNNPACK="$XNNPACK" \
+                       -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
+                       -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
+                       -DEXECUTORCH_BUILD_EXTENSION_RUNNER_UTIL=ON \
+                       -DEXECUTORCH_ENABLE_LOGGING=ON \
+                       -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
+                       -DEXECUTORCH_BUILD_PYBIND=ON"
+
+    echo $TARGET_LIBS
+    export CMAKE_BUILD_ARGS="--target $TARGET_LIBS"
+    pip install . --no-build-isolation
+}
+
+cmake_build_demo() {
+    echo "Building yolo12 runner"
+    retry cmake \
+        -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" \
+        -DUSE_OPENVINO_BACKEND="$OPENVINO" \
+        -DUSE_XNNPACK_BACKEND="$XNNPACK" \
+        -Bcmake-out/${DIR} \
+        ${DIR}
+    cmake --build cmake-out/${DIR} -j9 --config "$CMAKE_BUILD_TYPE"
+
+}
+
+cleanup_files() {
+ rm $EXPORTED_MODEL_NAME
+}
+
+prepare_artifacts_upload() {
+  if [ -n "${UPLOAD_DIR}" ]; then
+    echo "Preparing for uploading generated artifacs"
+    zip -j model.zip "${EXPORTED_MODEL_NAME}"
+    mkdir -p "${UPLOAD_DIR}"
+    mv model.zip "${UPLOAD_DIR}"
+    mv result.txt "${UPLOAD_DIR}"
+
+  fi
+}
+
+
+# Export model.
+EXPORTED_MODEL_NAME="${MODEL_NAME}_fp32_${MODE}.pte"
+echo "Exporting ${EXPORTED_MODEL_NAME}"
+EXPORT_ARGS="--model_name=${MODEL_NAME} --backend=${MODE}"
+
+# Add dynamically linked library location
+cmake_install_executorch_libraries
+
+$PYTHON_EXECUTABLE -m examples.models.yolo12.export_and_validate ${EXPORT_ARGS}
+
+
+RUNTIME_ARGS="--model_path=${EXPORTED_MODEL_NAME} --input_path=${VIDEO_PATH}"
+# Check build tool.
+cmake_build_demo
+# Run yolo12 runner
+NOW=$(date +"%H:%M:%S")
+echo "Starting to run yolo12 runner at ${NOW}"
+# shellcheck source=/dev/null
+cmake-out/examples/models/yolo12/Yolo12DetectionDemo ${RUNTIME_ARGS} > result.txt
+NOW=$(date +"%H:%M:%S")
+echo "Finished at ${NOW}"
+
+RESULT=$(cat result.txt)
+
+prepare_artifacts_upload
+cleanup_files

.github/workflows/lint.yml

@@ -83,8 +83,13 @@ jobs:
       script: |
         FILES_NEEDS_FORMAT=$(/opt/google-java-format -n \
           extension/android/executorch_android/src/main/java/org/pytorch/executorch/*.java \
+          extension/android/executorch_android/src/main/java/org/pytorch/executorch/extension/llm/*.java \
+          extension/android/executorch_android/src/main/java/org/pytorch/executorch/annotations/*.java \
+          extension/android/executorch_android/src/androidTest/java/org/pytorch/executorch/*.java \
           examples/demo-apps/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/*.java \
-          extension/benchmark/android/benchmark/app/src/main/java/org/pytorch/minibench/*.java)
+          examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/*.java \
+          extension/benchmark/android/benchmark/app/src/main/java/org/pytorch/minibench/*.java \
+          extension/benchmark/android/benchmark/app/src/androidTest/java/org/pytorch/minibench/*.java)
         if [ -n "$FILES_NEEDS_FORMAT" ]; then
           echo "Warning: The following files need formatting. Please use google-java-format."
           echo "Use a binary from https://github.com/google/google-java-format/releases/"

backends/apple/coreml/test/tester.py

@@ -0,0 +1,61 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+from typing import Any, List, Optional, Tuple
+
+import executorch
+import executorch.backends.test.harness.stages as BaseStages
+
+import torch
+from executorch.backends.apple.coreml.partition import CoreMLPartitioner
+from executorch.backends.test.harness import Tester as TesterBase
+from executorch.backends.test.harness.stages import StageType
+from executorch.exir import EdgeCompileConfig
+from executorch.exir.backend.partitioner import Partitioner
+
+
+class Partition(BaseStages.Partition):
+    def __init__(self, partitioner: Optional[Partitioner] = None):
+        super().__init__(
+            partitioner=partitioner or CoreMLPartitioner,
+        )
+
+
+class ToEdgeTransformAndLower(BaseStages.ToEdgeTransformAndLower):
+    def __init__(
+        self,
+        partitioners: Optional[List[Partitioner]] = None,
+        edge_compile_config: Optional[EdgeCompileConfig] = None,
+    ):
+        super().__init__(
+            default_partitioner_cls=CoreMLPartitioner,
+            partitioners=partitioners,
+            edge_compile_config=edge_compile_config,
+        )
+
+
+class CoreMLTester(TesterBase):
+    def __init__(
+        self,
+        module: torch.nn.Module,
+        example_inputs: Tuple[torch.Tensor],
+        dynamic_shapes: Optional[Tuple[Any]] = None,
+    ):
+        # Specialize for XNNPACK
+        stage_classes = (
+            executorch.backends.test.harness.Tester.default_stage_classes()
+            | {
+                StageType.PARTITION: Partition,
+                StageType.TO_EDGE_TRANSFORM_AND_LOWER: ToEdgeTransformAndLower,
+            }
+        )
+
+        super().__init__(
+            module=module,
+            stage_classes=stage_classes,
+            example_inputs=example_inputs,
+            dynamic_shapes=dynamic_shapes,
+        )

backends/arm/_passes/TARGETS

@@ -6,6 +6,7 @@ python_library(
     deps = [
         "//executorch/backends/arm:tosa_quant_utils",
         "//executorch/backends/arm:tosa_utils",
+        "//executorch/backends/arm/tosa/dialect:lib",
         "//executorch/backends/transforms:fuse_view_copy",
         "//executorch/backends/transforms:remove_getitem_op",
         "//executorch/backends/transforms:replace_scalar_with_tensor",

backends/arm/_passes/__init__.py

@@ -24,6 +24,7 @@
 from .convert_to_clamp import ConvertToClampPass  # noqa
 from .decompose_acosh_pass import DecomposeAcoshPass  # noqa
 from .decompose_adaptive_avg_pool2d_pass import DecomposeAdaptiveAvgPool2dPass  # noqa
+from .decompose_asin_pass import DecomposeAsinPass  # noqa
 from .decompose_atan_pass import DecomposeAtanPass  # noqa
 from .decompose_avg_pool2d import DecomposeAvgPool2d  # noqa
 from .decompose_batch_norm_no_stats import DecomposeBatchNormNoStatsPass  # noqa
@@ -50,6 +51,7 @@
 from .decompose_sqrt_pass import DecomposeSqrtPass  # noqa
 from .decompose_sum_pass import DecomposeSumPass  # noqa
 from .decompose_var_pass import DecomposeVarPass  # noqa
+from .decorate_fp32_to_int32_casting_pass import DecorateFp32toInt32CastingPass  # noqa
 from .fold_qdq_with_annotated_qparams_pass import (  # noqa
     FoldAndAnnotateQParamsPass,
     QuantizeOperatorArguments,

backends/arm/_passes/arm_pass_manager.py

@@ -29,6 +29,7 @@
     ConvertToClampPass,
     DecomposeAcoshPass,
     DecomposeAdaptiveAvgPool2dPass,
+    DecomposeAsinPass,
     DecomposeAtanPass,
     DecomposeAvgPool2d,
     DecomposeBatchNormNoStatsPass,
@@ -55,6 +56,7 @@
     DecomposeSqrtPass,
     DecomposeSumPass,
     DecomposeVarPass,
+    DecorateFp32toInt32CastingPass,
     FoldAndAnnotateQParamsPass,
     FuseBatchnorm2DPass,
     FuseConstantArgsPass,
@@ -158,6 +160,7 @@ def _tosa_080_BI_pipeline(self, exported_program: ExportedProgram) -> GraphModul
     def _tosa_080_MI_pipeline(self, exported_program: ExportedProgram) -> GraphModule:
         self.add_pass(DecomposeRoundPass())
         self.add_pass(DecomposeAcoshPass())
+        self.add_pass(DecomposeAsinPass())
         self.add_pass(DecomposeSqrtPass())
         self.add_pass(DecomposeAtanPass())
         self.add_pass(ConvertIntPowToMuls())
@@ -198,6 +201,9 @@ def _tosa_080_MI_pipeline(self, exported_program: ExportedProgram) -> GraphModul
         self.add_pass(MatchArgRanksPass(exported_program))
         self.add_pass(DecomposeAdaptiveAvgPool2dPass())
         self.add_pass(DecomposeAvgPool2d())
+        self.add_pass(
+            DecorateFp32toInt32CastingPass()
+        )  # Require that no new fp32->int32 is introduced after this pass
         self.add_pass(ComputeConstantOpsAOT(exported_program))
 
         self.add_pass(DecomposeGroupedConv())