Merge branch 'main' into jathu/move-gflags

Author: jathu

.lintrunner.toml

@@ -271,6 +271,10 @@ exclude_patterns = [
     'examples/**',
     'exir/verification/bindings.cpp',
     'extension/**',
+    # Uses properly-gated (ET_USE_PYTORCH_HEADERS) ATen include.
+    'kernels/portable/cpu/util/elementwise_util.h',
+    'kernels/portable/cpu/util/math_util.h',
+    'kernels/portable/cpu/util/vectorized_math.h',
     'kernels/optimized/**',
     'runtime/core/exec_aten/**',
     # Want to be able to keep c10 in sync with PyTorch core.

backends/arm/_passes/scalars_to_attribute_pass.py

@@ -12,8 +12,8 @@
 from executorch.backends.arm._passes.arm_pass_utils import get_first_fake_tensor
 
 from executorch.exir.pass_base import ExportPass, PassResult
-from torch.ao.quantization.fx.utils import get_new_attr_name_with_prefix
 from torch.fx import GraphModule, Node
+from torchao.quantization.pt2e.utils import get_new_attr_name_with_prefix
 
 
 class ScalarsToAttributePass(ExportPass):

backends/cadence/CMakeLists.txt

@@ -82,6 +82,7 @@ elseif(EXECUTORCH_FUSION_G3_OPT)
   ${EXECUTORCH_ROOT}/runtime/core/portable_type/c10)
 else()
   set(TARGET_DIR reference)
+  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${TARGET_DIR}/kernels)
 endif()
 
 

backends/cadence/aot/functions_hifi.yaml

@@ -32,6 +32,36 @@
     - arg_meta: null
       kernel_name: cadence::impl::HiFi::add_out
 
+- op: bitwise_and.Scalar_out
+  kernels:
+    - arg_meta: null
+      kernel_name: cadence::impl::HiFi::bitwise_and_Scalar_out
+
+- op: bitwise_and.Tensor_out
+  kernels:
+    - arg_meta: null
+      kernel_name: cadence::impl::HiFi::bitwise_and_Tensor_out
+
+- op: bitwise_or.Scalar_out
+  kernels:
+    - arg_meta: null
+      kernel_name: cadence::impl::HiFi::bitwise_or_Scalar_out
+
+- op: bitwise_or.Tensor_out
+  kernels:
+    - arg_meta: null
+      kernel_name: cadence::impl::HiFi::bitwise_or_Tensor_out
+
+- op: bitwise_xor.Scalar_out
+  kernels:
+    - arg_meta: null
+      kernel_name: cadence::impl::HiFi::bitwise_xor_Scalar_out
+
+- op: bitwise_xor.Tensor_out
+  kernels:
+    - arg_meta: null
+      kernel_name: cadence::impl::HiFi::bitwise_xor_Tensor_out
+
 - op: bmm.out
   kernels:
     - arg_meta: null
@@ -65,27 +95,82 @@
 - op: embedding.out
   kernels:
     - arg_meta: null
-      kernel_name: torch::executor::embedding_out
+      kernel_name: cadence::impl::HiFi::embedding_out
+
+- op: eq.Tensor_out
+  kernels:
+    - arg_meta: null
+      kernel_name: cadence::impl::HiFi::eq_tensor_out
+
+- op: fmod.Tensor_out
+  kernels:
+    - arg_meta: null
+      kernel_name: cadence::impl::HiFi::fmod_Tensor_out
+
+- op: fmod.Scalar_out
+  kernels:
+    - arg_meta: null
+      kernel_name: cadence::impl::HiFi::fmod_Scalar_out
 
 - op: full.out
   kernels:
     - arg_meta: null
       kernel_name: cadence::impl::HiFi::full_out
 
-- op: gt.Scalar_out
+- op: ge.Scalar_out
+  kernels:
+    - arg_meta: null
+      kernel_name: cadence::impl::HiFi::ge_scalar_out
+
+- op: ge.Tensor_out
   kernels:
     - arg_meta: null
-      kernel_name: torch::executor::gt_scalar_out
+      kernel_name: cadence::impl::HiFi::ge_tensor_out
 
 - op: gelu.out
   kernels:
     - arg_meta: null
       kernel_name: torch::executor::gelu_out
 
+- op: gt.Scalar_out
+  kernels:
+    - arg_meta: null
+      kernel_name: cadence::impl::HiFi::gt_scalar_out
+
+- op: gt.Tensor_out
+  kernels:
+    - arg_meta: null
+      kernel_name: cadence::impl::HiFi::gt_tensor_out
+
 - op: hardtanh.out
   kernels:
     - arg_meta: null
-      kernel_name: torch::executor::hardtanh_out
+      kernel_name: cadence::impl::HiFi::hardtanh_out
+
+- op: le.Scalar_out
+  kernels:
+    - arg_meta: null
+      kernel_name: cadence::impl::HiFi::le_scalar_out
+
+- op: le.Tensor_out
+  kernels:
+    - arg_meta: null
+      kernel_name: cadence::impl::HiFi::le_tensor_out
+
+- op: lt.Scalar_out
+  kernels:
+    - arg_meta: null
+      kernel_name: cadence::impl::HiFi::lt_scalar_out
+
+- op: lt.Tensor_out
+  kernels:
+    - arg_meta: null
+      kernel_name: cadence::impl::HiFi::lt_tensor_out
+
+- op: masked_fill.Scalar_out
+  kernels:
+    - arg_meta: null
+      kernel_name: cadence::impl::HiFi::masked_fill_scalar_out
 
 - op: max_pool2d_with_indices.out
   kernels:
@@ -117,6 +202,11 @@
     - arg_meta: null
       kernel_name: cadence::impl::HiFi::mul_out
 
+- op: ne.Tensor_out
+  kernels:
+    - arg_meta: null
+      kernel_name: cadence::impl::HiFi::ne_tensor_out
+
 - op: permute_copy.out
   kernels:
     - arg_meta: null
@@ -147,6 +237,11 @@
     - arg_meta: null
       kernel_name: cadence::impl::HiFi::rsqrt_out
 
+- op: select_copy.int_out
+  kernels:
+    - arg_meta: null
+      kernel_name: cadence::impl::HiFi::select_copy_int_out
+
 - op: sigmoid.out
   kernels:
     - arg_meta: null
@@ -239,6 +334,11 @@
     - arg_meta: null
       kernel_name: cadence::impl::HiFi::quantized_fully_connected_out
 
+- func: cadence::quantized_matmul.out(Tensor X, int X_zero_point, Tensor Y, int Y_zero_point, Tensor? bias, int out_multiplier, int out_shift, int out_zero_point, bool transposed, *, Tensor(a!) out) -> Tensor(a!)
+  kernels:
+    - arg_meta: null
+      kernel_name: cadence::impl::HiFi::quantized_matmul_out
+
 - func: cadence::quantized_fully_connected.per_tensor_out(Tensor src, Tensor weight, Tensor bias, int src_zero_point, int weight_zero_point, int out_multiplier, int out_shift, int out_zero_point, Tensor? offset, *, Tensor(a!) out) -> Tensor(a!)
   kernels:
     - arg_meta: null

backends/cadence/hifi/kernels/CMakeLists.txt

@@ -16,6 +16,9 @@ add_library(
   ${EXECUTORCH_ROOT}/backends/cadence/hifi/third-party/nnlib/xa_nn_elm_clamp_f32_broadcast.c
   ${EXECUTORCH_ROOT}/backends/cadence/hifi/third-party/nnlib/xa_nn_elm_div_f32_broadcast.c
   ${EXECUTORCH_ROOT}/backends/cadence/hifi/third-party/nnlib/xa_nn_elm_div_mode_f32_broadcast.c
+  ${EXECUTORCH_ROOT}/backends/cadence/hifi/third-party/nnlib/xa_nn_elm_fmod_broadcast_f32.c
+  ${EXECUTORCH_ROOT}/backends/cadence/hifi/third-party/nnlib/xa_nn_greater_lesser_equal_f32.c
+  ${EXECUTORCH_ROOT}/backends/cadence/hifi/third-party/nnlib/xa_nn_elm_logicalxor_bool_bool.c
   ${EXECUTORCH_ROOT}/backends/cadence/hifi/third-party/nnlib/xa_nn_elm_minimum_maximum_f32.c
   ${EXECUTORCH_ROOT}/backends/cadence/hifi/third-party/nnlib/xa_nn_elm_mul_f32_broadcast.c
   ${EXECUTORCH_ROOT}/backends/cadence/hifi/third-party/nnlib/xa_nn_elm_pow_f32.c

backends/cadence/hifi/kernels/kernels.h

@@ -8,7 +8,6 @@
 
 #pragma once
 #include <executorch/runtime/kernel/kernel_includes.h>
-#include <inttypes.h>
 #include <stddef.h>
 #include <xa_type_def.h>
 /* For NNLIB APIs */
@@ -88,6 +87,42 @@ extern "C" WORD32 xa_nn_elm_div_mode_broadcast_4D_f32xf32_f32(
     const WORD32* const p_inp2_shape,
     WORD32 mode);
 
+extern "C" WORD32 xa_nn_elm_greater_lesser_equal_f32xf32_f32(
+    WORD8* __restrict__ p_out,
+    const FLOAT32* __restrict__ p_inp1,
+    const FLOAT32* __restrict__ p_inp2,
+    WORD32 num_elm,
+    WORD32 kernel_type);
+
+extern "C" WORD32 xa_nn_elm_greater_lesser_equal_broadcast_4D_f32xf32_f32(
+    WORD8* __restrict__ p_out,
+    const WORD32* const p_out_shape,
+    const FLOAT32* __restrict__ p_inp1,
+    const WORD32* const p_inp1_shape,
+    const FLOAT32* __restrict__ p_inp2,
+    const WORD32* const p_inp2_shape,
+    WORD32 kernel_type);
+
+extern "C" WORD32 xa_nn_elm_fmod_f32xf32_f32(
+    FLOAT32* __restrict__ p_out,
+    const FLOAT32* __restrict__ p_inp1,
+    const FLOAT32* __restrict__ p_inp2,
+    WORD32 num_elm);
+
+extern "C" WORD32 xa_nn_elm_fmod_broadcast_4D_f32xf32_f32(
+    FLOAT32* __restrict__ p_out,
+    const WORD32* const p_out_shape,
+    const FLOAT32* __restrict__ p_inp1,
+    const WORD32* const p_inp1_shape,
+    const FLOAT32* __restrict__ p_inp2,
+    const WORD32* const p_inp2_shape);
+
+extern "C" WORD32 xa_nn_elm_logicalxor_boolxbool_bool(
+    WORD8* __restrict__ p_out,
+    const WORD8* __restrict__ p_inp1,
+    const WORD8* __restrict__ p_inp2,
+    WORD32 num_elm);
+
 extern "C" WORD32 xa_nn_elm_maximum_f32xf32_f32(
     FLOAT32* __restrict__ p_out,
     const FLOAT32* __restrict__ p_inp1,

backends/cadence/hifi/operators/CMakeLists.txt

@@ -18,20 +18,34 @@ include(${EXECUTORCH_ROOT}/tools/cmake/Codegen.cmake)
 set(_aten_ops__srcs
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_add.cpp"
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_atan2.cpp"
+    "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_bitwise_and.cpp"
+    "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_bitwise_or.cpp"
+    "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_bitwise_xor.cpp"
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_bmm.cpp"
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_cat.cpp"
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_clamp.cpp"
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_div.cpp"
+    "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_embedding.cpp"
+    "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_eq.cpp"
+    "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_fmod.cpp"
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_full.cpp"
+    "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_ge.cpp"
+    "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_gt.cpp"
+    "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_hardtanh.cpp"
+    "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_le.cpp"
+    "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_lt.cpp"
+    "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_masked_fill.cpp"
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_maximum.cpp"
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_mean.cpp"
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_minimum.cpp"
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_mm.cpp"
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_mul.cpp"
+    "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_ne.cpp"
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_permute_copy.cpp"
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_pow.cpp"
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_remainder.cpp"
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_rsqrt.cpp"
+    "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_select_copy.cpp"
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_slice_copy.cpp"
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_softmax.cpp"
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_split_with_sizes_copy.cpp"
@@ -41,22 +55,21 @@ set(_aten_ops__srcs
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_view_copy.cpp"
     "${EXECUTORCH_ROOT}/backends/cadence/hifi/operators/op_where.cpp"
     "${EXECUTORCH_ROOT}/kernels/portable/cpu/op_clone.cpp"
-    "${EXECUTORCH_ROOT}/kernels/portable/cpu/op_embedding.cpp"
-    "${EXECUTORCH_ROOT}/kernels/portable/cpu/op_gt.cpp"
     "${EXECUTORCH_ROOT}/kernels/portable/cpu/op_gelu.cpp"
-    "${EXECUTORCH_ROOT}/kernels/portable/cpu/op_hardtanh.cpp"
     "${EXECUTORCH_ROOT}/kernels/portable/cpu/op_max_pool2d_with_indices.cpp"
     "${EXECUTORCH_ROOT}/kernels/portable/cpu/op_to_copy.cpp"
     "${EXECUTORCH_ROOT}/kernels/portable/cpu/pattern/unary_ufunc_realhbbf16_to_floathbf16.cpp"
     "${EXECUTORCH_ROOT}/kernels/portable/cpu/util/activation_ops_util.cpp"
     "${EXECUTORCH_ROOT}/kernels/portable/cpu/util/broadcast_util.cpp"
     "${EXECUTORCH_ROOT}/kernels/portable/cpu/util/copy_ops_util.cpp"
     "${EXECUTORCH_ROOT}/kernels/portable/cpu/util/dtype_util.cpp"
+    "${EXECUTORCH_ROOT}/kernels/portable/cpu/util/delinearize_index.cpp"
     "${EXECUTORCH_ROOT}/kernels/portable/cpu/util/index_util.cpp"
     "${EXECUTORCH_ROOT}/kernels/portable/cpu/util/kernel_ops_util.cpp"
     "${EXECUTORCH_ROOT}/kernels/portable/cpu/util/matmul_ops_util.cpp"
     "${EXECUTORCH_ROOT}/kernels/portable/cpu/util/reduce_util.cpp"
     "${EXECUTORCH_ROOT}/kernels/portable/cpu/util/repeat_util.cpp"
+    "${EXECUTORCH_ROOT}/kernels/portable/cpu/util/select_copy_util.cpp"
     "${EXECUTORCH_ROOT}/kernels/portable/cpu/util/slice_util.cpp"
     "${EXECUTORCH_ROOT}/kernels/portable/cpu/util/delinearize_index.cpp"
     )
@@ -75,7 +88,7 @@ target_include_directories(
 
 # Custom ops that are needed to run the test model.
 add_library(
-  custom_ops "op_quantized_linear_out.cpp" "op_quantized_layer_norm.cpp"
+  custom_ops "op_quantized_linear_out.cpp" "op_quantized_layer_norm.cpp" "quantized_matmul_out.cpp"
              "op_quantize_per_tensor.cpp" "op_quantized_relu_out.cpp" "op_dequantize_per_tensor.cpp"
              "op_quantized_conv_out.cpp" "op_quantized_fully_connected_out"
 )