|
| 1 | +/* |
| 2 | + * Copyright (c) Meta Platforms, Inc. and affiliates. |
| 3 | + * All rights reserved. |
| 4 | + * |
| 5 | + * This source code is licensed under the BSD-style license found in the |
| 6 | + * LICENSE file in the root directory of this source tree. |
| 7 | + */ |
| 8 | + |
| 9 | +#include <executorch/kernels/portable/cpu/util/kernel_ops_util.h> |
| 10 | +#include <executorch/kernels/portable/cpu/util/reduce_util.h> |
| 11 | +#include <executorch/runtime/kernel/kernel_includes.h> |
| 12 | +#include <executorch/runtime/platform/assert.h> |
| 13 | + |
| 14 | +#include <executorch/backends/cadence/hifi/kernels/kernels.h> |
| 15 | + |
| 16 | +using exec_aten::ScalarType; |
| 17 | +using exec_aten::Tensor; |
| 18 | +using executorch::aten::RuntimeContext; |
| 19 | +using executorch::runtime::ArrayRef; |
| 20 | +using torch::executor::Error; |
| 21 | +using torch::executor::optional; |
| 22 | + |
| 23 | +namespace cadence { |
| 24 | +namespace impl { |
| 25 | +namespace HiFi { |
| 26 | +namespace native { |
| 27 | + |
| 28 | +int prepare_data( |
| 29 | + const Tensor& in, |
| 30 | + Tensor& out, |
| 31 | + optional<ArrayRef<int64_t>> dim_list, |
| 32 | + int* inp_shape, |
| 33 | + int* out_shape, |
| 34 | + int* p_axis, |
| 35 | + int num_inp_dims, |
| 36 | + int num_out_dims) { |
| 37 | + for (int i = 0; i < num_inp_dims; i++) { |
| 38 | + inp_shape[i] = in.size(i); |
| 39 | + } |
| 40 | + |
| 41 | + for (int i = 0; i < num_out_dims; i++) { |
| 42 | + out_shape[i] = out.size(i); |
| 43 | + } |
| 44 | + |
| 45 | + int num_axis_dims = 0; |
| 46 | + for (const auto& d : dim_list.value()) { |
| 47 | + if (d < 0) { |
| 48 | + p_axis[num_axis_dims] = num_inp_dims + d; |
| 49 | + num_axis_dims++; |
| 50 | + } else { |
| 51 | + p_axis[num_axis_dims] = d; |
| 52 | + num_axis_dims++; |
| 53 | + } |
| 54 | + } |
| 55 | + |
| 56 | + return num_axis_dims; |
| 57 | +} |
| 58 | + |
| 59 | +Tensor& mean_dim_out( |
| 60 | + RuntimeContext& ctx, |
| 61 | + const Tensor& in, |
| 62 | + optional<ArrayRef<int64_t>> dim_list, |
| 63 | + bool keepdim, |
| 64 | + optional<ScalarType> dtype, |
| 65 | + Tensor& out) { |
| 66 | + ET_KERNEL_CHECK( |
| 67 | + ctx, |
| 68 | + torch::executor::check_mean_dim_args(in, dim_list, keepdim, dtype, out), |
| 69 | + InvalidArgument, |
| 70 | + out); |
| 71 | + |
| 72 | + ET_KERNEL_CHECK( |
| 73 | + ctx, |
| 74 | + torch::executor::resize_reduction_out(in, dim_list, keepdim, out) == |
| 75 | + Error::Ok, |
| 76 | + InvalidArgument, |
| 77 | + out); |
| 78 | + |
| 79 | + constexpr auto name = "mean.out"; |
| 80 | + constexpr int kNnlibMaxDim = 4; |
| 81 | + |
| 82 | + bool optimized = 1; |
| 83 | + |
| 84 | + if (out.scalar_type() != ScalarType::Float) |
| 85 | + optimized = 0; |
| 86 | + |
| 87 | + if (in.dim() > kNnlibMaxDim) |
| 88 | + optimized = 0; |
| 89 | + |
| 90 | + if (optimized) { |
| 91 | + float* __restrict__ p_out = out.mutable_data_ptr<float>(); |
| 92 | + const float* __restrict__ p_inp = |
| 93 | + (const float* __restrict__)in.const_data_ptr<float>(); |
| 94 | + |
| 95 | + int num_elm = in.numel(); |
| 96 | + |
| 97 | + int num_inp_dims = in.dim(); |
| 98 | + int num_out_dims = out.dim(); |
| 99 | + |
| 100 | + int inp_shape[kNnlibMaxDim]; |
| 101 | + int out_shape[kNnlibMaxDim]; |
| 102 | + int p_axis[kNnlibMaxDim]; |
| 103 | + |
| 104 | + for (int i = 0; i < kNnlibMaxDim; i++) { |
| 105 | + out_shape[i] = 1; |
| 106 | + inp_shape[i] = 1; |
| 107 | + p_axis[i] = 1; |
| 108 | + } |
| 109 | + |
| 110 | + int num_axis_dims = prepare_data( |
| 111 | + in, |
| 112 | + out, |
| 113 | + dim_list, |
| 114 | + inp_shape, |
| 115 | + out_shape, |
| 116 | + p_axis, |
| 117 | + num_inp_dims, |
| 118 | + num_out_dims); |
| 119 | + |
| 120 | + if (num_axis_dims == num_inp_dims) { |
| 121 | + num_out_dims = 1; |
| 122 | + out_shape[0] = 1; |
| 123 | + } |
| 124 | + |
| 125 | + int scratch_size = xa_nn_reduce_getsize_nhwc( |
| 126 | + -3, inp_shape, num_inp_dims, p_axis, num_axis_dims, 1); |
| 127 | + |
| 128 | + void* __restrict__ p_scratch_in = (void* __restrict__)malloc(scratch_size); |
| 129 | + |
| 130 | + xa_nn_reduce_mean_4D_f32_f32( |
| 131 | + p_out, |
| 132 | + out_shape, |
| 133 | + p_inp, |
| 134 | + inp_shape, |
| 135 | + p_axis, |
| 136 | + num_out_dims, |
| 137 | + num_inp_dims, |
| 138 | + num_axis_dims, |
| 139 | + p_scratch_in); |
| 140 | + |
| 141 | + return out; |
| 142 | + } |
| 143 | + |
| 144 | + ET_SWITCH_REALHB_TYPES(in.scalar_type(), ctx, name, CTYPE_IN, [&] { |
| 145 | + ET_SWITCH_FLOATH_TYPES(out.scalar_type(), ctx, name, CTYPE_OUT, [&] { |
| 146 | + CTYPE_OUT* out_data = out.mutable_data_ptr<CTYPE_OUT>(); |
| 147 | + const size_t num = torch::executor::get_reduced_dim_product(in, dim_list); |
| 148 | + |
| 149 | + for (size_t out_ix = 0; out_ix < out.numel(); ++out_ix) { |
| 150 | + CTYPE_OUT sum = 0; |
| 151 | + if (in.numel() > 0) { |
| 152 | + sum = torch::executor::map_reduce_over_dim_list<CTYPE_IN, CTYPE_OUT>( |
| 153 | + [](CTYPE_IN v) { return static_cast<CTYPE_OUT>(v); }, |
| 154 | + [](CTYPE_OUT outv, CTYPE_OUT acc) { return acc + outv; }, |
| 155 | + in, |
| 156 | + dim_list, |
| 157 | + out_ix); |
| 158 | + } |
| 159 | + out_data[out_ix] = sum / static_cast<float>(num); |
| 160 | + } |
| 161 | + }); |
| 162 | + }); |
| 163 | + |
| 164 | + return out; |
| 165 | +} |
| 166 | + |
| 167 | +} // namespace native |
| 168 | +} // namespace HiFi |
| 169 | +} // namespace impl |
| 170 | +} // namespace cadence |
0 commit comments