Sync from tflite-micro. (tensorflow#176)

TFLM-bot · web-flow · commit 1f0bd2ea07b5 · 2023-02-19T08:50:31.000-08:00
diff --git a/src/tensorflow/lite/c/builtin_op_data.h b/src/tensorflow/lite/c/builtin_op_data.h
@@ -1,4 +1,4 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,8 +15,6 @@ limitations under the License.
 #ifndef TENSORFLOW_LITE_C_BUILTIN_OP_DATA_H_
 #define TENSORFLOW_LITE_C_BUILTIN_OP_DATA_H_
 
-/// For documentation, see
-/// third_party/tensorflow/lite/core/c/builtin_op_data.h.
 #include "tensorflow/lite/core/c/builtin_op_data.h"
 
 #endif  // TENSORFLOW_LITE_C_BUILTIN_OP_DATA_H_
diff --git a/src/tensorflow/lite/c/c_api_types.h b/src/tensorflow/lite/c/c_api_types.h
@@ -1,4 +1,4 @@
-/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -12,15 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-
-// This file declares types used by the pure C inference API defined in c_api.h,
-// some of which are also used in the C++ and C kernel and interpreter APIs.
-
 #ifndef TENSORFLOW_LITE_C_C_API_TYPES_H_
 #define TENSORFLOW_LITE_C_C_API_TYPES_H_
 
-/// For documentation, see
-/// third_party/tensorflow/lite/core/c/c_api_types.h.
 #include "tensorflow/lite/core/c/c_api_types.h"
 
 #endif  // TENSORFLOW_LITE_C_C_API_TYPES_H_
diff --git a/src/tensorflow/lite/c/common.h b/src/tensorflow/lite/c/common.h
@@ -36,8 +36,12 @@ limitations under the License.
 #ifndef TENSORFLOW_LITE_C_COMMON_H_
 #define TENSORFLOW_LITE_C_COMMON_H_
 
-/// For documentation, see
-/// third_party/tensorflow/lite/core/c/common.h.
 #include "tensorflow/lite/core/c/common.h"
 
+// TfLiteOpaqueDelegate: allows delegation of nodes to alternative backends.
+// TfLiteOpaqueDelegate is an abstract type that is intended to have the same
+// role as TfLiteDelegate, but without necessarily exposing the implementation
+// details of how delegates are implemented.
+typedef TfLiteDelegate TfLiteOpaqueDelegate;
+
 #endif  // TENSORFLOW_LITE_C_COMMON_H_
diff --git a/src/tensorflow/lite/micro/kernels/activations_common.cpp b/src/tensorflow/lite/micro/kernels/activations_common.cpp
@@ -55,8 +55,8 @@ void CalculateReluOpData(const TfLiteTensor* input, TfLiteTensor* output,
                          ReluOpData* data) {
   float act_min = 0.0;
   float act_max = std::numeric_limits<float>::infinity();
-  double real_multiplier = static_cast<double>(input->params.scale) /
-                           static_cast<double>(output->params.scale);
+  double real_multiplier =
+      static_cast<double>(input->params.scale / output->params.scale);
 
   const RuntimeShape input_shape = GetTensorShape(input);
   const RuntimeShape output_shape = GetTensorShape(output);
diff --git a/src/tensorflow/lite/micro/kernels/cmsis_nn/conv.cpp b/src/tensorflow/lite/micro/kernels/cmsis_nn/conv.cpp
@@ -20,7 +20,6 @@ limitations under the License.
 #include "tensorflow/lite/c/builtin_op_data.h"
 #include "tensorflow/lite/c/common.h"
 #include "tensorflow/lite/kernels/internal/common.h"
-#include "tensorflow/lite/kernels/internal/portable_tensor_utils.h"
 #include "tensorflow/lite/kernels/internal/quantization_util.h"
 #include "tensorflow/lite/kernels/internal/reference/conv.h"
 #include "tensorflow/lite/kernels/internal/reference/integer_ops/conv.h"
@@ -366,9 +365,11 @@ TfLiteStatus EvalInt8(TfLiteContext* context, TfLiteNode* node) {
       *(reinterpret_cast<TfLiteConvParams*>(node->builtin_data));
   TFLITE_DCHECK(node->user_data != nullptr);
   const OpData& data = *(static_cast<const OpData*>(node->user_data));
+  TfLiteEvalTensor filter_int8 = tflite::micro::MakeUnpackedInt4Tensor(
+      context, data.reference_op_data.filter_buffer_index, filter);
 
-  return EvalQuantizedPerChannel(context, node, params, data, input, filter,
-                                 bias, output);
+  return EvalQuantizedPerChannel(context, node, params, data, input,
+                                 &filter_int8, bias, output);
 }
 
 TfLiteStatus EvalInt16x8(TfLiteContext* context, TfLiteNode* node) {
@@ -419,22 +420,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
           (input->type == kTfLiteInt8 && filter->type == kTfLiteInt4),
       "Hybrid models are not supported on TFLite Micro.");
 
-  TfLiteEvalTensor filter_int8;
-
-  if (filter->type == kTfLiteInt4) {
-    filter_int8.data.data = static_cast<int8_t*>(context->GetScratchBuffer(
-        context, data.reference_op_data.filter_buffer_index));
-
-    filter_int8.dims = filter->dims;
-    filter_int8.type = kTfLiteInt8;
-    tflite::tensor_utils::UnpackDenseInt4IntoInt8(
-        tflite::micro::GetTensorData<int8_t>(filter),
-        tflite::micro::GetTensorShape(filter).FlatSize(),
-        tflite::micro::GetTensorData<int8_t>(&filter_int8));
-
-  } else {
-    filter_int8 = *filter;
-  }
+  TfLiteEvalTensor filter_int8 = tflite::micro::MakeUnpackedInt4Tensor(
+      context, data.reference_op_data.filter_buffer_index, filter);
 
   switch (input->type) {  // Already know in/out types are same.
     case kTfLiteFloat32: {
diff --git a/src/tensorflow/lite/micro/kernels/cmsis_nn/depthwise_conv.cpp b/src/tensorflow/lite/micro/kernels/cmsis_nn/depthwise_conv.cpp
@@ -19,7 +19,6 @@ limitations under the License.
 #include "tensorflow/lite/c/builtin_op_data.h"
 #include "tensorflow/lite/c/common.h"
 #include "tensorflow/lite/kernels/internal/common.h"
-#include "tensorflow/lite/kernels/internal/portable_tensor_utils.h"
 #include "tensorflow/lite/kernels/internal/quantization_util.h"
 #include "tensorflow/lite/kernels/internal/reference/depthwiseconv_float.h"
 #include "tensorflow/lite/kernels/internal/reference/integer_ops/depthwise_conv.h"
@@ -337,20 +336,9 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
       (NumInputs(node) == 3)
           ? tflite::micro::GetEvalInput(context, node, kDepthwiseConvBiasTensor)
           : nullptr;
-  TfLiteEvalTensor filter_int8;
 
-  if (filter->type == kTfLiteInt4) {
-    filter_int8.data.data = static_cast<int8_t*>(context->GetScratchBuffer(
-        context, data.reference_op_data.filter_buffer_index));
-    filter_int8.dims = filter->dims;
-    filter_int8.type = kTfLiteInt8;
-    tflite::tensor_utils::UnpackDenseInt4IntoInt8(
-        tflite::micro::GetTensorData<int8_t>(filter),
-        tflite::micro::GetTensorShape(filter).FlatSize(),
-        tflite::micro::GetTensorData<int8_t>(&filter_int8));
-  } else {
-    filter_int8 = *filter;
-  }
+  TfLiteEvalTensor filter_int8 = tflite::micro::MakeUnpackedInt4Tensor(
+      context, data.reference_op_data.filter_buffer_index, filter);
 
   switch (input->type) {  // Already know in/out types are same.
     case kTfLiteFloat32: {
@@ -411,8 +399,11 @@ TfLiteStatus EvalInt8(TfLiteContext* context, TfLiteNode* node) {
           ? tflite::micro::GetEvalInput(context, node, kDepthwiseConvBiasTensor)
           : nullptr;
 
-  EvalQuantizedPerChannel(context, node, params, data, input, filter, bias,
-                          output);
+  TfLiteEvalTensor filter_int8 = tflite::micro::MakeUnpackedInt4Tensor(
+      context, data.reference_op_data.filter_buffer_index, filter);
+
+  EvalQuantizedPerChannel(context, node, params, data, input, &filter_int8,
+                          bias, output);
   return kTfLiteOk;
 }
 
diff --git a/src/tensorflow/lite/micro/kernels/cmsis_nn/fully_connected.cpp b/src/tensorflow/lite/micro/kernels/cmsis_nn/fully_connected.cpp
@@ -319,22 +319,9 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   TFLITE_DCHECK(node->user_data != nullptr);
   const OpData& data = *(static_cast<const OpData*>(node->user_data));
 
-  TfLiteEvalTensor filter_int8;
+  TfLiteEvalTensor filter_int8 = tflite::micro::MakeUnpackedInt4Tensor(
+      context, data.reference_op_data.filter_buffer_index, filter);
 
-  if (filter->type == kTfLiteInt4) {
-    filter_int8.data.data = static_cast<int8_t*>(context->GetScratchBuffer(
-        context, data.reference_op_data.filter_buffer_index));
-
-    filter_int8.dims = filter->dims;
-    filter_int8.type = kTfLiteInt8;
-    tflite::tensor_utils::UnpackDenseInt4IntoInt8(
-        tflite::micro::GetTensorData<int8_t>(filter),
-        tflite::micro::GetTensorShape(filter).FlatSize(),
-        tflite::micro::GetTensorData<int8_t>(&filter_int8));
-
-  } else {
-    filter_int8 = *filter;
-  }
   // Checks in Prepare ensure input, output and filter types are all the same.
   switch (input->type) {
     case kTfLiteFloat32: {
@@ -352,8 +339,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
       break;
     }
     case kTfLiteInt8: {
-      switch (filter->type) {
-        case kTfLiteInt4:
+      switch (filter_int8.type) {
         case kTfLiteInt8:
           return EvalQuantizedInt8(context, node, data, input, &filter_int8,
                                    bias, output);
@@ -403,7 +389,11 @@ TfLiteStatus EvalInt8(TfLiteContext* context, TfLiteNode* node) {
     return kTfLiteError;
   }
 
-  return EvalQuantizedInt8(context, node, data, input, filter, bias, output);
+  TfLiteEvalTensor filter_int8 = tflite::micro::MakeUnpackedInt4Tensor(
+      context, data.reference_op_data.filter_buffer_index, filter);
+
+  return EvalQuantizedInt8(context, node, data, input, &filter_int8, bias,
+                           output);
 }
 
 TfLiteStatus EvalInt16(TfLiteContext* context, TfLiteNode* node) {
diff --git a/src/tensorflow/lite/micro/kernels/fully_connected.h b/src/tensorflow/lite/micro/kernels/fully_connected.h
@@ -73,7 +73,7 @@ TfLiteStatus CalculateOpDataFullyConnected(
 // (reference or optimized) must define this function.
 TfLiteRegistration Register_FULLY_CONNECTED();
 
-#if defined(ARDUINO) || defined(HEXAGON)
+#if defined(ARDUINO) || defined(HEXAGON) || defined(XTENSA)
 // Returns a TfLiteRegistration struct for kernel variant that only supports
 // int8.
 TfLiteRegistration Register_FULLY_CONNECTED_INT8();
diff --git a/src/tensorflow/lite/micro/kernels/fully_connected_common.cpp b/src/tensorflow/lite/micro/kernels/fully_connected_common.cpp
@@ -64,12 +64,13 @@ TfLiteStatus CalculateOpDataFullyConnected(
     QuantizeMultiplier(real_multiplier, &data->output_multiplier,
                        &data->output_shift);
 
-    data->input_zero_point = input->params.zero_point;
     // Filter weights will always be symmetric quantized since we only support
     // int8 quantization. See
     // https://github.com/tensorflow/tensorflow/issues/44912 for additional
     // context.
     TFLITE_DCHECK(filter->params.zero_point == 0);
+
+    data->input_zero_point = input->params.zero_point;
     data->filter_zero_point = filter->params.zero_point;
     data->output_zero_point = output->params.zero_point;
 
diff --git a/src/tensorflow/lite/micro/kernels/kernel_util.cpp b/src/tensorflow/lite/micro/kernels/kernel_util.cpp
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/lite/micro/kernels/kernel_util.h"
 
 #include "tensorflow/lite/c/common.h"
+#include "tensorflow/lite/kernels/internal/portable_tensor_utils.h"
 #include "tensorflow/lite/micro/memory_helpers.h"
 #include "tensorflow/lite/micro/micro_log.h"
 
@@ -256,5 +257,24 @@ TfLiteStatus CopySubgraphOutputsToOpOutputs(TfLiteContext* context,
   return kTfLiteOk;
 }
 
+TfLiteEvalTensor MakeUnpackedInt4Tensor(TfLiteContext* context,
+                                        int scratch_buffer_index,
+                                        const TfLiteEvalTensor* tensor) {
+  if (tensor->type != kTfLiteInt4) {
+    return *tensor;
+  }
+
+  TfLiteEvalTensor new_tensor;
+  new_tensor.data.data = static_cast<int8_t*>(
+      context->GetScratchBuffer(context, scratch_buffer_index));
+  new_tensor.dims = tensor->dims;
+  new_tensor.type = kTfLiteInt8;
+  tflite::tensor_utils::UnpackDenseInt4IntoInt8(
+      tflite::micro::GetTensorData<int8_t>(tensor),
+      tflite::micro::GetTensorShape(tensor).FlatSize(),
+      tflite::micro::GetTensorData<int8_t>(&new_tensor));
+  return new_tensor;
+}
+
 }  // namespace micro
 }  // namespace tflite
diff --git a/src/tensorflow/lite/micro/kernels/kernel_util.h b/src/tensorflow/lite/micro/kernels/kernel_util.h
@@ -131,6 +131,14 @@ TfLiteStatus CopySubgraphOutputsToOpOutputs(TfLiteContext* context,
                                             MicroGraph* graph_info,
                                             int subgraph_idx);
 
+// If tensor is INT4, make a new TfLiteEvalTensor with data unpacked into
+// a scratch buffer. The returned tensor will have the kTfLiteInt8 type.
+// Assume scratch buffer is previously requested in Prepare, and
+// scratch_buffer_index can be used to retrieve that buffer.
+// If the tensor is not INT4, a shallow copy is returned.
+TfLiteEvalTensor MakeUnpackedInt4Tensor(TfLiteContext* context,
+                                        int scratch_buffer_index,
+                                        const TfLiteEvalTensor* tensor);
 }  // namespace micro
 }  // namespace tflite
 
diff --git a/src/tensorflow/lite/micro/kernels/sub_common.cpp b/src/tensorflow/lite/micro/kernels/sub_common.cpp
@@ -52,14 +52,12 @@ TfLiteStatus CalculateOpDataSub(TfLiteContext* context, TfLiteSubParams* params,
     const float twice_max_input_scale =
         2 * std::max(input1->params.scale, input2->params.scale);
     const double real_input1_multiplier =
-        static_cast<double>(input1->params.scale) /
-        static_cast<double>(twice_max_input_scale);
+        static_cast<double>(input1->params.scale / twice_max_input_scale);
     const double real_input2_multiplier =
-        static_cast<double>(input2->params.scale) /
-        static_cast<double>(twice_max_input_scale);
+        static_cast<double>(input2->params.scale / twice_max_input_scale);
     const double real_output_multiplier =
-        static_cast<double>(twice_max_input_scale) /
-        ((1 << data->left_shift) * static_cast<double>(output->params.scale));
+        static_cast<double>(twice_max_input_scale /
+                            ((1 << data->left_shift) * output->params.scale));
 
     QuantizeMultiplierSmallerThanOneExp(
         real_input1_multiplier, &data->input1_multiplier, &data->input1_shift);
diff --git a/src/tensorflow/lite/micro/kernels/svdf_common.cpp b/src/tensorflow/lite/micro/kernels/svdf_common.cpp
@@ -451,21 +451,19 @@ TfLiteStatus PrepareSvdf(TfLiteContext* context, TfLiteNode* node) {
 
     TF_LITE_ENSURE_TYPES_EQ(context, output->type, kTfLiteInt8);
 
-    const double effective_scale_1 =
-        static_cast<double>(input->params.scale) *
-        static_cast<double>(weights_feature->params.scale) /
-        static_cast<double>(activation_state->params.scale);
+    const double effective_scale_1 = static_cast<double>(
+        input->params.scale * weights_feature->params.scale /
+        activation_state->params.scale);
     const double effective_scale_2 =
-        static_cast<double>(activation_state->params.scale) *
-        static_cast<double>(weights_time->params.scale) /
-        static_cast<double>(output->params.scale);
+        static_cast<double>(activation_state->params.scale *
+                            weights_time->params.scale / output->params.scale);
 
     // TODO(b/162018098): Use TF_LITE_ENSURE_NEAR when it is ready.
     TF_LITE_ENSURE(
         context,
         std::abs(static_cast<double>(bias->params.scale) -
-                 (static_cast<double>(activation_state->params.scale) *
-                  static_cast<double>(weights_time->params.scale))) < 1e-5);
+                 static_cast<double>(activation_state->params.scale *
+                                     weights_time->params.scale)) < 1e-5);
 
     QuantizeMultiplier(effective_scale_1, &(data->effective_scale_1_a),
                        &(data->effective_scale_1_b));
diff --git a/src/tensorflow/lite/micro/micro_resource_variable.cpp b/src/tensorflow/lite/micro/micro_resource_variable.cpp
@@ -63,6 +63,7 @@ int MicroResourceVariables::CreateIdIfNoneFound(const char* container,
   resource_variables_[resource_id].shared_name = shared_name;
   resource_variables_[resource_id].resource_buffer = nullptr;
   resource_variables_[resource_id].bytes = 0;
+  resource_variables_[resource_id].default_value = 0;
   return resource_id;
 }
 
@@ -96,9 +97,17 @@ TfLiteStatus MicroResourceVariables::Allocate(int id, TfLiteContext* context,
       MicroPrintf("Failed to allocate resource buffer.");
       return kTfLiteError;
     }
-    // Zero out resource buffers by deafult. Buffers can be initialized to
-    // nonzero values using ASSIGN_VARIABLE.
-    memset(variable.resource_buffer, 0, variable.bytes);
+    // Set resource buffers to the zero_point by default. Buffers can be
+    // initialized to nonzero values using ASSIGN_VARIABLE.
+    // See comment#2 in b/269648474 for more details why we use zero_point.
+    if (tensor->quantization.params != nullptr) {
+      auto* quantization_data = reinterpret_cast<TfLiteAffineQuantization*>(
+          tensor->quantization.params);
+      int8_t zero_point = quantization_data->zero_point[0].data[0];
+      variable.default_value = zero_point;
+    }
+    // TODO(b/269669735): Explains why casting zero_point to int8 and memset.
+    memset(variable.resource_buffer, variable.default_value, variable.bytes);
   }
 
   return kTfLiteOk;
@@ -127,7 +136,8 @@ TfLiteStatus MicroResourceVariables::Assign(int id,
 TfLiteStatus MicroResourceVariables::ResetAll() {
   for (int i = 0; i < num_resource_variables_; i++) {
     MicroResourceVariable variable = resource_variables_[i];
-    memset(variable.resource_buffer, 0, variable.bytes);
+    // TODO(b/269669735): Explains why casting zero_point to int8 and memset.
+    memset(variable.resource_buffer, variable.default_value, variable.bytes);
   }
   return kTfLiteOk;
 }
diff --git a/src/tensorflow/lite/micro/micro_resource_variable.h b/src/tensorflow/lite/micro/micro_resource_variable.h
@@ -69,6 +69,8 @@ class MicroResourceVariables {
 
     // This is only for verifying read size.
     size_t bytes;
+    // Initialization default value
+    int8_t default_value;
   };
 
   MicroResourceVariables(MicroResourceVariable* variables,
