Update ArmNN and ACL pins to 21.08 and change naming in source from tflite to armnn_tflite where applicable to reflect new repository name.

Remove gpu tuning level as option in gpu execution accelerator params for armnn.

Author: jishminor

CMakeLists.txt

@@ -1,24 +1,27 @@
 cmake_minimum_required (VERSION 3.18)
 
-project(tritontflitebackend LANGUAGES C CXX)
+project(tritonarmnntflitebackend LANGUAGES C CXX)
 
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE Release)
 endif()
 
 set(TARGET_ARCH ${CMAKE_HOST_SYSTEM_PROCESSOR})
 
+# Triton Options
 option(TRITON_ENABLE_STATS "Include statistics collections in backend" ON)
 option(TRITON_ENABLE_MALI_GPU "Enable Arm MALI GPU support in backend utilities" OFF)
 
 set(TRITON_BACKEND_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/backend repo")
 set(TRITON_CORE_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo")
 set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo")
 
+# TFLite Options
 set(TFLITE_TAG "v2.4.1" CACHE STRING "Version of TFLite to build")
+option(TFLITE_ENABLE_RUY "Use RUY library in TFLite build" ON)
 
 # ArmNN Options
-set(ARMNN_TAG "v21.05" CACHE STRING "Tag for ArmNN repo")
+set(ARMNN_TAG "v21.08" CACHE STRING "Tag for ArmNN repo")
 set(ARMNN_BUILD_TYPE ${CMAKE_BUILD_TYPE} CACHE STRING "Set ArmNN build type")
 option(ARMNN_DELEGATE_ENABLE "Build ArmNN Delegate" ON)
 
@@ -56,7 +59,7 @@ if (NOT DEFINED ACL_ARCH)
 endif()
 
 # ACL before 21.05 compiled with only gcc7
-if (NOT(ACL_TAG STREQUAL "v21.05"))
+if (NOT((ACL_TAG STREQUAL "v21.05") OR (ACL_TAG STREQUAL "v21.08")))
   if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7)
     set(ACL_CC_COMPILER "gcc-7")
     set(ACL_CXX_COMPILER "g++-7")
@@ -105,7 +108,7 @@ FetchContent_MakeAvailable(repo-common repo-core repo-backend tensorflow-lite)
 #
 # Shared library implementing the Triton Backend API
 #
-configure_file(src/libtriton_tflite.ldscript libtriton_tflite.ldscript COPYONLY)
+configure_file(src/libtriton_armnn_tflite.ldscript libtriton_armnn_tflite.ldscript COPYONLY)
 
 include(ExternalProject)
 
@@ -132,6 +135,7 @@ if (ARMNN_DELEGATE_ENABLE)
     GIT_REPOSITORY https://review.mlplatform.org/ml/ComputeLibrary
     GIT_TAG ${ACL_TAG}
     GIT_SHALLOW ON
+    PATCH_COMMAND git apply ${CMAKE_CURRENT_SOURCE_DIR}/patches/acl.patch || true
     CONFIGURE_COMMAND ""
     BUILD_COMMAND CC=${ACL_CC_COMPILER} CXX=${ACL_CXX_COMPILER} scons -j${JOBS} arch=${ACL_ARCH} debug=${ACL_DEBUG} neon=${ACL_ENABLE_NEON} opencl=${ACL_ENABLE_CL} embed_kernels=1 extra_cxx_flags=-fPIC benchmark_tests=0 build=native validation_tests=0 internal_only=0 examples=0
     BUILD_IN_SOURCE ON
@@ -156,32 +160,32 @@ endif()
 #
 
 add_library(
-  triton-tflite-backend SHARED
+  triton-armnn-tflite-backend SHARED
   src/tflite.cc
   src/tflite_utils.cc
   src/tflite_utils.h
 )
 
 if (ARMNN_DELEGATE_ENABLE)
-  add_dependencies(triton-tflite-backend armnn)
+  add_dependencies(triton-armnn-tflite-backend armnn)
 endif()
 
 add_library(
-  TritonTFLiteBackend::triton-tflite-backend ALIAS triton-tflite-backend
+  TritonArmNNTFLiteBackend::triton-armnn-tflite-backend ALIAS triton-armnn-tflite-backend
 )
 
 target_include_directories(
-  triton-tflite-backend
+  triton-armnn-tflite-backend
   PRIVATE
     ${CMAKE_CURRENT_SOURCE_DIR}/src
     ${CMAKE_CURRENT_BINARY_DIR}/_deps/tensorflow-lite-src
     ${ARMNN_LOCATION}/include # for armnn headers
     ${ARMNN_LOCATION}/src/armnn/delegate/include # for delegate headers
 )
 
-target_compile_features(triton-tflite-backend PRIVATE cxx_std_11)
+target_compile_features(triton-armnn-tflite-backend PRIVATE cxx_std_11)
 target_compile_options(
-  triton-tflite-backend PRIVATE
+  triton-armnn-tflite-backend PRIVATE
   $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>:
     -Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror>
 )
@@ -195,20 +199,20 @@ if(${ARMNN_DELEGATE_ENABLE})
 endif() # ARMNN_DELEGATE_ENABLE
 
 set_target_properties(
-  triton-tflite-backend
+  triton-armnn-tflite-backend
   PROPERTIES
     POSITION_INDEPENDENT_CODE ON
-    OUTPUT_NAME triton_tflite
+    OUTPUT_NAME triton_armnn_tflite
     SKIP_BUILD_RPATH TRUE
     BUILD_WITH_INSTALL_RPATH TRUE
     INSTALL_RPATH_USE_LINK_PATH FALSE
     INSTALL_RPATH "$\{ORIGIN\}"
-    LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_tflite.ldscript
-    LINK_FLAGS "-Wl,--no-as-needed,--version-script libtriton_tflite.ldscript"
+    LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_armnn_tflite.ldscript
+    LINK_FLAGS "-Wl,--no-as-needed,--version-script libtriton_armnn_tflite.ldscript"
 )
 
 target_link_libraries(
-  triton-tflite-backend
+  triton-armnn-tflite-backend
   PRIVATE
     triton-core-serverapi  # from repo-core
     triton-core-backendapi # from repo-core
@@ -220,7 +224,7 @@ target_link_libraries(
 
 if (ARMNN_DELEGATE_ENABLE)
   target_link_libraries(
-    triton-tflite-backend
+    triton-armnn-tflite-backend
     PRIVATE
       "-L${ARMNN_LOCATION}/lib" # from armnn
       -larmnn
@@ -232,31 +236,31 @@ endif()
 # Install
 #
 include(GNUInstallDirs)
-set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TritonTFLiteBackend)
+set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TritonArmNNTFLiteBackend)
 
 install(
   TARGETS
-    triton-tflite-backend
+    triton-armnn-tflite-backend
   EXPORT
-    triton-tflite-backend-targets
-  LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/tflite
-  ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/tflite
+    triton-armnn-tflite-backend-targets
+  LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/armnn_tflite
+  ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/armnn_tflite
 )
 
 if (ARMNN_DELEGATE_ENABLE)
   # Install ArmNN libraries and license
   install(
     DIRECTORY
       ${ARMNN_LOCATION}/lib/
-    DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/tflite
+    DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/armnn_tflite
     FILES_MATCHING PATTERN "*.so*"
   )
 
   install(
     FILES
       ${ARMNN_LOCATION}/src/armnn/LICENSE
     RENAME armnn.LICENSE
-    DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/tflite
+    DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/armnn_tflite
   )
 endif()
 
@@ -265,40 +269,40 @@ install(
   FILES
   ${CMAKE_CURRENT_BINARY_DIR}/_deps/tensorflow-lite-src/LICENSE
   RENAME tensorflow.LICENSE
-  DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/tflite
+  DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/armnn_tflite
 )
 
 install(
   EXPORT
-    triton-tflite-backend-targets
+    triton-armnn-tflite-backend-targets
   FILE
-    TritonTFLiteBackendTargets.cmake
+    TritonArmNNTFLiteBackendTargets.cmake
   NAMESPACE
-    TritonTFLiteBackend::
+    TritonArmNNTFLiteBackend::
   DESTINATION
     ${INSTALL_CONFIGDIR}
 )
 
 include(CMakePackageConfigHelpers)
 configure_package_config_file(
-  ${CMAKE_CURRENT_LIST_DIR}/cmake/TritonTFLiteBackendConfig.cmake.in
-  ${CMAKE_CURRENT_BINARY_DIR}/TritonTFLiteBackendConfig.cmake
+  ${CMAKE_CURRENT_LIST_DIR}/cmake/TritonArmNNTFLiteBackendConfig.cmake.in
+  ${CMAKE_CURRENT_BINARY_DIR}/TritonArmNNTFLiteBackendConfig.cmake
   INSTALL_DESTINATION ${INSTALL_CONFIGDIR}
 )
 
 install(
   FILES
-  ${CMAKE_CURRENT_BINARY_DIR}/TritonTFLiteBackendConfig.cmake
+  ${CMAKE_CURRENT_BINARY_DIR}/TritonArmNNTFLiteBackendConfig.cmake
   DESTINATION ${INSTALL_CONFIGDIR}
 )
 
 #
 # Export from build tree
 #
 export(
-  EXPORT triton-tflite-backend-targets
-  FILE ${CMAKE_CURRENT_BINARY_DIR}/TritonTFLiteBackendTargets.cmake
-  NAMESPACE TritonTFLiteBackend::
+  EXPORT triton-armnn-tflite-backend-targets
+  FILE ${CMAKE_CURRENT_BINARY_DIR}/TritonArmNNTFLiteBackendTargets.cmake
+  NAMESPACE TritonArmNNTFLiteBackend::
 )
 
-export(PACKAGE TritonTFLiteBackend)
+export(PACKAGE TritonArmNNTFLiteBackend)

Dockerfile

@@ -1,6 +1,6 @@
 ARG UBUNTU_VERSION=20.04
 
-FROM ubuntu:${UBUNTU_VERSION} as tflite_backend
+FROM ubuntu:${UBUNTU_VERSION} as armnn_tflite_backend
 
 # Triton version pins, assumed same across backend, core, and common
 ARG TRITON_REPO_TAG=main
@@ -12,18 +12,18 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && \
     apt-get install -yqq --no-install-recommends \
-        git \
-        wget \
-        scons \
-        ca-certificates \
-        curl \
-        autoconf \
-        libtool \
-        build-essential \
-        libssl-dev \
-        xxd \
-        rapidjson-dev \
-        unzip
+    git \
+    wget \
+    scons \
+    ca-certificates \
+    curl \
+    autoconf \
+    libtool \
+    build-essential \
+    libssl-dev \
+    xxd \
+    rapidjson-dev \
+    unzip
 
 # Install cmake from source
 RUN build=1 && \
@@ -36,18 +36,19 @@ RUN build=1 && \
     make -j$(nproc) && \
     make install
 
-# Build TFLite Backend
-WORKDIR /opt/tflite_backend
+# Build ArmNN TFLite Backend
+WORKDIR /opt/armnn_tflite_backend
 COPY . .
 RUN mkdir build && \
     cd build && \
     cmake .. \
-        -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install \
-        -DTRITON_BACKEND_REPO_TAG=${TRITON_REPO_TAG} \
-        -DTRITON_CORE_REPO_TAG=${TRITON_REPO_TAG} \
-        -DTRITON_COMMON_REPO_TAG=${TRITON_REPO_TAG} \
-        -DTRITON_ENABLE_GPU=OFF \
-        -DTRITON_ENABLE_MALI_GPU=ON \
-        -DJOBS=$(nproc) \
+    -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install \
+    -DTRITON_BACKEND_REPO_TAG=${TRITON_REPO_TAG} \
+    -DTRITON_CORE_REPO_TAG=${TRITON_REPO_TAG} \
+    -DTRITON_COMMON_REPO_TAG=${TRITON_REPO_TAG} \
+    -DTRITON_ENABLE_GPU=OFF \
+    -DTRITON_ENABLE_MALI_GPU=ON \
+    -DTFLITE_ENABLE_RUY=ON \
+    -DJOBS=$(nproc) \
     && \
     make -j$(nproc) install

README.md

@@ -1,6 +1,6 @@
-# TFLite Backend
+# ArmNN TFLite Backend
 
-The Triton backend for [TFLite](https://www.tensorflow.org/lite). 
+The Triton backend for [TFLite](https://www.tensorflow.org/lite) with support for ArmNN acceleration. 
 You can learn more about Triton backends in the [backend
 repo](https://github.com/triton-inference-server/backend). Ask
 questions or report problems on the [issues
@@ -12,15 +12,15 @@ This backend was developed using the existing [Triton PyTorch Backend](https://g
 
 This backend is only currently available for **linux arm64** platforms.
 
-## Build the TFLite Backend
-The TFLite backend can be built either integrated with the build process for the [triton server repo](https://github.com/triton-inference-server/server) or it may be built independently using only this repository.
+## Build the ArmNN TFLite Backend
+The ArmNN TFLite backend can be built either integrated with the build process for the [triton server repo](https://github.com/triton-inference-server/server) or it may be built independently using only this repository.
 
 ### Build with Triton Build Convenience Script
-The easiest way to get up and running with the triton tflite backend is to build a custom triton docker image using the `build.py` script available in the triton server repo. 
+The easiest way to get up and running with the triton armnn tflite backend is to build a custom triton docker image using the `build.py` script available in the triton server repo. 
 
-To build a triton server docker image with the tflite backend built in simply run the following command from the root of the server repo:
+To build a triton server docker image with the armnn tflite backend built in simply run the following command from the root of the server repo:
 ```bash
-./build.py --cmake-dir=/workspace/build --build-dir=/tmp/citritonbuild --target-platform=ubuntu/arm64 --enable-logging --enable-stats --enable-tracing --enable-metrics --endpoint=http --endpoint=grpc --backend=tflite
+./build.py --cmake-dir=/workspace/build --build-dir=/tmp/citritonbuild --target-platform=ubuntu/arm64 --enable-logging --enable-stats --enable-tracing --enable-metrics --endpoint=http --endpoint=grpc --backend=armnn_tflite
 ```
 
 ### Build Independently with CMake
@@ -142,11 +142,9 @@ optimization { execution_accelerators {
     parameters { key: "reduce_fp32_to_fp16" value: "<on/off>" }
     parameters { key: "reduce_fp32_to_bf16" value: "<on/off>" }
     parameters { key: "fast_math_enabled" value: "<on/off>" }
-    parameters { key: "tuning_level" value: "<0-3>" }
   }]
 }}
 ```
-Note that for MALI GPU tuning level the value corresponds to the following: `(0=UseOnly(default) | 1=RapidTuning | 2=NormalTuning | 3=ExhaustiveTuning)`
 
 ### XNNPACK Delegate Optimization Options
 Users also have the ability to specify XNNPACK specific optimizations. 
@@ -159,8 +157,8 @@ optimization { execution_accelerators {
 }}
 ```
 
-## Running TFLite Backend on MALI GPU
-The best way to run the TFLite backend on a platform with a MALI GPU is via Docker. For example on a hikey 970, we can run the following after building our custom tritonserver image using the command from the build with convenience script above:
+## Running ArmNN TFLite Backend on MALI GPU
+The best way to run the ArmNN TFLite backend on a platform with a MALI GPU is via Docker. For example on a hikey 970, we can run the following after building our custom tritonserver image using the command from the build with convenience script above:
 ```
 docker run --rm -it --device /dev/mali0 -v /usr/lib/aarch64-linux-gnu/libmali.so:/usr/lib/aarch64-linux-gnu/libmali.so -v <full path to your model repo on host>:/models -p 8000:8000 -p 8001:8001 -p 8002:8002 tritonserver:latest
 ```

cmake/TritonTFLiteBackendConfig.cmake.in renamed to cmake/TritonArmNNTFLiteBackendConfig.cmake.in

@@ -27,13 +27,13 @@
 include(CMakeFindDependencyMacro)
 
 get_filename_component(
-  TRITONTFLITEBACKEND_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH
+  TRITONARMNNTFLITEBACKEND_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH
 )
 
-list(APPEND CMAKE_MODULE_PATH ${TRITONTFLITEBACKEND_CMAKE_DIR})
+list(APPEND CMAKE_MODULE_PATH ${TRITONARMNNTFLITEBACKEND_CMAKE_DIR})
 
-if(NOT TARGET TritonTFLiteBackend::triton-tflite-backend)
-  include("${TRITONTFLITEBACKEND_CMAKE_DIR}/TritonTFLiteBackendTargets.cmake")
+if(NOT TARGET TritonArmNNTFLiteBackend::triton-armnn-tflite-backend)
+  include("${TRITONARMNNTFLITEBACKEND_CMAKE_DIR}/TritonArmNNTFLiteBackendTargets.cmake")
 endif()
 
-set(TRITONTFLITEBACKEND_LIBRARIES TritonTFLiteBackend::triton-tflite-backend)
+set(TRITONARMNNTFLITEBACKEND_LIBRARIES TritonArmNNTFLiteBackend::triton-armnn-tflite-backend)

patches/acl.patch

@@ -0,0 +1,13 @@
+diff --git a/arm_compute/core/Size2D.h b/arm_compute/core/Size2D.h
+index bcd89cb31..03bd1eaee 100644
+--- a/arm_compute/core/Size2D.h
++++ b/arm_compute/core/Size2D.h
+@@ -41,7 +41,7 @@ public:
+      * @param[in] w Width of the image or rectangle
+      * @param[in] h Height of the image or rectangle
+      */
+-    Size2D(size_t w, size_t h)
++    Size2D(size_t w, size_t h) noexcept
+         : width(w), height(h)
+     {
+     }

patches/xnnpack_commit.patch

@@ -7,7 +7,7 @@ index e5f205c0b0b..df2f7345136 100644
    xnnpack
    GIT_REPOSITORY https://github.com/google/xnnpack
 -  GIT_TAG 0af63ab36b899559bd1a92bbc327f8137e53c15c
-+  GIT_TAG 4422f106e1aa572e80d69c948c36e01966932fad
++  GIT_TAG be3d8fdffe8eb71b80835f168d4a550bb5d80f12
    GIT_PROGRESS TRUE
    PREFIX "${CMAKE_BINARY_DIR}"
    SOURCE_DIR "${CMAKE_BINARY_DIR}/xnnpack"

qa/config-template.pbtxt

@@ -1,5 +1,5 @@
 name: "{{ model.name }}"
-backend: "tflite"
+backend: "armnn_tflite"
 max_batch_size: {{ model.max_batch_size }}
 input [
     {% for input in model.inputs %}