v23.05 (#16)

Signed-off-by: Josh Minor <[email protected]>

Author: jishminor

.github/workflows/build.yml

@@ -64,12 +64,12 @@ on:
       tflite_tag:
         description: "TFLite tag to build against"
         required: true
-        default: v2.4.1
+        default: v2.10.0
         type: string
-      armnn_tag:
+      ARMNN_VERSION:
         description: "ArmNN tag to build against"
         required: true
-        default: v22.08
+        default: 23.02
         type: string
       enable_armnn_delegate:
         description: "Enable ArmNN Delegate in build"
@@ -89,8 +89,8 @@ env:
   TFLITE_ENABLE_RUY: ${{ inputs.enable_ruy || 'ON' }}
   TFLITE_BAZEL_BUILD: ${{ inputs.enable_bazel || 'OFF' }}
   TFLITE_ENABLE_FLEX_OPS: ${{ inputs.enable_flex_ops || 'OFF' }}
-  TFLITE_TAG: ${{ inputs.tflite_tag || 'v2.4.1' }}
-  ARMNN_TAG: ${{ inputs.armnn_tag || 'v22.08' }}
+  TFLITE_TAG: ${{ inputs.tflite_tag || 'v2.10.0' }}
+  ARMNN_VERSION: ${{ inputs.ARMNN_VERSION || '23.02' }}
   ARMNN_DELEGATE_ENABLE: ${{ inputs.enable_armnn_delegate || 'ON' }}
 
 jobs:
@@ -181,9 +181,9 @@ jobs:
           -DTFLITE_BAZEL_BUILD=${{env.TFLITE_BAZEL_BUILD}} \
           -DTFLITE_ENABLE_FLEX_OPS=${{env.TFLITE_ENABLE_FLEX_OPS}} \
           -DTFLITE_TAG=${{env.TFLITE_TAG}} \
-          -DARMNN_TAG=${{env.ARMNN_TAG}} \
+          -DARMNN_VERSION=${{env.ARMNN_VERSION}} \
           -DARMNN_DELEGATE_ENABLE=${{env.ARMNN_DELEGATE_ENABLE}} \
-          -DACL_TAG=${{env.ARMNN_TAG}} \
+          -DACL_VERSION=${{env.ARMNN_VERSION}} \
           -DJOBS=$(nproc)
 
       - name: Cmake build

.github/workflows/triton_build.yml

@@ -52,6 +52,7 @@ jobs:
 
   build:
     needs: get_triton_branches
+    if: needs.get_triton_branches.outputs.triton_branches != "{}"
     strategy:
       fail-fast: false
       matrix:
@@ -115,3 +116,4 @@ jobs:
           docker tag tritonserver:latest ghcr.io/smarter-project/armnn_tflite_backend/triton:$branch_tag_name
           docker push $image_name
           docker push ghcr.io/smarter-project/armnn_tflite_backend/triton:$branch_tag_name
+          docker rmi $image_name ghcr.io/smarter-project/armnn_tflite_backend/triton:$branch_tag_name tritonserver:latest

CMakeLists.txt

@@ -25,7 +25,7 @@ set(TRITON_COMMON_REPO_TAG
 
 # TFLite Options
 set(TFLITE_TAG
-    "v2.4.1"
+    "v2.10.0"
     CACHE STRING "Version of TFLite to build")
 set(BAZEL_BINARY
     "/usr/bin/bazel"
@@ -36,17 +36,17 @@ option(TFLITE_ENABLE_FLEX_OPS
        "Build tflite with flex ops support (must build with bazel)" OFF)
 
 # ArmNN Options
-set(ARMNN_TAG
-    "v22.08"
+set(ARMNN_VERSION
+    "23.02"
     CACHE STRING "Tag for ArmNN repo")
 set(ARMNN_BUILD_TYPE
     ${CMAKE_BUILD_TYPE}
     CACHE STRING "Set ArmNN build type")
 option(ARMNN_DELEGATE_ENABLE "Build ArmNN Delegate" ON)
 
 # ACL Options
-set(ACL_TAG
-    ${ARMNN_TAG}
+set(ACL_VERSION
+    ${ARMNN_VERSION}
     CACHE STRING "Tag for ACL repo")
 set(ACL_DEBUG
     "0"
@@ -65,7 +65,7 @@ set(ACL_ENABLE_CL
     CACHE STRING "Enable OpenCL Backend in build")
 
 set(FLATBUFFERS_VERSION
-    "1.12.0"
+    "2.0.6"
     CACHE STRING "Version of Flatbuffers to use")
 set(JOBS
     "8"
@@ -101,7 +101,7 @@ if(NOT DEFINED ACL_ARCH)
 endif()
 
 # ACL before 21.05 compiled with only gcc7
-if(NOT (ACL_TAG MATCHES "(v21.(05|08|11))|(v22.(02|05|08))"))
+if(NOT (ACL_VERSION VERSION_GREATER "21.05"))
   if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7)
     set(ACL_CC_COMPILER "gcc-7")
     set(ACL_CXX_COMPILER "g++-7")
@@ -114,8 +114,7 @@ set(CMAKE_C_FLAGS "-funsafe-math-optimizations")
 
 set(ACL_BUILD_FLAGS "-j${JOBS}" "arch=${ACL_ARCH}" "debug=${ACL_DEBUG}" "neon=${ACL_ENABLE_NEON}" "opencl=${ACL_ENABLE_CL}" "embed_kernels=1" "extra_cxx_flags=-fPIC" "benchmark_tests=0" "build=native" "validation_tests=0" "examples=0")
 
-# ACL before 21.05 compiled with only gcc7
-if(NOT (ACL_TAG MATCHES "v22.(02|05|08)"))
+if(NOT (ACL_VERSION VERSION_GREATER "21.05"))
   list(APPEND ACL_BUILD_FLAGS "internal_only=0")
 endif()
 
@@ -154,7 +153,8 @@ if(NOT TFLITE_BAZEL_BUILD)
     PATCH_COMMAND
       git apply ${CMAKE_CURRENT_SOURCE_DIR}/patches/xnnpack_commit.patch ||
       true # patch updates the commit hash for xnnpack for tensorflow v2.4.1
-      SOURCE_SUBDIR tensorflow/lite)
+    SOURCE_SUBDIR tensorflow/lite
+  )
 
   list(APPEND MAKE_AVAILABLE_LIST tensorflow-lite)
 endif()
@@ -234,9 +234,9 @@ if(ARMNN_DELEGATE_ENABLE)
   set(ARMNN_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/external/armnn)
   file(MAKE_DIRECTORY ${ARMNN_LOCATION})
 
-  if(ACL_TAG STREQUAL "v21.08")
+  if(ACL_VERSION VERSION_EQUAL "21.08")
     set(ACL_PATCH_COMMAND "git" "apply" "${CMAKE_CURRENT_SOURCE_DIR}/patches/acl_21_08.patch" "||" "true")
-  elseif(ACL_TAG STREQUAL "v21.11")
+  elseif(ACL_VERSION VERSION_EQUAL "21.11")
     set(ACL_PATCH_COMMAND "git" "apply" "${CMAKE_CURRENT_SOURCE_DIR}/patches/acl_21_11.patch" "||" "true")
   else()
     set(ACL_PATCH_COMMAND "")
@@ -262,7 +262,7 @@ if(ARMNN_DELEGATE_ENABLE)
     acl
     PREFIX ${ACL_LOCATION}
     GIT_REPOSITORY https://review.mlplatform.org/ml/ComputeLibrary
-    GIT_TAG ${ACL_TAG}
+    GIT_TAG v${ACL_VERSION}
     GIT_SHALLOW ON
     PATCH_COMMAND ${ACL_PATCH_COMMAND}
     CONFIGURE_COMMAND ""
@@ -273,25 +273,30 @@ if(ARMNN_DELEGATE_ENABLE)
 
   set(ARMNN_PATCH_COMMAND "")
 
-  if(NOT (ARMNN_TAG MATCHES "v22.(02|05|08)"))
+  if(ARMNN_VERSION VERSION_LESS "22")
     # Never fail as patch is always applied after repo clone
     set(ARMNN_PATCH_COMMAND
         "git" "apply"
         "${CMAKE_CURRENT_SOURCE_DIR}/patches/tflite_armnn_delegate.patch" "||"
         "true")
-  elseif(ARMNN_TAG MATCHES "v22.(02|05|08)")
+  elseif(ARMNN_VERSION VERSION_GREATER_EQUAL "22" AND ARMNN_VERSION VERSION_LESS "23")
     set(ARMNN_PATCH_COMMAND
         "git" "apply"
         "${CMAKE_CURRENT_SOURCE_DIR}/patches/tflite_armnn_delegate22.patch" "||"
         "true")
+  elseif(ARMNN_VERSION VERSION_GREATER_EQUAL "23")
+    set(ARMNN_PATCH_COMMAND
+      "git" "apply"
+      "${CMAKE_CURRENT_SOURCE_DIR}/patches/tflite_armnn_delegate23.patch" "||"
+      "true")
   endif()
 
   # ArmNN and its tflite delegate
   ExternalProject_Add(
     armnn
     PREFIX ${ARMNN_LOCATION}
     GIT_REPOSITORY https://review.mlplatform.org/ml/armnn
-    GIT_TAG ${ARMNN_TAG}
+    GIT_TAG v${ARMNN_VERSION}
     GIT_SHALLOW ON
     PATCH_COMMAND ${ARMNN_PATCH_COMMAND}
     CMAKE_ARGS
@@ -335,7 +340,7 @@ set(BACKEND_INCLUDE_DIRS
 )
 
 # As per https://review.mlplatform.org/c/ml/armnn/+/7327
-if(ARMNN_TAG MATCHES "v22.(05|08)")
+if(ARMNN_VERSION VERSION_GREATER_EQUAL "22.05")
   list(APPEND BACKEND_INCLUDE_DIRS ${ARMNN_LOCATION}/src/armnn/profiling)
 endif()
 
@@ -359,6 +364,7 @@ target_compile_options(
     -Wextra
     -Wno-unused-parameter
     -Wno-type-limits
+    -Wno-comment
     -Werror>)
 
 # ARMNN_DELEGATE_ENABLE exposed in header so set PUBLIC

Dockerfile

@@ -50,10 +50,10 @@ ARG TRITON_ENABLE_MALI_GPU=ON
 ARG TFLITE_ENABLE_RUY=ON
 ARG TFLITE_BAZEL_BUILD=OFF
 ARG TFLITE_ENABLE_FLEX_OPS=OFF
-ARG TFLITE_TAG=v2.4.1
-ARG ARMNN_TAG=v22.08
+ARG TFLITE_TAG=v2.10.0
+ARG ARMNN_VERSION=23.02
 ARG ARMNN_DELEGATE_ENABLE=ON
-ARG ACL_TAG=${ARMNN_TAG}
+ARG ACL_VERSION=${ARMNN_VERSION}
 
 # Install Bazel from source
 RUN if [ "$TFLITE_BAZEL_BUILD" = "ON" ]; then wget -O bazel-3.1.0-dist.zip https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-dist.zip && \
@@ -82,8 +82,8 @@ RUN cmake -S . -B build \
     -DTFLITE_BAZEL_BUILD=${TFLITE_BAZEL_BUILD} \
     -DTFLITE_ENABLE_FLEX_OPS=${TFLITE_ENABLE_FLEX_OPS} \
     -DTFLITE_TAG=${TFLITE_TAG} \
-    -DARMNN_TAG=${ARMNN_TAG} \
+    -DARMNN_VERSION=${ARMNN_VERSION} \
     -DARMNN_DELEGATE_ENABLE=${ARMNN_DELEGATE_ENABLE} \
-    -DACL_TAG=${ACL_TAG} \
+    -DACL_VERSION=${ACL_VERSION} \
     -DJOBS=$(nproc) && \
     cmake --build build -j $(nproc) -t install 

README.md

@@ -49,8 +49,8 @@ but the listed CMake argument can be used to override.
 You can update the version pins for TFLite, [ArmNN](https://github.com/ARM-software/armnn) and [Flatbuffers](https://github.com/google/flatbuffers)) using the following CMake arguments:
 
 * TFLite tag: -DTFLITE_TAG=[tag]
-* ArmNN tag: -DARMNN_TAG=[tag]
-* Flatbuffers tag: -DFLATBUFFERS_VERSION=[tag]
+* ArmNN version: -DARMNN_VERSION=[version]
+* Flatbuffers version: -DFLATBUFFERS_VERSION=[version]
 
 ## Model Repository Structure
 The layout for your model repoitory remains the exact same as for other standard triton backends. Your model name should be set to `model.tflite`. An example model repository layout for ssd_mobilenetv1_coco is shown below:

patches/tflite_armnn_delegate23.patch

@@ -0,0 +1,12 @@
+diff --git a/delegate/cmake/Modules/FindTfLite.cmake b/delegate/cmake/Modules/FindTfLite.cmake
+index 338cde169..69ff03374 100644
+--- a/delegate/cmake/Modules/FindTfLite.cmake
++++ b/delegate/cmake/Modules/FindTfLite.cmake
+@@ -22,6 +22,7 @@ find_library(TfLite_LIB NAMES "libtensorflow_lite_all.so" "libtensorflowlite.so"
+ # If the static library was found, gather all of its dependencies
+ if (TfLite_LIB MATCHES .a$)
+     message("-- Static tensorflow lite library found, using for ArmNN build")
++    set(TFLITE_LIB_ROOT ${TFLITE_LIB_ROOT}/../..) 
+     find_library(TfLite_abseilstrings_LIB "libabsl_strings.a"
+                  PATH ${TFLITE_LIB_ROOT}/_deps/abseil-cpp-build/absl/strings)
+     find_library(TfLite_abseil_synchronization_LIB "libabsl_synchronization.a"

qa/README.md

@@ -16,7 +16,6 @@ apt-get install -y --no-install-recommends \
     libre2-dev \
     libssl-dev \
     libtool \
-    libboost-dev \
     libcurl4-openssl-dev \
     libb64-dev \
     patchelf \
@@ -33,6 +32,13 @@ apt-get install -y --no-install-recommends \
     pkg-config \
     uuid-dev \
     libnuma-dev
+
+# Need boost > 1.78
+wget -O /tmp/boost.tar.gz \
+    https://boostorg.jfrog.io/artifactory/main/release/1.80.0/source/boost_1_80_0.tar.gz && \
+    (cd /tmp && tar xzf boost.tar.gz) && \
+    mv /tmp/boost_1_80_0/boost /usr/include/boost
+    
 ./build.py -v --no-container-build --build-dir=`pwd`/build --backend=identity --endpoint=grpc --endpoint=http --enable-logging --enable-stats --enable-tracing --enable-metrics
 ```
 

qa/tests/thread_count_test.py

@@ -190,7 +190,7 @@ def test_correct_thread_count(
             )
             # Cpu usage should be spread relatively evenly across the threads
             assert relative_std_err < 20.0
-        assert sum(top_percents) > 75.0
+        assert sum(top_percents) > 70.0
 
         # At this point we have verified the number of threads was correct, so we can add
         # the thread ids used for inference directly to the threads to ignore next time

src/tflite.cc

@@ -785,7 +785,7 @@ TRITONSERVER_Error*
 ModelInstanceState::BuildInterpreter()
 {
   // Build the tflite interpreter
-  tflite::ops::builtin::BuiltinOpResolver resolver;
+  tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates resolver;
   tflite::InterpreterBuilder builder(*model_, resolver);
   builder(&interpreter_);
   if (!interpreter_) {

src/tflite_utils.h

@@ -7,6 +7,7 @@
 #include <sstream>
 
 #include "tensorflow/lite/model.h"
+
 #include "triton/core/tritonserver.h"
 
 namespace triton { namespace backend { namespace tensorflowlite {