Merge branch 'feature/triton-qa-model-tests' into 'dev'

Feature/triton qa model tests

See merge request arm-research/smarter/armnn_tflite_backend!27

Author: jishminor

.gitattributes

@@ -0,0 +1 @@
+*.tflite filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -4,4 +4,3 @@
 *.so
 **/__pycache__
 **/*.pyc
-/qa/test_model_repo

Dockerfile

@@ -2,8 +2,7 @@ ARG UBUNTU_VERSION=20.04
 
 FROM ubuntu:${UBUNTU_VERSION} as armnn_tflite_backend
 
-# Triton version pins, assumed same across backend, core, and common
-ARG TRITON_REPO_TAG=main
+ENV DEBIAN_FRONTEND=noninteractive
 
 # Cmake Version options
 ARG CMAKE_VERSION=3.21.1
@@ -43,6 +42,21 @@ RUN apt-get update && \
     pip3 install -U pip wheel && \
     rm -rf /var/lib/apt/lists/*
 
+# Triton version pins, assumed same across backend, core, and common
+# Note that this is set to the rX.XX branches, not the vX.X.X tags
+ARG TRITON_REPO_TAG=main
+
+# CMake build arguments defaults
+ARG CMAKE_BUILD_TYPE=RELEASE
+ARG TRITON_ENABLE_MALI_GPU=ON
+ARG TFLITE_ENABLE_RUY=ON
+ARG TFLITE_BAZEL_BUILD=OFF
+ARG TFLITE_ENABLE_FLEX_OPS=OFF
+ARG TFLITE_TAG=v2.4.1
+ARG ARMNN_TAG=v21.08
+ARG ARMNN_DELEGATE_ENABLE=ON
+ARG ACL_TAG=${ARMNN_TAG}
+
 # Install Bazel from source
 RUN wget -O bazel-3.1.0-dist.zip https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-dist.zip && \
     unzip -d bazel bazel-3.1.0-dist.zip && \
@@ -58,12 +72,20 @@ COPY . .
 RUN mkdir build && \
     cd build && \
     cmake .. \
+    -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
     -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install \
     -DTRITON_BACKEND_REPO_TAG=${TRITON_REPO_TAG} \
     -DTRITON_CORE_REPO_TAG=${TRITON_REPO_TAG} \
     -DTRITON_COMMON_REPO_TAG=${TRITON_REPO_TAG} \
     -DTRITON_ENABLE_GPU=OFF \
-    -DTRITON_ENABLE_MALI_GPU=ON \
-    -DTFLITE_ENABLE_RUY=ON \
-    -DJOBS=$(nproc) && \
+    -DTRITON_ENABLE_MALI_GPU=${TRITON_ENABLE_MALI_GPU}} \
+    -DTFLITE_ENABLE_RUY=${TFLITE_ENABLE_RUY} \
+    -DTFLITE_BAZEL_BUILD=${TFLITE_BAZEL_BUILD} \
+    -DTFLITE_ENABLE_FLEX_OPS=${TFLITE_ENABLE_FLEX_OPS} \
+    -DTFLITE_TAG=${TFLITE_TAG} \
+    -DARMNN_TAG=${ARMNN_TAG} \
+    -DARMNN_DELEGATE_ENABLE=${ARMNN_DELEGATE_ENABLE} \
+    -DACL_TAG=${ACL_TAG} \
+    -DJOBS=$(nproc) \
+    && \
     make -j$(nproc) install

Dockerfile.QA

@@ -0,0 +1,179 @@
+#
+# Copyright © 2021 Arm Ltd. All rights reserved.
+# SPDX-License-Identifier: MIT
+#
+
+#
+# Multistage build.
+#
+
+ARG BASE_IMAGE=tritonserver
+ARG ARMNN_TFLITE_BACKEND_IMAGE=armnn_tflite_backend
+ARG BUILD_IMAGE=tritonserver_build
+ARG TRITON_REPO_TAG=main
+ARG TRITON_COMMON_REPO_TAG=${TRITON_REPO_TAG}
+ARG TRITON_CORE_REPO_TAG=${TRITON_REPO_TAG}
+ARG TRITON_THIRD_PARTY_REPO_TAG=${TRITON_REPO_TAG}
+ARG TRITON_BACKEND_REPO_TAG=${TRITON_REPO_TAG}
+ARG TRITON_ENABLE_MALI_GPU=OFF
+
+
+############################################################################
+## Build tests in the BUILD_IMAGE since it has already been configured
+## correctly and has some existing build artifacts. Copy artifacts
+## into QA area.
+############################################################################
+FROM ${BUILD_IMAGE} AS build
+
+# Ensure apt-get won't prompt for selecting options
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Build the client library and examples
+ARG TRITON_REPO_TAG
+ARG TRITON_COMMON_REPO_TAG
+ARG TRITON_CORE_REPO_TAG
+ARG TRITON_BACKEND_REPO_TAG
+ARG TRITON_THIRD_PARTY_REPO_TAG
+ARG TRITON_ENABLE_MALI_GPU
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    software-properties-common \
+    autoconf \
+    automake \
+    build-essential \
+    curl \
+    git \
+    libb64-dev \
+    libopencv-dev \
+    libopencv-core-dev \
+    libssl-dev \
+    libtool \
+    pkg-config \
+    python3 \
+    python3-pip \
+    python3-dev \
+    rapidjson-dev \
+    vim \
+    wget \
+    python3-pdfkit \
+    maven \
+    default-jdk && \
+    pip3 install --upgrade wheel setuptools && \
+    pip3 install --upgrade grpcio-tools && \
+    pip3 install --upgrade pip
+
+# Build the client repo
+WORKDIR /workspace
+RUN git clone --single-branch --depth=1 -b ${TRITON_REPO_TAG} https://github.com/triton-inference-server/client.git client
+WORKDIR /workspace/build
+RUN cmake -DCMAKE_INSTALL_PREFIX=/workspace/install \
+    -DTRITON_VERSION=`cat /workspace/TRITON_VERSION` \
+    -DTRITON_COMMON_REPO_TAG=${TRITON_COMMON_REPO_TAG} \
+    -DTRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG} \
+    -DTRITON_BACKEND_REPO_TAG=${TRITON_BACKEND_REPO_TAG} \
+    -DTRITON_THIRD_PARTY_REPO_TAG=${TRITON_THIRD_PARTY_REPO_TAG} \
+    -DTRITON_ENABLE_CC_HTTP=ON -DTRITON_ENABLE_CC_GRPC=ON \
+    -DTRITON_ENABLE_PYTHON_HTTP=ON -DTRITON_ENABLE_PYTHON_GRPC=ON \
+    -DTRITON_ENABLE_JAVA_HTTP=ON \
+    -DTRITON_ENABLE_PERF_ANALYZER=ON \
+    -DTRITON_ENABLE_EXAMPLES=ON -DTRITON_ENABLE_TESTS=ON \
+    -DTRITON_ENABLE_GPU=OFF /workspace/client
+RUN make -j$(nproc) cc-clients python-clients java-clients
+
+WORKDIR /workspace
+RUN cd install && \
+    export VERSION=`cat /workspace/TRITON_VERSION` && \
+    tar zcf /workspace/v$VERSION.clients.tar.gz *
+
+# Install the dependencies needed to run the client examples. These
+# are not needed for building but including them allows this image to
+# be used to run the client examples.
+RUN pip3 install --upgrade numpy pillow attrdict && \
+    find install/python/ -maxdepth 1 -type f -name \
+    "tritonclient-*linux*.whl" | xargs printf -- '%s[all]' | \
+    xargs pip3 install --upgrade
+
+# Populate the /workspace/qa directory
+WORKDIR /workspace
+RUN mkdir -p qa/clients && mkdir -p qa/pkgs && \
+    cp -a install/bin/* qa/clients/. && \
+    cp install/lib/libgrpcclient.so qa/clients/. && \
+    cp install/lib/libhttpclient.so qa/clients/. && \
+    cp install/python/*.py qa/clients/. && \
+    cp install/python/triton*.whl qa/pkgs/. && \
+    cp install/java/examples/*.jar qa/clients/.
+RUN cp client/src/grpc_generated/go/*.go qa/L0_simple_go_client/. && \
+    cp -r client/src/grpc_generated/java qa/L0_client_java/.
+
+
+############################################################################
+## Reference ArmNN TFLite image
+############################################################################
+FROM $ARMNN_TFLITE_BACKEND_IMAGE as armnn_tflite_backend
+
+############################################################################
+## Create CI enabled image
+############################################################################
+FROM $BASE_IMAGE
+
+ARG TARGETPLATFORM
+
+# Ensure apt-get won't prompt for selecting options
+ENV DEBIAN_FRONTEND=noninteractive
+
+# CI/QA for memcheck requires valgrind
+# libarchive-dev is required by Python backend
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    curl \
+    libopencv-dev \
+    libarchive-dev \
+    libopencv-core-dev \
+    libzmq3-dev \
+    python3-dev \
+    python3-pip \
+    python3-protobuf \
+    python3-setuptools \
+    swig \
+    nginx \
+    libpng-dev \
+    protobuf-compiler \
+    g++ \
+    maven \
+    jq \
+    valgrind && \
+    rm -rf /var/lib/apt/lists/*
+
+# CI/QA expects "python" executable (not python3).
+RUN rm -f /usr/bin/python && \
+    ln -s /usr/bin/python3 /usr/bin/python
+
+RUN pip3 install --upgrade wheel setuptools && \
+    pip3 install --upgrade numpy pillow attrdict future grpcio requests gsutil awscli six grpcio-channelz && \
+    pip3 install --upgrade pytest pytest-xdist pytest-xprocess jinja2
+
+# CI expects tests in /opt/tritonserver/qa. The triton-server (1000)
+# user should own all artifacts in case CI is run using triton-server
+# user.
+WORKDIR /opt/tritonserver
+COPY --chown=1000:1000 --from=build /workspace/qa/ qa/
+
+# Copy in identity backend used in some qa tests
+COPY --from=build /tmp/tritonbuild/identity/install/backends/identity /opt/tritonserver/backends/identity
+
+# Copy in pytest files from the current repo
+# These are the tests we use to validate the backend functionality
+COPY qa/ qa/armnn_tflite_qa
+
+# Remove CI tests that are meant to run only on build image and
+# install the tritonserver/triton python client APIs.
+RUN rm -fr qa/L0_copyrights qa/L0_build_variants && \
+    find qa/pkgs/ -maxdepth 1 -type f -name \
+    "tritonclient-*linux*.whl" | xargs printf -- '%s[all]' | \
+    xargs pip3 install --upgrade
+
+ENV LD_LIBRARY_PATH /opt/tritonserver/qa/clients:${LD_LIBRARY_PATH}
+
+# Copy in the built armnn_tflite_backend
+COPY --from=armnn_tflite_backend /opt/armnn_tflite_backend/build/install/backends/armnn_tflite /opt/tritonserver/backends/armnn_tflite
+

qa/README.md

@@ -0,0 +1,46 @@
+# Testing
+In order to run the test suite for this backend perform the following steps on an Arm64 machine:
+
+Ensure you have git lfs installed and initialized on your machine, as the tflite models used for testing are tracked using lfs:
+```
+sudo apt install git-lfs
+git lfs install
+```
+
+First clone the server repository from the fork and branch specified in the following snippet and move to the repo directory:
+```
+git clone -b feature/armnn_tflite_test_support https://github.com/jishminor/server
+cd server
+```
+
+Next run the `build.py` script to just build the triton server with the identity and ensemble backend but without including this backend:
+```bash
+./build.py --cmake-dir=/workspace/build --build-dir=/tmp/citritonbuild --image=base,arm64v8/ubuntu:20.04 --enable-logging --enable-stats --enable-tracing --enable-metrics --endpoint=http --endpoint=grpc --backend=identity --backend=ensemble
+```
+
+Build the ArmNN TFLite Backend using the Dockerfile in this repo:
+```bash
+cd <backend_directory>
+docker build --build-arg TFLITE_BAZEL_BUILD=ON --build-arg TFLITE_ENABLE_FLEX_OPS=ON -t armnn_tflite_backend .
+```
+
+This will produce the `tritonserver_build` and `tritonserver` images on your machine.
+
+Build the QA image by running the following:
+```bash
+docker build -t tritonserver_qa -f Dockerfile.QA .
+```
+
+Now run the QA container and mount the QA model repositories into the container so the tests will be able to access them.
+
+```bash
+docker run -it --rm -v <backend_directory>/qa/triton_qa_models:/data/inferenceserver tritonserver_qa
+```
+
+Within the container the QA tests are in /opt/tritonserver/qa. To run a test simply change directory to the test and run the test.sh script.
+
+To run the the full test suite run:
+```bash
+cd qa/armnn_tflite_qa/
+python3 -m pytest tests/ --model-repo-path /data/inferenceserver/accuracy_test_model_repo
+```

qa/conftest.py

@@ -2,13 +2,15 @@
 # SPDX-License-Identifier: MIT
 
 import pytest
+from xprocess import ProcessStarter
 
 from jinja2 import Environment, Template
 
 import tritonclient.http as httpclient
 import tritonclient.grpc as grpcclient
 
 from time import sleep
+import requests
 
 
 @pytest.fixture
@@ -73,19 +75,78 @@ def pytest_addoption(parser):
         required=False,
         help="Path to triton backends",
     )
+    parser.addoption(
+        "--triton-qa-model-repo-ver",
+        action="store",
+        default="21.09",
+        required=False,
+        help="Version of the generated qa-model-repo for triton qa tests",
+    )
+
+
+@pytest.fixture(scope="module")
+def tritonserver(xprocess, host, model_repo_path):
+    """
+    Starts an instance of the triton server
+    """
+
+    class Starter(ProcessStarter):
+        pattern = "Started \w* at \d.\d.\d.\d:\d*"
+
+        # checks if triton is ready with request to health endpoint
+        def startup_check(self):
+            try:
+                response = requests.get(f"http://{host}:8000/v2/health/ready")
+                if response.status_code == 200:
+                    return True
+                else:
+                    return False
+            except requests.exceptions.RequestException:
+                return False
+
+        # command to start process
+        args = [
+            "tritonserver",
+            "--model-repository",
+            model_repo_path,
+            "--model-control-mode",
+            "explicit",
+        ]
+
+        terminate_on_interrupt = True
+
+    # ensure process is running and return its logfile
+    logfile = xprocess.ensure("tritonserver", Starter)
+
+    yield
+
+    # clean up whole process tree afterwards
+    xprocess.getinfo("tritonserver").terminate()
 
 
 def pytest_generate_tests(metafunc):
     """
     Makes the program option 'host' available to all tests as a function fixture
     """
     if "host" in metafunc.fixturenames:
-        metafunc.parametrize("host", [metafunc.config.getoption("host")])
+        metafunc.parametrize(
+            "host", [metafunc.config.getoption("host")], scope="module"
+        )
     if "model_repo_path" in metafunc.fixturenames:
         metafunc.parametrize(
-            "model_repo_path", [metafunc.config.getoption("model_repo_path")]
+            "model_repo_path",
+            [metafunc.config.getoption("model_repo_path")],
+            scope="module",
         )
     if "backend_directory" in metafunc.fixturenames:
         metafunc.parametrize(
-            "backend_directory", [metafunc.config.getoption("backend_directory")]
+            "backend_directory",
+            [metafunc.config.getoption("backend_directory")],
+            scope="module",
+        )
+    if "triton_qa_model_repo_ver" in metafunc.fixturenames:
+        metafunc.parametrize(
+            "triton_qa_model_repo_ver",
+            [metafunc.config.getoption("triton_qa_model_repo_ver")],
+            scope="module",
         )