v23.01 (#13)

Signed-off-by: Josh Minor <[email protected]>

Author: jishminor

.github/workflows/build.yml

@@ -242,13 +242,22 @@ jobs:
           apt install -yqq --no-install-recommends libssl-dev zlib1g-dev rapidjson-dev python3-dev python3-pip wget build-essential git
           python3 -m pip install --upgrade pip setuptools wheel 
           python3 -m pip install --upgrade grpcio-tools
-          python3 -m pip install --upgrade numpy pillow attrdict future grpcio requests gsutil awscli six grpcio-channelz
+          python3 -m pip install --upgrade numpy pillow attrdict future grpcio requests gsutil awscli six grpcio-channelz filelock jinja2 psutil
           python3 -m pip install --upgrade tritonclient[all]
-          python3 -m pip install --upgrade pytest pytest-xdist pytest-xprocess jinja2
+          python3 -m pip install --upgrade pytest pytest-xdist
 
-      - name: Run Tests
+      - name: Fetch test models
+        working-directory: ./qa
         run: |
-          cd qa
           wget -q https://images.getsmarter.io/ml-models/armnn_tflite_backend_triton_model_repo.tar.gz
           tar -xzf armnn_tflite_backend_triton_model_repo.tar.gz
-          python3 -m pytest -v tests/ --model-repo-path $(pwd)/triton_qa_models/accuracy_test_model_repo --triton-path /opt/tritonserver/bin/tritonserver --backend-directory /opt/tritonserver/backends
+
+      - name: Run HTTP Tests
+        working-directory: ./qa
+        run: |
+          python3 -m pytest -x -n $(( $(nproc) / 2 )) -v tests/ --model-repo-path $(pwd)/triton_qa_models/accuracy_test_model_repo --triton-path /opt/tritonserver/bin/tritonserver --backend-directory /opt/tritonserver/backends
+
+      - name: Run gRPC Tests
+        working-directory: ./qa
+        run: |
+          python3 -m pytest -x -n $(( $(nproc) / 2 )) -v tests/ --client-type grpc --model-repo-path $(pwd)/triton_qa_models/accuracy_test_model_repo --triton-path /opt/tritonserver/bin/tritonserver --backend-directory /opt/tritonserver/backends

.github/workflows/triton_build.yml

@@ -23,7 +23,7 @@ jobs:
         run: |
           git clone https://github.com/triton-inference-server/server 
           cd server
-          readarray -d '' -t branches < <( git branch -r --list | grep -E 'r2[2-9]+\.[0-9]+$' | tr '\n' ' ')
+          readarray -d '' -t branches < <( git branch -r --list | grep -E 'origin/r2[2-9]+\.[0-9]+$' | tr '\n' ' ')
           branches+=("origin/main")
           echo "All branches: ${branches[@]}"
           declare -a branches_to_build
@@ -53,6 +53,7 @@ jobs:
   build:
     needs: get_triton_branches
     strategy:
+      fail-fast: false
       matrix:
         branch: ${{ fromJson(needs.get_triton_branches.outputs.triton_branches) }}
         build_arch:

.gitignore

@@ -3,6 +3,7 @@
 /.vscode
 *.so
 **/__pycache__
+**/.pytest_cache
 **/*.pyc
 /deps
 /.devcontainer

qa/config-template.pbtxt

@@ -24,6 +24,16 @@ output [
     {% endfor %}
 
 ]
+
+{% if model.tflite_num_threads %}
+parameters: {
+key: "tflite_num_threads"
+value: {
+string_value:"{{ model.tflite_num_threads }}"
+}
+}
+{% endif %}
+
 instance_group [
     {% if model.gpu > 0 %}
     {
@@ -46,15 +56,15 @@ optimization { execution_accelerators {
     {% if model.armnn_cpu %}
     cpu_execution_accelerator : [ {
     name : "armnn"
-    {% for k,v in model.armnn_cpu_parameters %}
+    {% for k,v in model.armnn_cpu_parameters.items() %}
     parameters { key: "{{k}}" value: "{{v}}" }
     {% endfor %}
     }]
     {% endif %}
     {% if model.armnn_gpu %}
     gpu_execution_accelerator : [ {
     name : "armnn"
-    {% for k,v in model.armnn_gpu_parameters %}
+    {% for k,v in model.armnn_gpu_parameters.items() %}
     parameters { key: "{{k}}" value: "{{v}}" }
     {% endfor %}
     }]
@@ -65,9 +75,30 @@ optimization { execution_accelerators {
 optimization { execution_accelerators {
     cpu_execution_accelerator : [ {
     name : "xnnpack"
-    {% for k,v in model.xnnpack_parameters %}
+    {% for k,v in model.xnnpack_parameters.items() %}
     parameters { key: "{{k}}" value: "{{v}}" }
     {% endfor %}
     }]
 }}
 {% endif %}
+
+{% if model.warm_up %}
+model_warmup [
+{
+    name: "regular sample"
+    count: 5
+    batch_size: {{ 1 if model.max_batch_size == 0 else model.max_batch_size }}
+    {% for input in model.inputs %}
+    inputs {
+        key: "{{ input.name }}"
+        value: {
+            data_type: {{ input.datatype }}
+            dims: [ {{ input.dims|join(', ') }} ]
+            random_data: true
+        }
+    }
+    {% endfor %}
+}
+]
+{% endif %}
+