Test 2

Author: kirklandsign

.ci/docker/requirements-ci.txt

@@ -9,6 +9,7 @@ torchsr==1.0.4
 transformers==4.47.1
 zstd==1.5.5.1
 pandas>=2.2.2; python_version >= '3.10'
+prettytable>=3.14.0
 pytest==7.2.0
 pytest-cov==4.1.0
 expecttest==0.1.6

.github/workflows/android-perf.yml

@@ -120,6 +120,10 @@ jobs:
           # so let's just sed it
           sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' android-llm-device-farm-test-spec.yml.j2
 
+
+          BINARY_PATH="https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/xnn_executor_runner
+          sed -i -e 's,{{ binary_path }},'"${BINARY_PATH}"',g' android-llm-device-farm-test-spec.yml.j2
+
           BENCHMARK_CONFIG_ID=$(echo "${{ matrix.model }}_${{ matrix.config }}" | sed -e 's/[^A-Za-z0-9._-]/_/g')
           # The config for this benchmark runs, we save it in the test spec so that it can be fetched
           # later by the upload script
@@ -364,6 +368,7 @@ jobs:
 
         export ANDROID_ABIS="arm64-v8a"
         PYTHON_EXECUTABLE=python EXECUTORCH_BUILD_QNN=ON QNN_SDK_ROOT=/tmp/qnn/2.28.0.241029 bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME}
+        cp cmake-out-android-arm64-v8a/backends/xnnpack/xnn_executor_runner  ${ARTIFACTS_DIR_NAME}
 
   # Let's see how expensive this job is, we might want to tone it down by running it periodically
   benchmark-on-device:

extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml.j2

@@ -13,6 +13,9 @@ phases:
 
       # Download the model from S3
       - curl -s --fail '{{ model_path }}' -o model.zip
+      - curl -s --fail '{{ binary_path }}' -o xnn_executor_runner
+
+
       - unzip model.zip && ls -la
 
       # Copy the model to sdcard. This prints too much progress info when the files
@@ -21,9 +24,12 @@ phases:
       - adb -s $DEVICEFARM_DEVICE_UDID push *.model /sdcard > /dev/null && echo OK
       - adb -s $DEVICEFARM_DEVICE_UDID push *.pte /sdcard > /dev/null && echo OK
 
+      - adb -s $DEVICEFARM_DEVICE_UDID push xnn_executor_runner /sdcard > /dev/null && echo OK
+
       # Prepare the model and the tokenizer
       - adb -s $DEVICEFARM_DEVICE_UDID shell "ls -la /sdcard/"
       - adb -s $DEVICEFARM_DEVICE_UDID shell "mkdir -p /data/local/tmp/minibench/"
+      - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/xnn_executor_runner /data/local/tmp/minibench/"
       - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/*.bin /data/local/tmp/minibench/"
       - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/*.model /data/local/tmp/minibench/"
       - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/*.pte /data/local/tmp/minibench/"
@@ -33,6 +39,8 @@ phases:
       - adb -s $DEVICEFARM_DEVICE_UDID shell "ls -la /data/local/tmp/minibench/"
       - adb -s $DEVICEFARM_DEVICE_UDID shell "run-as org.pytorch.minibench rm -rf files"
 
+      - adb -s $DEVICEFARM_DEVICE_UDID shell chmod +x /data/local/tmp/minibench/xnn_executor_runner
+
   test:
     commands:
       # By default, the following ADB command is used by Device Farm to run your Instrumentation test.
@@ -117,6 +125,9 @@ phases:
           adb -s $DEVICEFARM_DEVICE_UDID shell am start -W -n org.pytorch.minibench/.BenchmarkActivity \
             --es "model_dir" "/data/local/tmp/minibench"
         fi
+        adb -s $DEVICEFARM_DEVICE_UDID shell /data/local/tmp/minibench/xnn_executor_runner --model_path /data/local/tmp/minibench/*.pte --prof_result_path /data/local/tmp/prof_result.bin
+        adb -s $DEVICEFARM_DEVICE_UDID pull /data/local/tmp/prof_result.bin
+        python -m profiler.profiler_results_cli --prof_results_bin prof_result.bin
 
 
   post_test:

profiler/profiler_results_cli.py

@@ -7,7 +7,7 @@
 import argparse
 import sys
 
-from executorch.profiler.parse_profiler_results import (  # type: ignore[import-not-found]
+from .parse_profiler_results import (  # type: ignore[import-not-found]
     deserialize_profile_results,
     mem_profile_table,
     profile_aggregate_framework_tax,