fix test and add more tests

Author: cccclai

.ci/scripts/test_qnn_static_llama_eval.sh

@@ -7,22 +7,45 @@
 
 set -euo pipefail
 
+source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
+
+# Download QNN_SDK. If already downloaded, export environment path
+source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh"
+install_qnn
+
+export EXECUTORCH_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
+export LD_LIBRARY_PATH="${QNN_SDK_ROOT}/lib/x86_64-linux-clang"
+export PYTHONPATH=".."
+cp schema/program.fbs exir/_serialize/program.fbs
+cp schema/scalar_type.fbs exir/_serialize/scalar_type.fbs
+cp -f build-x86/backends/qualcomm/PyQnnManagerAdaptor.cpython-310-x86_64-linux-gnu.so backends/qualcomm/python
+cp -f build-x86/backends/qualcomm/PyQnnWrapperAdaptor.cpython-310-x86_64-linux-gnu.so backends/qualcomm/python
+
+if [[ -z "${PYTHON_EXECUTABLE:-}" ]]; then
+  PYTHON_EXECUTABLE=python3
+fi
+
+which "${PYTHON_EXECUTABLE}"
+
 # Config
 PYTHON_EXECUTABLE="${PYTHON_EXECUTABLE:-python3}"
 MODEL="qwen2_5-0_5b"
 MAX_SEQ=1024
 PTQ="16a4w"
 THRESHOLD=62.0   # regression guardrail
 
-# Run command and capture log
+EXTRA_FLAGS="$@"
+
+# Run command and capture *both stdout and stderr*
 LOG_FILE="eval_${MODEL}_$(date +%Y%m%d_%H%M%S).log"
 
 echo ">>> Running evaluation..."
 $PYTHON_EXECUTABLE -m executorch.examples.qualcomm.oss_scripts.llama.eval_llama_qnn \
   --decoder_model "$MODEL" \
   --quant_linear_only \
   --max_seq_length "$MAX_SEQ" \
-  --ptq "$PTQ" | tee "$LOG_FILE"
+  --ptq "$PTQ" \
+  $EXTRA_FLAGS 2>&1 | tee "$LOG_FILE"
 
 # Extract last word_perplexity
 LAST_PERP=$(grep "INFO:root:wikitext:" "$LOG_FILE" | tail -n 1 | sed -E "s/.*'word_perplexity,none': ([0-9.]+).*/\1/")

.github/workflows/pull.yml

@@ -601,6 +601,12 @@ jobs:
       contents: read
     strategy:
       fail-fast: false
+    matrix:
+      config:
+        - { name: "baseline", flags: "" }
+        - { name: "spinquant", flags: "--spinquant" }
+        - { name: "range-setting", flags: "--range_setting mse_with_act_loss" }
+        - { name: "spinquant+range-setting", flags: "--spinquant --range_setting mse_with_act_loss" }
     with:
       runner: linux.2xlarge
       docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk
@@ -624,7 +630,8 @@ jobs:
         PYTHON_EXECUTABLE=python bash examples/models/llama/install_requirements.sh
 
         # Test static llama weight sharing and accuracy
-        PYTHON_EXECUTABLE=python bash .ci/scripts/test_qnn_static_llama_eval.sh
+        echo ">>> Running config: ${{ matrix.config.name }}"
+        PYTHON_EXECUTABLE=python bash .ci/scripts/test_qnn_static_llama_eval.sh ${{ matrix.config.flags }}
 
 
   # test-qnn-models-linux: