Fix reporting backends and dtyep to benchmark results (#6073)

Fix reporting backends and dtyep to benchmark results (#6023)

Summary:
Couple minor fixes for reporting the benchmarking results:
 - qnn models are not reporting "backend" and "dtype" info in the benchmark_results.json (Android)
 - tinyllama mdoel is not reporting "backend" and "dtype" info in the benchmark_results.json (Android)
 - include compute precision to the exported coreml model name
 - rename "llama2" to "tinyllama" to eliminate confusion (many people thought it was llama2-7b)

Pull Request resolved: #6023

Reviewed By: huydhn

Differential Revision: D64074262

Pulled By: guangy10

fbshipit-source-id: c6c53d004c4fb3ad410a792639af2c22a6978b67
(cherry picked from commit 012cba9)

Co-authored-by: Guang Yang <[email protected]>

Author: pytorchbot

.ci/scripts/test_llama.sh

@@ -171,7 +171,7 @@ else
 fi
 
 # Check dtype.
-EXPORTED_MODEL_NAME="llama2"
+EXPORTED_MODEL_NAME="tinyllama_${MODE}_${DTYPE}"
 if [[ "${DTYPE}" == "fp16" ]]; then
   EXPORTED_MODEL_NAME="${EXPORTED_MODEL_NAME}_h"
 elif [[ "${DTYPE}" == "fp32" ]]; then

.ci/scripts/test_model.sh

@@ -155,30 +155,24 @@ test_model_with_qnn() {
 
   if [[ "${MODEL_NAME}" == "dl3" ]]; then
     EXPORT_SCRIPT=deeplab_v3
-    EXPORTED_MODEL_NAME=dlv3_qnn.pte
   elif [[ "${MODEL_NAME}" == "mv3" ]]; then
     EXPORT_SCRIPT=mobilenet_v3
-    EXPORTED_MODEL_NAME=mv3_qnn.pte
   elif [[ "${MODEL_NAME}" == "mv2" ]]; then
     EXPORT_SCRIPT=mobilenet_v2
-    EXPORTED_MODEL_NAME=mv2_qnn.pte
   elif [[ "${MODEL_NAME}" == "ic4" ]]; then
     EXPORT_SCRIPT=inception_v4
-    EXPORTED_MODEL_NAME=ic4_qnn.pte
   elif [[ "${MODEL_NAME}" == "ic3" ]]; then
     EXPORT_SCRIPT=inception_v3
-    EXPORTED_MODEL_NAME=ic3_qnn.pte
   elif [[ "${MODEL_NAME}" == "vit" ]]; then
     EXPORT_SCRIPT=torchvision_vit
-    EXPORTED_MODEL_NAME=vit_qnn.pte
   fi
 
   # Use SM8450 for S22, SM8550 for S23, and SM8560 for S24
   # TODO(guangyang): Make QNN chipset matches the target device
   QNN_CHIPSET=SM8450
 
   "${PYTHON_EXECUTABLE}" -m examples.qualcomm.scripts.${EXPORT_SCRIPT} -b ${CMAKE_OUTPUT_DIR} -m ${QNN_CHIPSET} --compile_only
-  EXPORTED_MODEL=./${EXPORT_SCRIPT}/${EXPORTED_MODEL_NAME}
+  EXPORTED_MODEL=$(find "./${EXPORT_SCRIPT}" -type f -name "${MODEL_NAME}*.pte" -print -quit)
 }
 
 test_model_with_coreml() {
@@ -187,8 +181,20 @@ test_model_with_coreml() {
     exit 1
   fi
 
-  "${PYTHON_EXECUTABLE}" -m examples.apple.coreml.scripts.export --model_name="${MODEL_NAME}"
+  DTYPE=float16
+
+  "${PYTHON_EXECUTABLE}" -m examples.apple.coreml.scripts.export --model_name="${MODEL_NAME}" --compute_precision "${DTYPE}"
   EXPORTED_MODEL=$(find "." -type f -name "${MODEL_NAME}*.pte" -print -quit)
+  # TODO:
+  if [ -n "$EXPORTED_MODEL" ]; then
+    EXPORTED_MODEL_WITH_DTYPE="${EXPORTED_MODEL%.pte}_${DTYPE}.pte"
+    mv "$EXPORTED_MODEL" "$EXPORTED_MODEL_WITH_DTYPE"
+    EXPORTED_MODEL="$EXPORTED_MODEL_WITH_DTYPE"
+    echo "Renamed file path: $EXPORTED_MODEL"
+  else
+    echo "No .pte file found"
+    exit 1
+  fi
 }
 
 if [[ "${BACKEND}" == "portable" ]]; then

examples/qualcomm/scripts/deeplab_v3.py

@@ -82,7 +82,7 @@ def main(args):
             data_size=data_num, dataset_dir=args.artifact, download=args.download
         )
 
-    pte_filename = "dlv3_qnn"
+    pte_filename = "dl3_qnn_q8"
     instance = DeepLabV3ResNet101Model()
 
     build_executorch_binary(

examples/qualcomm/scripts/edsr.py

@@ -108,7 +108,7 @@ def main(args):
     )
 
     inputs, targets, input_list = dataset.lr, dataset.hr, dataset.get_input_list()
-    pte_filename = "edsr_qnn"
+    pte_filename = "edsr_qnn_q8"
     instance = EdsrModel()
 
     build_executorch_binary(

examples/qualcomm/scripts/inception_v3.py

@@ -78,7 +78,7 @@ def main(args):
             dataset_path=f"{args.dataset}",
             data_size=data_num,
         )
-    pte_filename = "ic3_qnn"
+    pte_filename = "ic3_qnn_q8"
     instance = InceptionV3Model()
     build_executorch_binary(
         instance.get_eager_model().eval(),

examples/qualcomm/scripts/inception_v4.py

@@ -77,7 +77,7 @@ def main(args):
             dataset_path=f"{args.dataset}",
             data_size=data_num,
         )
-    pte_filename = "ic4_qnn"
+    pte_filename = "ic4_qnn_q8"
     instance = InceptionV4Model()
     build_executorch_binary(
         instance.get_eager_model().eval(),

examples/qualcomm/scripts/mobilenet_v2.py

@@ -78,7 +78,7 @@ def main(args):
             dataset_path=f"{args.dataset}",
             data_size=data_num,
         )
-    pte_filename = "mv2_qnn"
+    pte_filename = "mv2_qnn_q8"
     instance = MV2Model()
     build_executorch_binary(
         instance.get_eager_model().eval(),

examples/qualcomm/scripts/mobilenet_v3.py

@@ -77,7 +77,7 @@ def main(args):
             dataset_path=f"{args.dataset}",
             data_size=data_num,
         )
-    pte_filename = "mv3_qnn"
+    pte_filename = "mv3_qnn_float16"
     instance = MV3Model()
     build_executorch_binary(
         instance.get_eager_model().eval(),

examples/qualcomm/scripts/torchvision_vit.py

@@ -70,7 +70,7 @@ def main(args):
             data_size=data_num,
         )
 
-    pte_filename = "vit_qnn"
+    pte_filename = "vit_qnn_q8"
     instance = TorchVisionViTModel()
     build_executorch_binary(
         instance.get_eager_model().eval(),