Fix reporting backends and dtyep to benchmark results

Author: Guang Yang

.ci/scripts/test_llama.sh

@@ -171,7 +171,7 @@ else
 fi
 
 # Check dtype.
-EXPORTED_MODEL_NAME="llama2"
+EXPORTED_MODEL_NAME="tinyllama_${MODE}_${DTYPE}"
 if [[ "${DTYPE}" == "fp16" ]]; then
   EXPORTED_MODEL_NAME="${EXPORTED_MODEL_NAME}_h"
 elif [[ "${DTYPE}" == "bf16" ]]; then

.ci/scripts/test_model.sh

@@ -152,32 +152,34 @@ test_model_with_qnn() {
 
   export LD_LIBRARY_PATH=$QNN_SDK_ROOT/lib/x86_64-linux-clang/
   export PYTHONPATH=$EXECUTORCH_ROOT/..
+  # QNN support float16 only
+  DTYPE=float16
 
   if [[ "${MODEL_NAME}" == "dl3" ]]; then
     EXPORT_SCRIPT=deeplab_v3
-    EXPORTED_MODEL_NAME=dlv3_qnn.pte
+    EXPORTED_MODEL_NAME=dl3_qnn_${DTYPE}.pte
   elif [[ "${MODEL_NAME}" == "mv3" ]]; then
     EXPORT_SCRIPT=mobilenet_v3
-    EXPORTED_MODEL_NAME=mv3_qnn.pte
+    EXPORTED_MODEL_NAME=mv3_qnn_${DTYPE}.pte
   elif [[ "${MODEL_NAME}" == "mv2" ]]; then
     EXPORT_SCRIPT=mobilenet_v2
-    EXPORTED_MODEL_NAME=mv2_qnn.pte
+    EXPORTED_MODEL_NAME=mv2_qnn_${DTYPE}.pte
   elif [[ "${MODEL_NAME}" == "ic4" ]]; then
     EXPORT_SCRIPT=inception_v4
-    EXPORTED_MODEL_NAME=ic4_qnn.pte
+    EXPORTED_MODEL_NAME=ic4_qnn_${DTYPE}.pte
   elif [[ "${MODEL_NAME}" == "ic3" ]]; then
     EXPORT_SCRIPT=inception_v3
-    EXPORTED_MODEL_NAME=ic3_qnn.pte
+    EXPORTED_MODEL_NAME=ic3_qnn_${DTYPE}.pte
   elif [[ "${MODEL_NAME}" == "vit" ]]; then
     EXPORT_SCRIPT=torchvision_vit
-    EXPORTED_MODEL_NAME=vit_qnn.pte
+    EXPORTED_MODEL_NAME=vit_qnn_${DTYPE}.pte
   fi
 
   # Use SM8450 for S22, SM8550 for S23, and SM8560 for S24
   # TODO(guangyang): Make QNN chipset matches the target device
   QNN_CHIPSET=SM8450
 
-  "${PYTHON_EXECUTABLE}" -m examples.qualcomm.scripts.${EXPORT_SCRIPT} -b ${CMAKE_OUTPUT_DIR} -m ${QNN_CHIPSET} --compile_only
+  "${PYTHON_EXECUTABLE}" -m examples.qualcomm.scripts.${EXPORT_SCRIPT} -b ${CMAKE_OUTPUT_DIR} -m ${QNN_CHIPSET} -o ${EXPORTED_MODEL_NAME} --compile_only
   EXPORTED_MODEL=./${EXPORT_SCRIPT}/${EXPORTED_MODEL_NAME}
 }
 
@@ -187,8 +189,20 @@ test_model_with_coreml() {
     exit 1
   fi
 
-  "${PYTHON_EXECUTABLE}" -m examples.apple.coreml.scripts.export --model_name="${MODEL_NAME}"
+  DTYPE=float16
+
+  "${PYTHON_EXECUTABLE}" -m examples.apple.coreml.scripts.export --model_name="${MODEL_NAME}" --compute_precision "${DTYPE}"
   EXPORTED_MODEL=$(find "." -type f -name "${MODEL_NAME}*.pte" -print -quit)
+  # TODO:
+  if [ -n "$EXPORTED_MODEL" ]; then
+    EXPORTED_MODEL_WITH_DTYPE="${EXPORTED_MODEL%.pte}_${DTYPE}.pte"
+    mv "$EXPORTED_MODEL" "$EXPORTED_MODEL_WITH_DTYPE"
+    EXPORTED_MODEL="$EXPORTED_MODEL_WITH_DTYPE"
+    echo "Renamed file path: $EXPORTED_MODEL"
+  else
+    echo "No .pte file found"
+    exit 1
+  fi
 }
 
 if [[ "${BACKEND}" == "portable" ]]; then

examples/qualcomm/scripts/deeplab_v3.py

@@ -81,7 +81,7 @@ def main(args):
             data_size=data_num, dataset_dir=args.artifact, download=args.download
         )
 
-    pte_filename = "dlv3_qnn"
+    pte_filename = "dlv3_qnn" if not args.output_filename else args.output_filename
     instance = DeepLabV3ResNet101Model()
 
     build_executorch_binary(
@@ -195,6 +195,14 @@ def post_process():
         default=False,
     )
 
+    parser.add_argument(
+        "-o",
+        "--output_filename",
+        help="name of the output pte file",
+        type=str,
+        required=False,
+    )
+
     args = parser.parse_args()
     try:
         main(args)

examples/qualcomm/scripts/edsr.py

@@ -107,7 +107,7 @@ def main(args):
     )
 
     inputs, targets, input_list = dataset.lr, dataset.hr, dataset.get_input_list()
-    pte_filename = "edsr_qnn"
+    pte_filename = "edsr_qnn" if not args.output_filename else args.output_filename
     instance = EdsrModel()
 
     build_executorch_binary(
@@ -217,6 +217,14 @@ def post_process():
         default=False,
     )
 
+    parser.add_argument(
+        "-o",
+        "--output_filename",
+        help="name of the output pte file",
+        type=str,
+        required=False,
+    )
+
     args = parser.parse_args()
     try:
         main(args)

examples/qualcomm/scripts/inception_v3.py

@@ -46,7 +46,7 @@ def main(args):
             image_shape=(256, 256),
             crop_size=224,
         )
-    pte_filename = "ic3_qnn"
+    pte_filename = "ic3_qnn" if not args.output_filename else args.output_filename
     instance = InceptionV3Model()
     build_executorch_binary(
         instance.get_eager_model().eval(),
@@ -125,6 +125,14 @@ def main(args):
         type=str,
     )
 
+    parser.add_argument(
+        "-o",
+        "--output_filename",
+        help="name of the output pte file",
+        type=str,
+        required=False,
+    )
+
     args = parser.parse_args()
     try:
         main(args)

examples/qualcomm/scripts/inception_v4.py

@@ -45,7 +45,7 @@ def main(args):
             data_size=data_num,
             image_shape=(299, 299),
         )
-    pte_filename = "ic4_qnn"
+    pte_filename = "ic4_qnn" if not args.output_filename else args.output_filename
     instance = InceptionV4Model()
     build_executorch_binary(
         instance.get_eager_model().eval(),
@@ -124,6 +124,14 @@ def main(args):
         type=str,
     )
 
+    parser.add_argument(
+        "-o",
+        "--output_filename",
+        help="name of the output pte file",
+        type=str,
+        required=False,
+    )
+
     args = parser.parse_args()
     try:
         main(args)

examples/qualcomm/scripts/mobilenet_v2.py

@@ -46,7 +46,7 @@ def main(args):
             image_shape=(256, 256),
             crop_size=224,
         )
-    pte_filename = "mv2_qnn"
+    pte_filename = "mv2_qnn" if not args.output_filename else args.output_filename
     instance = MV2Model()
     build_executorch_binary(
         instance.get_eager_model().eval(),
@@ -125,6 +125,14 @@ def main(args):
         type=str,
     )
 
+    parser.add_argument(
+        "-o",
+        "--output_filename",
+        help="name of the output pte file",
+        type=str,
+        required=False,
+    )
+
     args = parser.parse_args()
     try:
         main(args)

examples/qualcomm/scripts/mobilenet_v3.py

@@ -45,7 +45,7 @@ def main(args):
             image_shape=(256, 256),
             crop_size=224,
         )
-    pte_filename = "mv3_qnn"
+    pte_filename = "mv3_qnn" if not args.output_filename else args.output_filename
     instance = MV3Model()
     build_executorch_binary(
         instance.get_eager_model().eval(),
@@ -123,6 +123,14 @@ def main(args):
         type=str,
     )
 
+    parser.add_argument(
+        "-o",
+        "--output_filename",
+        help="name of the output pte file",
+        type=str,
+        required=False,
+    )
+
     args = parser.parse_args()
     try:
         main(args)

examples/qualcomm/scripts/torchvision_vit.py

@@ -38,7 +38,7 @@ def main(args):
             crop_size=224,
         )
 
-    pte_filename = "vit_qnn"
+    pte_filename = "vit_qnn" if not args.output_filename else args.output_filename
     instance = TorchVisionViTModel()
     build_executorch_binary(
         instance.get_eager_model().eval(),
@@ -113,6 +113,14 @@ def main(args):
         type=str,
     )
 
+    parser.add_argument(
+        "-o",
+        "--output_filename",
+        help="name of the output pte file",
+        type=str,
+        required=False,
+    )
+
     args = parser.parse_args()
     try:
         main(args)