update smoke test

Author: jathu

.ci/scripts/gather_test_models.py

@@ -8,6 +8,7 @@
 import itertools
 import json
 import os
+import sys
 from typing import Any
 
 from examples.models import MODEL_NAME_TO_MODEL
@@ -54,6 +55,12 @@ def parse_args() -> Any:
     from argparse import ArgumentParser
 
     parser = ArgumentParser("Gather all models to test on CI for the target OS")
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        default=False,
+        help="output json",
+    )
     parser.add_argument(
         "--target-os",
         type=str,
@@ -176,7 +183,10 @@ def export_models_for_ci() -> dict[str, dict]:
 
         models["include"].append(record)
 
-    set_output("models", json.dumps(models))
+    if args.json:
+        print(json.dumps(models))
+    else:
+        set_output("models", json.dumps(models))
 
 
 if __name__ == "__main__":

.github/workflows/build-wheels-linux.yml

@@ -41,7 +41,7 @@ jobs:
           - repository: pytorch/executorch
             pre-script: build/packaging/pre_build_script.sh
             post-script: build/packaging/post_build_script.sh
-            smoke-test-script: build/packaging/smoke_test.py
+            smoke-test-script: build/util/packaging/wheel_test.sh
             package-name: executorch
     name: ${{ matrix.repository }}
     uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main

.github/workflows/build-wheels-macos.yml

@@ -41,7 +41,7 @@ jobs:
           - repository: pytorch/executorch
             pre-script: build/packaging/pre_build_script.sh
             post-script: build/packaging/post_build_script.sh
-            smoke-test-script: build/packaging/smoke_test.py
+            smoke-test-script: build/util/packaging/wheel_test.sh
             package-name: executorch
     name: ${{ matrix.repository }}
     uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main

build/packaging/smoke_test.py

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -euxo pipefail
+
+_repo_root_dir=$(git rev-parse --show-toplevel)
+export PYTHONPATH=${_repo_root_dir}
+
+# Event refers to the type of models that will be downloaded. "pull_request"
+# uses higher priority and fast models.
+_models=$(python .ci/scripts/gather_test_models.py --event pull_request --json)
+
+echo "${_models}" | jq -c '.include[]' | while read model; do
+    _model_name=$(echo "$model" | jq -r '.model')
+    _build_tool=$(echo "$model" | jq -r '.["build-tool"]')
+    _backend=$(echo "$model" | jq -r '.backend')
+    _demo_backend_delegation=$(echo "$model" | jq -r '.demo_backend_delegation')
+
+    PYTHON_EXECUTABLE=python .ci/scripts/test_model.sh "${_model_name}" "${_build_tool}" "${_backend}" "${_demo_backend_delegation}"
+done