Update

[ghstack-poisoned]

Author: GregoryComer

.ci/scripts/test_backend_linux.sh

@@ -54,7 +54,6 @@ fi
 PYTHON_EXECUTABLE=python CMAKE_ARGS="$EXTRA_BUILD_ARGS" .ci/scripts/setup-linux.sh --build-tool cmake --build-mode Release --editable true
 
 EXIT_CODE=0
-python -m executorch.backends.test.suite.runner $SUITE --flow $FLOW --report "$REPORT_FILE" || EXIT_CODE=$?
-
+pytest -c /dev/nul -n auto backends/test/suite/$SUITE/ -m flow_$FLOW --json-report --json-report-file="$REPORT_FILE" || EXIT_CODE=$?
 # Generate markdown summary.
-python -m executorch.backends.test.suite.generate_markdown_summary "$REPORT_FILE" > ${GITHUB_STEP_SUMMARY:-"step_summary.md"} --exit-code $EXIT_CODE
+python -m executorch.backends.test.suite.generate_markdown_summary_json "$REPORT_FILE" > ${GITHUB_STEP_SUMMARY:-"step_summary.md"} --exit-code $EXIT_CODE

backends/test/suite/generate_markdown_summary_json.py

@@ -168,6 +168,10 @@ def generate_markdown(json_path: str, exit_code: int = 0):  # noqa (C901)
 
     # Generate Failed Tests section
     print("# Failed Tests\n")
+    print("To reproduce, run the following command from the root of the ExecuTorch repository:")
+    print("```")
+    print("pytest -c /dev/nul backends/test/suite/ -k \"<test_id>\"")
+    print("```")
     if results.failed_tests:
         header = build_header(results.failed_tests)
 

backends/test/suite/operators/test_add.py

@@ -26,7 +26,7 @@ def forward(self, x, y):
 
 
 @pytest.mark.parametrize(
-    "dtype", [torch.float16, torch.float32], ids=lambda s: str(s)[6:]
+    "dtype", [torch.float32], ids=lambda s: str(s)[6:]
 )
 def test_add_dtype(test_runner, dtype) -> None:
     test_runner.lower_and_run_model(