diff --git a/.ci/scripts/test_backend_linux.sh b/.ci/scripts/test_backend_linux.sh index d2282bd7bc0..254d974160a 100755 --- a/.ci/scripts/test_backend_linux.sh +++ b/.ci/scripts/test_backend_linux.sh @@ -10,6 +10,8 @@ SUITE=$1 FLOW=$2 ARTIFACT_DIR=$3 +REPORT_FILE="$ARTIFACT_DIR/test-report-$FLOW-$SUITE.csv" + echo "Running backend test job for suite $SUITE, flow $FLOW." echo "Saving job artifacts to $ARTIFACT_DIR." @@ -48,4 +50,8 @@ fi # We need the runner to test the built library. PYTHON_EXECUTABLE=python CMAKE_ARGS="$EXTRA_BUILD_ARGS" .ci/scripts/setup-linux.sh --build-tool cmake --build-mode Release --editable true -python -m executorch.backends.test.suite.runner $SUITE --flow $FLOW --report "$ARTIFACT_DIR/test_results.csv" +EXIT_CODE=0 +python -m executorch.backends.test.suite.runner $SUITE --flow $FLOW --report "$REPORT_FILE" || EXIT_CODE=$? + +# Generate markdown summary. +python -m executorch.backends.test.suite.generate_markdown_summary "$REPORT_FILE" > ${GITHUB_STEP_SUMMARY:-"step_summary.md"} --exit-code $EXIT_CODE diff --git a/.ci/scripts/test_backend_macos.sh b/.ci/scripts/test_backend_macos.sh index 08ac59809dd..c31fd504b03 100755 --- a/.ci/scripts/test_backend_macos.sh +++ b/.ci/scripts/test_backend_macos.sh @@ -10,6 +10,8 @@ SUITE=$1 FLOW=$2 ARTIFACT_DIR=$3 +REPORT_FILE="$ARTIFACT_DIR/test-report-$FLOW-$SUITE.csv" + echo "Running backend test job for suite $SUITE, flow $FLOW." echo "Saving job artifacts to $ARTIFACT_DIR." @@ -21,4 +23,8 @@ eval "$(conda shell.bash hook)" PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output .ci/scripts/setup-macos.sh --build-tool cmake --build-mode Release -${CONDA_RUN} --no-capture-output python -m executorch.backends.test.suite.runner $SUITE --flow $FLOW --report "$ARTIFACT_DIR/test_results.csv" +EXIT_CODE=0 +${CONDA_RUN} --no-capture-output python -m executorch.backends.test.suite.runner $SUITE --flow $FLOW --report "$REPORT_FILE" || EXIT_CODE=$? + +# Generate markdown summary. +${CONDA_RUN} --no-capture-output python -m executorch.backends.test.suite.generate_markdown_summary "$REPORT_FILE" > ${GITHUB_STEP_SUMMARY:-"step_summary.md"} --exit-code $EXIT_CODE diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1ef89c2ed6d..c220b371c0a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -57,11 +57,8 @@ jobs: upload-artifact: test-report-${{ matrix.flow }}-${{ matrix.suite }} script: | set -eux - # Intentionally suppressing exit code for now. - # TODO (gjcomer) Remove this when jobs are stable. - EXIT_CODE=0 - .ci/scripts/test_backend_linux.sh "${{ matrix.suite }}" "${{ matrix.flow }}" "${RUNNER_ARTIFACT_DIR}" || EXIT_CODE=$? - echo "Test run complete with exit code $EXIT_CODE." + + source .ci/scripts/test_backend_linux.sh "${{ matrix.suite }}" "${{ matrix.flow }}" "${RUNNER_ARTIFACT_DIR}" backend-test-macos: uses: pytorch/test-infra/.github/workflows/macos_job.yml@main @@ -86,6 +83,4 @@ jobs: # This is needed to get the prebuilt PyTorch wheel from S3 ${CONDA_RUN} --no-capture-output pip install awscli==1.37.21 - EXIT_CODE=0 - .ci/scripts/test_backend_macos.sh "${{ matrix.suite }}" "${{ matrix.flow }}" "${RUNNER_ARTIFACT_DIR}" || EXIT_CODE=$? - echo "Test run complete with exit code $EXIT_CODE." + source .ci/scripts/test_backend_macos.sh "${{ matrix.suite }}" "${{ matrix.flow }}" "${RUNNER_ARTIFACT_DIR}" diff --git a/backends/test/suite/generate_markdown_summary.py b/backends/test/suite/generate_markdown_summary.py new file mode 100644 index 00000000000..37bf758fed0 --- /dev/null +++ b/backends/test/suite/generate_markdown_summary.py @@ -0,0 +1,124 @@ +import argparse +import csv +import sys + +# +# A standalone script to generate a Markdown representation of a test report. +# This is primarily intended to be used with GitHub actions to generate a nice +# representation of the test results when looking at the action run. +# +# Usage: python executorch/backends/test/suite/generate_markdown_summary.py +# Markdown is written to stdout. +# + + +def generate_markdown(csv_path: str, exit_code: int = 0): # noqa (C901) + # Print warning if exit code is non-zero + if exit_code != 0: + print("> [!WARNING]") + print( + f"> Exit code {exit_code} was non-zero. Test process may have crashed. Check the job logs for more information.\n" + ) + + with open(csv_path, newline="", encoding="utf-8") as f: + reader = csv.reader(f) + rows = list(reader) + + header = rows[0] + data_rows = rows[1:] + + # Find the Result and Result Detail column indices + result_column_index = None + result_detail_column_index = None + for i, col in enumerate(header): + if col.lower() == "result": + result_column_index = i + elif col.lower() == "result detail": + result_detail_column_index = i + + # Count results and prepare data + pass_count = 0 + fail_count = 0 + skip_count = 0 + failed_tests = [] + processed_rows = [] + result_detail_counts = {} + + for row in data_rows: + # Make a copy of the row to avoid modifying the original + processed_row = row.copy() + + # Count results and collect failed tests + if result_column_index is not None and result_column_index < len(row): + result_value = row[result_column_index].strip().lower() + if result_value == "pass": + pass_count += 1 + processed_row[result_column_index] = ( + 'Pass' + ) + elif result_value == "fail": + fail_count += 1 + processed_row[result_column_index] = ( + 'Fail' + ) + failed_tests.append(processed_row.copy()) + elif result_value == "skip": + skip_count += 1 + processed_row[result_column_index] = ( + 'Skip' + ) + + # Count result details (excluding empty ones) + if result_detail_column_index is not None and result_detail_column_index < len( + row + ): + result_detail_value = row[result_detail_column_index].strip() + if result_detail_value: # Only count non-empty result details + if result_detail_value in result_detail_counts: + result_detail_counts[result_detail_value] += 1 + else: + result_detail_counts[result_detail_value] = 1 + + processed_rows.append(processed_row) + + # Generate Summary section + total_rows = len(data_rows) + print("# Summary\n") + print(f"- **Pass**: {pass_count}/{total_rows}") + print(f"- **Fail**: {fail_count}/{total_rows}") + print(f"- **Skip**: {skip_count}/{total_rows}") + + print("## Failure Breakdown:") + total_rows_with_result_detail = sum(result_detail_counts.values()) + for detail, count in sorted(result_detail_counts.items()): + print(f"- **{detail}**: {count}/{total_rows_with_result_detail}") + + # Generate Failed Tests section + print("# Failed Tests\n") + if failed_tests: + print("| " + " | ".join(header) + " |") + print("|" + "|".join(["---"] * len(header)) + "|") + for row in failed_tests: + print("| " + " | ".join(row) + " |") + else: + print("No failed tests.\n") + + +def main(): + parser = argparse.ArgumentParser( + description="Generate a Markdown representation of a test report." + ) + parser.add_argument("csv_path", help="Path to the test report CSV file.") + parser.add_argument( + "--exit-code", type=int, default=0, help="Exit code from the test process." + ) + args = parser.parse_args() + try: + generate_markdown(args.csv_path, args.exit_code) + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main()