|
| 1 | +import argparse |
| 2 | +import csv |
| 3 | +import sys |
| 4 | + |
| 5 | +# |
| 6 | +# A standalone script to generate a Markdown representation of a test report. |
| 7 | +# This is primarily intended to be used with GitHub actions to generate a nice |
| 8 | +# representation of the test results when looking at the action run. |
| 9 | +# |
| 10 | +# Usage: python executorch/backends/test/suite/generate_markdown_summary.py <path to test report CSV file> |
| 11 | +# Markdown is written to stdout. |
| 12 | +# |
| 13 | + |
| 14 | +def generate_markdown(csv_path: str): |
| 15 | + with open(csv_path, newline='', encoding='utf-8') as f: |
| 16 | + reader = csv.reader(f) |
| 17 | + rows = list(reader) |
| 18 | + |
| 19 | + header = rows[0] |
| 20 | + data_rows = rows[1:] |
| 21 | + |
| 22 | + # Find the Result and Result Detail column indices |
| 23 | + result_column_index = None |
| 24 | + result_detail_column_index = None |
| 25 | + for i, col in enumerate(header): |
| 26 | + if col.lower() == 'result': |
| 27 | + result_column_index = i |
| 28 | + elif col.lower() == 'result detail': |
| 29 | + result_detail_column_index = i |
| 30 | + |
| 31 | + # Count results and prepare data |
| 32 | + pass_count = 0 |
| 33 | + fail_count = 0 |
| 34 | + skip_count = 0 |
| 35 | + failed_tests = [] |
| 36 | + processed_rows = [] |
| 37 | + result_detail_counts = {} |
| 38 | + |
| 39 | + for row in data_rows: |
| 40 | + # Make a copy of the row to avoid modifying the original |
| 41 | + processed_row = row.copy() |
| 42 | + |
| 43 | + # Count results and collect failed tests |
| 44 | + if result_column_index is not None and result_column_index < len(row): |
| 45 | + result_value = row[result_column_index].strip().lower() |
| 46 | + if result_value == 'pass': |
| 47 | + pass_count += 1 |
| 48 | + processed_row[result_column_index] = '<span style="color:green">Pass</span>' |
| 49 | + elif result_value == 'fail': |
| 50 | + fail_count += 1 |
| 51 | + processed_row[result_column_index] = '<span style="color:red">Fail</span>' |
| 52 | + failed_tests.append(processed_row.copy()) |
| 53 | + elif result_value == 'skip': |
| 54 | + skip_count += 1 |
| 55 | + processed_row[result_column_index] = '<span style="color:gray">Skip</span>' |
| 56 | + |
| 57 | + # Count result details (excluding empty ones) |
| 58 | + if result_detail_column_index is not None and result_detail_column_index < len(row): |
| 59 | + result_detail_value = row[result_detail_column_index].strip() |
| 60 | + if result_detail_value: # Only count non-empty result details |
| 61 | + if result_detail_value in result_detail_counts: |
| 62 | + result_detail_counts[result_detail_value] += 1 |
| 63 | + else: |
| 64 | + result_detail_counts[result_detail_value] = 1 |
| 65 | + |
| 66 | + processed_rows.append(processed_row) |
| 67 | + |
| 68 | + # Generate Summary section |
| 69 | + total_rows = len(data_rows) |
| 70 | + print("# Summary\n") |
| 71 | + print(f"- **Pass**: {pass_count}/{total_rows}") |
| 72 | + print(f"- **Fail**: {fail_count}/{total_rows}") |
| 73 | + print(f"- **Skip**: {skip_count}/{total_rows}") |
| 74 | + |
| 75 | + print("## Failure Breakdown:") |
| 76 | + total_rows_with_result_detail = sum(result_detail_counts.values()) |
| 77 | + for detail, count in sorted(result_detail_counts.items()): |
| 78 | + print(f"- **{detail}**: {count}/{total_rows_with_result_detail}") |
| 79 | + |
| 80 | + # Generate Failed Tests section |
| 81 | + print("# Failed Tests\n") |
| 82 | + if failed_tests: |
| 83 | + print("| " + " | ".join(header) + " |") |
| 84 | + print("|" + "|".join(['---'] * len(header)) + "|") |
| 85 | + for row in failed_tests: |
| 86 | + print("| " + " | ".join(row) + " |") |
| 87 | + else: |
| 88 | + print("No failed tests.\n") |
| 89 | + print() |
| 90 | + |
| 91 | + # Generate Test Cases section |
| 92 | + print("# Test Cases\n") |
| 93 | + print("| " + " | ".join(header) + " |") |
| 94 | + print("|" + "|".join(['---'] * len(header)) + "|") |
| 95 | + for row in processed_rows: |
| 96 | + print("| " + " | ".join(row) + " |") |
| 97 | + |
| 98 | +def main(): |
| 99 | + parser = argparse.ArgumentParser(description="Generate a Markdown representation of a test report.") |
| 100 | + parser.add_argument("csv_path", help="Path to the test report CSV file.") |
| 101 | + args = parser.parse_args() |
| 102 | + try: |
| 103 | + generate_markdown(args.csv_path) |
| 104 | + except Exception as e: |
| 105 | + print(f"Error: {e}", file=sys.stderr) |
| 106 | + sys.exit(1) |
| 107 | +if __name__ == "__main__": |
| 108 | + main() |
0 commit comments