diff --git a/backends/test/suite/reporting.py b/backends/test/suite/reporting.py index 15c19bf7c8e..e054bb1685b 100644 --- a/backends/test/suite/reporting.py +++ b/backends/test/suite/reporting.py @@ -1,6 +1,7 @@ import csv from collections import Counter from dataclasses import dataclass +from datetime import timedelta from enum import IntEnum from functools import reduce from typing import TextIO @@ -108,6 +109,12 @@ class TestCaseSummary: a single output tensor. """ + quantize_time: timedelta | None = None + """ The total runtime of the quantization stage, or none, if the test did not run the quantize stage. """ + + lower_time: timedelta | None = None + """ The total runtime of the to_edge_transform_and_lower stage, or none, if the test did not run the quantize stage. """ + class TestSessionState: test_case_summaries: list[TestCaseSummary] @@ -190,6 +197,8 @@ def generate_csv_report(summary: RunSummary, output: TextIO): "Backend", "Flow", "Result", + "Quantize Time (s)", + "Lowering Time (s)", ] # Tests can have custom parameters. We'll want to report them here, so we need @@ -230,6 +239,12 @@ def generate_csv_report(summary: RunSummary, output: TextIO): "Backend": record.backend, "Flow": record.flow, "Result": record.result.display_name(), + "Quantize Time (s)": ( + record.quantize_time.total_seconds() if record.quantize_time else None + ), + "Lowering Time (s)": ( + record.lower_time.total_seconds() if record.lower_time else None + ), } if record.params is not None: row.update({k.capitalize(): v for k, v in record.params.items()}) diff --git a/backends/test/suite/runner.py b/backends/test/suite/runner.py index 6655cf9653b..6ce9c788432 100644 --- a/backends/test/suite/runner.py +++ b/backends/test/suite/runner.py @@ -1,8 +1,10 @@ import argparse import importlib import re +import time import unittest +from datetime import timedelta from typing import Any import torch @@ -44,6 +46,7 @@ def run_test( # noqa: C901 """ error_statistics: list[ErrorStatistics] = [] + extra_stats = {} # Helper method to construct the summary. def build_result( @@ -58,6 +61,7 @@ def build_result( result=result, error=error, tensor_error_statistics=error_statistics, + **extra_stats, ) # Ensure the model can run in eager mode. @@ -72,11 +76,16 @@ def build_result( return build_result(TestResult.UNKNOWN_FAIL, e) if flow.quantize: + start_time = time.perf_counter() try: tester.quantize( flow.quantize_stage_factory() if flow.quantize_stage_factory else None ) + elapsed = time.perf_counter() - start_time + extra_stats["quantize_time"] = timedelta(seconds=elapsed) except Exception as e: + elapsed = time.perf_counter() - start_time + extra_stats["quantize_time"] = timedelta(seconds=elapsed) return build_result(TestResult.QUANTIZE_FAIL, e) try: @@ -87,9 +96,14 @@ def build_result( except Exception as e: return build_result(TestResult.EXPORT_FAIL, e) + lower_start_time = time.perf_counter() try: tester.to_edge_transform_and_lower() + elapsed = time.perf_counter() - lower_start_time + extra_stats["lower_time"] = timedelta(seconds=elapsed) except Exception as e: + elapsed = time.perf_counter() - lower_start_time + extra_stats["lower_time"] = timedelta(seconds=elapsed) return build_result(TestResult.LOWER_FAIL, e) is_delegated = any( @@ -185,6 +199,7 @@ def parse_args(): "--report", nargs="?", help="A file to write the test report to, in CSV format.", + default="backend_test_report.csv", ) return parser.parse_args()