update

Author: shink

.ci/benchmark.py

@@ -0,0 +1,8 @@
+import sys
+from src.benchmark.utils import read_metrics, to_html_table
+
+if __name__ == "__main__":
+    # Generate statistics report
+    metrics = read_metrics(sys.argv[1], metric="accuracy")
+    html_table = to_html_table(metrics)
+    print(html_table)

.ci/benchmark.sh

@@ -70,15 +70,10 @@ jobs:
         run: |
           pip install ${{ inputs.artifact_name }}
 
-      - name: Checkout
+      - name: Checkout benchmark
         uses: actions/checkout@v4
         with:
-          # TODO(shink): Use pytorch/benchmark once following PRs merged:
-          # https://github.com/pytorch/benchmark/pull/2545
-          # https://github.com/pytorch/benchmark/pull/2546
-          # repository: pytorch/benchmark
-          repository: shink/benchmark
-          ref: test
+          repository: pytorch/benchmark
           path: benchmark
 
       - name: Install benchmark dependencies
@@ -99,12 +94,20 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: ascend_npu_benchmark.json
-          path: ascend_npu_benchmark.json
+          path: benchmark/ascend_npu_benchmark.json
           if-no-files-found: error
           retention-days: 1
           overwrite: true
 
+      - name: Generate statistics report
+        id: report
+        run: |
+          output_path=$(realpath benchmark/ascend_npu_benchmark.json)
+          table=$(python .ci/benchmark.py)
+          echo "table=${table}" >> $GITHUB_OUTPUT
+
       - name: Write to workflow job summary
+        if: ${{ steps.report.outputs.table }}
         run: |
           echo "## Torchbenchmark statistics report" >> $GITHUB_STEP_SUMMARY
-          echo "TODO" >> $GITHUB_STEP_SUMMARY
+          echo "${{ steps.report.outputs.table }}" >> $GITHUB_STEP_SUMMARY

.github/workflows/_ascend_npu_benchmark.yml

@@ -67,7 +67,6 @@ jobs:
           # repository: Ascend/pytorch
           repository: shink/torchnpu
           ref: feat/autoload
-          submodules: recursive
           path: torch_npu
 
       - name: Download distribution artifact

.github/workflows/_ascend_npu_test.yml

@@ -7,6 +7,8 @@ across various devices by running comprehensive GitHub workflows.
 
 ## Accelerator Integration Test Results
 
+<!-- Start -->
+
 | [torch_npu][1]                   |
 |----------------------------------|
 | [![Ascend NPU Test Suite][2]][3] | 
@@ -17,6 +19,8 @@ across various devices by running comprehensive GitHub workflows.
 
 [3]: https://github.com/cosdt/pytorch-integration-tests/actions/workflows/ascend_npu_test.yml
 
+<!-- End -->
+
 ## Overview
 
 This repository contains workflows and scripts that automate the testing

README.md

@@ -72,12 +72,6 @@ def read_metrics(path: str, *, metric=None) -> List[TorchBenchModelMetric]:
     return metrics
 
 
-def filter_result(metrics: List[TorchBenchModelMetric], *, model, device):
-    for metric in metrics:
-        if metric.key.name == model and metric.key.device == device:
-            return metric
-
-
 def generate_table(data):
     template = Template("""
     <table border="1">
@@ -96,6 +90,12 @@ def generate_table(data):
 def to_html_table(metrics: List[TorchBenchModelMetric]):
     models = list({metric.key.name for metric in metrics})
     devices = list({metric.key.device for metric in metrics})
+
+    def filter_result(metrics: List[TorchBenchModelMetric], *, model, device):
+        for metric in metrics:
+            if metric.key.name == model and metric.key.device == device:
+                return metric
+
     rows = []
     for model in models:
         row = [model]