refactor workflow

Author: shink

.ci/benchmark.py

@@ -1,15 +1,64 @@
+import argparse
 import os
 import sys
 from src.benchmark.utils import read_metrics, to_markdown_table
 
-if __name__ == "__main__":
-    # Generate statistics report
-    statistics_path = sys.argv[1]
-    metrics = read_metrics(statistics_path, metric="accuracy")
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--path", type=str, required=True, help="Report path.")
+    parser.add_argument("--write-gh-job-summary", action="store_true", help="Write to GitHub job summary.")
+    parser.add_argument("--update-readme", action="store_true", help="Update statistics report in README.md.")
+    return parser.parse_args()
+
+
+def generate_report(path: str):
+    metrics = read_metrics(path, metric="accuracy")
     html_table = to_markdown_table(metrics)
+    return html_table
 
-    # Write to workflow job summary
+
+def write_job_summary(report):
     summary_path = os.environ["GITHUB_STEP_SUMMARY"]
     with open(summary_path, "a") as f:
         f.write("## Torchbenchmark statistics report\n")
-        f.write(html_table)
+        f.write(report)
+
+
+def update_readme(report):
+    project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    readme_path = os.path.join(project_path, "README.md")
+    print(readme_path)
+    with open(readme_path, "r") as f:
+        readme_content = f.read()
+
+    start_marker = "<!-- Torchbenchmark start -->"
+    end_marker = "<!-- Torchbenchmark end -->"
+    start_index = readme_content.find(start_marker)
+    end_index = readme_content.find(end_marker)
+    assert start_index != -1
+    assert end_index != -1
+
+    start_index += len(start_marker)
+    new_readme_content = (
+            readme_content[:start_index] + "\n\n" +
+            report + "\n\n" +
+            readme_content[end_index:]
+    )
+    with open(readme_path, "w") as f:
+        f.write(new_readme_content)
+
+
+if __name__ == "__main__":
+    args = parse_args()
+
+    # Generate statistics report
+    report = generate_report(args.path)
+
+    # Write to workflow job summary
+    if args.write_gh_job_summary:
+        write_job_summary(report)
+
+    # Update README.md
+    if args.update_readme:
+        update_readme(report)

.github/workflows/_ascend_npu_benchmark.yml renamed to .github/workflows/ascend_npu/_ascend_npu_benchmark.yml

@@ -132,17 +132,23 @@ jobs:
       - name: Write to workflow job summary
         id: report
         run: |
-          set -x
-          realpath benchmark/ascend_npu_benchmark.json
-          ls benchmark
-          cat benchmark/ascend_npu_benchmark.json
+          python .ci/benchmark.py --write-gh-job-summary \
+                  --path benchmark/ascend_npu_benchmark.json
 
-          output_path=$(realpath benchmark/ascend_npu_benchmark.json)
-          python .ci/benchmark.py ${output_path}
-
-      # TODO(shink)
       - name: Update README.md
-        if: ${{ github.event_name == 'push' }}
+        if: ${{ github.event_name == 'push' && github.event_name == 'schedule' }}
         run: |
-          echo "${{ github.event_name }}"
-          echo "${{ github.event_name == 'push' }}"
+          python .ci/benchmark.py --update-readme \
+                  --path benchmark/ascend_npu_benchmark.json
+
+      - name: Update README.md
+        if: ${{ github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
+        uses: peter-evans/create-pull-request@v7
+        with:
+          add-paths: README.md
+          branch: ascend-npu/benchmark
+          title: "[Ascend NPU] Update torchbenchmark report in README.md"
+          commit-message: "Update README.md"
+          reviewers: |
+            shink
+            FFFrog

.github/workflows/_ascend_npu_build.yml renamed to .github/workflows/ascend_npu/_ascend_npu_build.yml

@@ -77,7 +77,7 @@ jobs:
     name: Build torch_npu
     needs:
       - prepare
-    uses: ./.github/workflows/_ascend_npu_build.yml
+    uses: ./.github/workflows/ascend_npu/_ascend_npu_build.yml
     with:
       runner: ${{ needs.prepare.outputs.runner }}
       image: ${{ needs.prepare.outputs.image }}
@@ -87,7 +87,7 @@ jobs:
     needs:
       - prepare
       - build
-    uses: ./.github/workflows/_ascend_npu_test.yml
+    uses: ./.github/workflows/ascend_npu/_ascend_npu_test.yml
     with:
       runner: ${{ needs.prepare.outputs.runner }}
       image: ${{ needs.prepare.outputs.image }}
@@ -100,7 +100,7 @@ jobs:
       - prepare
       - build
       - test
-    uses: ./.github/workflows/_ascend_npu_benchmark.yml
+    uses: ./.github/workflows/ascend_npu/_ascend_npu_benchmark.yml
     with:
       runner: ${{ needs.prepare.outputs.runner }}
       image: ${{ needs.prepare.outputs.image }}

.github/workflows/_ascend_npu_test.yml renamed to .github/workflows/ascend_npu/_ascend_npu_test.yml

@@ -7,7 +7,11 @@ across various devices by running comprehensive GitHub workflows.
 
 ## Accelerator Integration Test Results
 
-<!-- Start -->
+<details>
+
+<summary>Torchbenchmark statistics report</summary>
+
+<!-- Torchbenchmark start -->
 
 |                                 | [torch_npu][1] |
 |---------------------------------|----------------|
@@ -121,7 +125,9 @@ across various devices by running comprehensive GitHub workflows.
 
 [3]: https://github.com/cosdt/pytorch-integration-tests/actions/workflows/ascend_npu_test.yml
 
-<!-- End -->
+<!-- Torchbenchmark end -->
+
+</details>
 
 ## Overview
 

.github/workflows/ascend_npu_test.yml

@@ -0,0 +1,10 @@
+device: "npu"
+backend_extension: "torch_npu"
+link: https://github.com/Ascend/pytorch
+torchbenchmark:
+  test:
+    - train
+    - eval
+  models:
+  skip:
+    - llava