Merge remote-tracking branch 'origin/main' into expose-static-inline-functions-from-gmf

Author: jiixyj

.ci/generate_test_report.py

@@ -5,11 +5,11 @@
 # python3 -m unittest discover -p generate_test_report.py
 
 import argparse
+import subprocess
 import unittest
 from io import StringIO
 from junitparser import JUnitXml, Failure
 from textwrap import dedent
-from subprocess import check_call
 
 
 def junit_from_xml(xml):
@@ -18,7 +18,7 @@ def junit_from_xml(xml):
 
 class TestReports(unittest.TestCase):
     def test_title_only(self):
-        self.assertEqual(_generate_report("Foo", []), ("", None))
+        self.assertEqual(_generate_report("Foo", []), ("", "success"))
 
     def test_no_tests_in_testsuite(self):
         self.assertEqual(
@@ -105,7 +105,7 @@ def test_report_single_file_single_testsuite(self):
           * 1 test skipped
           * 2 tests failed
 
-          ## Failed tests
+          ## Failed Tests
           (click to see output)
 
           ### Bar
@@ -137,7 +137,7 @@ def test_report_single_file_single_testsuite(self):
         * 1 test skipped
         * 2 tests failed
 
-        ## Failed tests
+        ## Failed Tests
         (click to see output)
 
         ### ABC
@@ -233,12 +233,84 @@ def test_report_multiple_files_multiple_testsuites(self):
             self.MULTI_SUITE_OUTPUT,
         )
 
+    def test_report_dont_list_failures(self):
+        self.assertEqual(
+            _generate_report(
+                "Foo",
+                [
+                    junit_from_xml(
+                        dedent(
+                            """\
+          <?xml version="1.0" encoding="UTF-8"?>
+          <testsuites time="0.02">
+          <testsuite name="Bar" tests="1" failures="1" skipped="0" time="0.02">
+          <testcase classname="Bar/test_1" name="test_1" time="0.02">
+            <failure><![CDATA[Output goes here]]></failure>
+          </testcase>
+          </testsuite>
+          </testsuites>"""
+                        )
+                    )
+                ],
+                list_failures=False,
+            ),
+            (
+                dedent(
+                    """\
+          # Foo
 
-def _generate_report(title, junit_objects):
-    style = None
+          * 1 test failed
 
+          Failed tests and their output was too large to report. Download the build's log file to see the details."""
+                ),
+                "error",
+            ),
+        )
+
+    def test_report_size_limit(self):
+        self.assertEqual(
+            _generate_report(
+                "Foo",
+                [
+                    junit_from_xml(
+                        dedent(
+                            """\
+          <?xml version="1.0" encoding="UTF-8"?>
+          <testsuites time="0.02">
+          <testsuite name="Bar" tests="1" failures="1" skipped="0" time="0.02">
+          <testcase classname="Bar/test_1" name="test_1" time="0.02">
+            <failure><![CDATA[Some long output goes here...]]></failure>
+          </testcase>
+          </testsuite>
+          </testsuites>"""
+                        )
+                    )
+                ],
+                size_limit=128,
+            ),
+            (
+                dedent(
+                    """\
+          # Foo
+
+          * 1 test failed
+
+          Failed tests and their output was too large to report. Download the build's log file to see the details."""
+                ),
+                "error",
+            ),
+        )
+
+
+# Set size_limit to limit the byte size of the report. The default is 1MB as this
+# is the most that can be put into an annotation. If the generated report exceeds
+# this limit and failures are listed, it will be generated again without failures
+# listed. This minimal report will always fit into an annotation.
+# If include failures is False, total number of test will be reported but their names
+# and output will not be.
+def _generate_report(title, junit_objects, size_limit=1024 * 1024, list_failures=True):
     if not junit_objects:
-        return ("", style)
+        return ("", "success")
 
     failures = {}
     tests_run = 0
@@ -264,7 +336,7 @@ def _generate_report(title, junit_objects):
                     )
 
     if not tests_run:
-        return ("", style)
+        return ("", None)
 
     style = "error" if tests_failed else "success"
     report = [f"# {title}", ""]
@@ -281,8 +353,17 @@ def plural(num_tests):
     if tests_failed:
         report.append(f"* {tests_failed} {plural(tests_failed)} failed")
 
-    if failures:
-        report.extend(["", "## Failed tests", "(click to see output)"])
+    if not list_failures:
+        report.extend(
+            [
+                "",
+                "Failed tests and their output was too large to report. "
+                "Download the build's log file to see the details.",
+            ]
+        )
+    elif failures:
+        report.extend(["", "## Failed Tests", "(click to see output)"])
+
         for testsuite_name, failures in failures.items():
             report.extend(["", f"### {testsuite_name}"])
             for name, output in failures:
@@ -298,7 +379,11 @@ def plural(num_tests):
                     ]
                 )
 
-    return "\n".join(report), style
+    report = "\n".join(report)
+    if len(report.encode("utf-8")) > size_limit:
+        return _generate_report(title, junit_objects, size_limit, list_failures=False)
+
+    return report, style
 
 
 def generate_report(title, junit_files):
@@ -315,14 +400,24 @@ def generate_report(title, junit_files):
     args = parser.parse_args()
 
     report, style = generate_report(args.title, args.junit_files)
-    check_call(
-        [
-            "buildkite-agent",
-            "annotate",
-            "--context",
-            args.context,
-            "--style",
-            style,
-            report,
-        ]
-    )
+
+    if report:
+        p = subprocess.Popen(
+            [
+                "buildkite-agent",
+                "annotate",
+                "--context",
+                args.context,
+                "--style",
+                style,
+            ],
+            stdin=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+        )
+
+        # The report can be larger than the buffer for command arguments so we send
+        # it over stdin instead.
+        _, err = p.communicate(input=report)
+        if p.returncode:
+            raise RuntimeError(f"Failed to send report to buildkite-agent:\n{err}")

.ci/metrics/Dockerfile

@@ -0,0 +1,7 @@
+FROM docker.io/python:3.12
+
+COPY requirements.lock.txt ./
+RUN pip3 install --no-cache-dir -r requirements.lock.txt
+COPY metrics.py ./
+
+CMD ["python3", "metrics.py"]

.ci/metrics/metrics.py

@@ -0,0 +1,182 @@
+import requests
+import time
+import os
+from dataclasses import dataclass
+import sys
+
+import github
+from github import Github
+from github import Auth
+
+GRAFANA_URL = (
+    "https://influx-prod-13-prod-us-east-0.grafana.net/api/v1/push/influx/write"
+)
+GITHUB_PROJECT = "llvm/llvm-project"
+WORKFLOWS_TO_TRACK = ["Check code formatting"]
+SCRAPE_INTERVAL_SECONDS = 5 * 60
+
+
+@dataclass
+class JobMetrics:
+    job_name: str
+    queue_time: int
+    run_time: int
+    status: int
+    created_at_ns: int
+    workflow_id: int
+
+
+def get_metrics(github_repo: github.Repository, workflows_to_track: dict[str, int]):
+    """Gets the metrics for specified Github workflows.
+
+    This function takes in a list of workflows to track, and optionally the
+    workflow ID of the last tracked invocation. It grabs the relevant data
+    from Github, returning it to the caller.
+
+    Args:
+      github_repo: A github repo object to use to query the relevant information.
+      workflows_to_track: A dictionary mapping workflow names to the last
+        invocation ID where metrics have been collected, or None to collect the
+        last five results.
+
+    Returns:
+      Returns a list of JobMetrics objects, containing the relevant metrics about
+      the workflow.
+    """
+    workflow_runs = iter(github_repo.get_workflow_runs())
+
+    workflow_metrics = []
+
+    workflows_to_include = set(workflows_to_track.keys())
+
+    while len(workflows_to_include) > 0:
+        workflow_run = next(workflow_runs)
+        if workflow_run.status != "completed":
+            continue
+
+        # This workflow was already sampled for this run, or is not tracked at
+        # all. Ignoring.
+        if workflow_run.name not in workflows_to_include:
+            continue
+
+        # There were no new workflow invocations since the previous scrape.
+        # The API returns a sorted list with the most recent invocations first,
+        # so we can stop looking for this particular workflow. Continue to grab
+        # information on the other workflows of interest, if present.
+        if workflows_to_track[workflow_run.name] == workflow_run.id:
+            workflows_to_include.remove(workflow_run.name)
+            continue
+
+        workflow_jobs = workflow_run.jobs()
+        if workflow_jobs.totalCount == 0:
+            continue
+        if workflow_jobs.totalCount > 1:
+            raise ValueError(
+                f"Encountered an unexpected number of jobs: {workflow_jobs.totalCount}"
+            )
+
+        created_at = workflow_jobs[0].created_at
+        started_at = workflow_jobs[0].started_at
+        completed_at = workflow_jobs[0].completed_at
+
+        job_result = int(workflow_jobs[0].conclusion == "success")
+
+        queue_time = started_at - created_at
+        run_time = completed_at - started_at
+
+        if run_time.seconds == 0:
+            continue
+
+        if (
+            workflows_to_track[workflow_run.name] is None
+            or workflows_to_track[workflow_run.name] == workflow_run.id
+        ):
+            workflows_to_include.remove(workflow_run.name)
+        if (
+            workflows_to_track[workflow_run.name] is not None
+            and len(workflows_to_include) == 0
+        ):
+            break
+
+        # The timestamp associated with the event is expected by Grafana to be
+        # in nanoseconds.
+        created_at_ns = int(created_at.timestamp()) * 10**9
+
+        workflow_metrics.append(
+            JobMetrics(
+                workflow_run.name,
+                queue_time.seconds,
+                run_time.seconds,
+                job_result,
+                created_at_ns,
+                workflow_run.id,
+            )
+        )
+
+    return workflow_metrics
+
+
+def upload_metrics(workflow_metrics, metrics_userid, api_key):
+    """Upload metrics to Grafana.
+
+    Takes in a list of workflow metrics and then uploads them to Grafana
+    through a REST request.
+
+    Args:
+      workflow_metrics: A list of metrics to upload to Grafana.
+      metrics_userid: The userid to use for the upload.
+      api_key: The API key to use for the upload.
+    """
+    metrics_batch = []
+    for workflow_metric in workflow_metrics:
+        workflow_formatted_name = workflow_metric.job_name.lower().replace(" ", "_")
+        metrics_batch.append(
+            f"{workflow_formatted_name} queue_time={workflow_metric.queue_time},run_time={workflow_metric.run_time},status={workflow_metric.status} {workflow_metric.created_at_ns}"
+        )
+
+    request_data = "\n".join(metrics_batch)
+    response = requests.post(
+        GRAFANA_URL,
+        headers={"Content-Type": "text/plain"},
+        data=request_data,
+        auth=(metrics_userid, api_key),
+    )
+
+    if response.status_code < 200 or response.status_code >= 300:
+        print(
+            f"Failed to submit data to Grafana: {response.status_code}", file=sys.stderr
+        )
+
+
+def main():
+    # Authenticate with Github
+    auth = Auth.Token(os.environ["GITHUB_TOKEN"])
+    github_object = Github(auth=auth)
+    github_repo = github_object.get_repo("llvm/llvm-project")
+
+    grafana_api_key = os.environ["GRAFANA_API_KEY"]
+    grafana_metrics_userid = os.environ["GRAFANA_METRICS_USERID"]
+
+    workflows_to_track = {}
+    for workflow_to_track in WORKFLOWS_TO_TRACK:
+        workflows_to_track[workflow_to_track] = None
+
+    # Enter the main loop. Every five minutes we wake up and dump metrics for
+    # the relevant jobs.
+    while True:
+        current_metrics = get_metrics(github_repo, workflows_to_track)
+        if len(current_metrics) == 0:
+            print("No metrics found to upload.", file=sys.stderr)
+            continue
+
+        upload_metrics(current_metrics, grafana_metrics_userid, grafana_api_key)
+        print(f"Uploaded {len(current_metrics)} metrics", file=sys.stderr)
+
+        for workflow_metric in reversed(current_metrics):
+            workflows_to_track[workflow_metric.job_name] = workflow_metric.workflow_id
+
+        time.sleep(SCRAPE_INTERVAL_SECONDS)
+
+
+if __name__ == "__main__":
+    main()