Remove fake document (used for poking test runs).

Finish updating metrics.py to collect libc++ testing metrics.

Author: cmtice

.ci/metrics/metrics.py

@@ -30,7 +30,7 @@
 # This metric name is also used as a key in the job->name map.
 GITHUB_WORKFLOW_TO_TRACK = {
     "CI Checks": "github_llvm_premerge_checks",
-    "Build and Test libc++": "github_libc++_premerge_checks",
+    "Build and Test libc++": "github_libcxx_premerge_checks",
 }
 
 # Lists the Github jobs to track for a given workflow. The key is the stable
@@ -42,10 +42,10 @@
         "Build and Test Linux": "premerge_linux",
         "Build and Test Windows": "premerge_windows",
     },
-    "github_libc++_premerge_checks": {
-        "libc++ Stage1 Testing": "premerge_libcxx_stage1",
-        "libc++ Stage2 Testing": "premerge_libcxx_stage2",
-        "libc++ Stage3 Testing": "premerge_libcxx_stage3",
+    "github_libcxx_premerge_checks": {
+        "stage1": "premerge_libcxx_stage1",
+        "stage2": "premerge_libcxx_stage2",
+        "stage3": "premerge_libcxx_stage3",
     },
 }
 
@@ -70,13 +70,14 @@
 # by trial and error).
 GRAFANA_METRIC_MAX_AGE_MN = 120
 
-
 @dataclass
 class JobMetrics:
     job_name: str
     queue_time: int
     run_time: int
     status: int
+    created_at_ns: int
+    started_at_ns: int
     completed_at_ns: int
     workflow_id: int
     workflow_name: str
@@ -89,6 +90,139 @@ class GaugeMetric:
     time_ns: int
 
 
+@dataclass
+class AggregateMetric:
+    aggregate_name: str
+    aggregate_queue_time: int
+    aggregate_run_time: int
+    aggregate_status: int
+    workflow_id: int
+
+
+def create_and_append_libcxx_aggregates(
+    workflow_metrics: list[JobMetrics]) -> list[JobMetrics,AggregateMetric]:
+    """
+    Find libc++ JobMetric entries and create aggregate metrics for them.
+
+    Sort the libc++ JobMetric entries by workflow id, and for each workflow
+    id group them by stages.  Create an aggreate metric for each stage for each
+    unique workflow id.  Append each aggregate metric to the workflow_metrics
+    list.
+
+    How aggreates are computed:
+    queue time: Time from when first job in group is created until last job in
+                group has started.
+    run time: Time from when first job in group starts running until last job
+              in group finishes running.
+    status: logical 'or' of all the job statuses in the group.
+    """
+    # Separate the jobs by workflow_id. Only look at JobMetrics entries.
+    aggregate_data = dict()
+    for job in workflow_metrics:
+        # Only want to look at JobMetrics
+        if not isinstance(job, JobMetrics):
+            continue
+        # Only want libc++ jobs.
+        if job.workflow_name != "Build and Test libc++":
+            continue
+        if job.workflow_id not in aggregate_data.keys():
+            aggregate_data[job.workflow_id] = [ job ]
+        else:
+            aggregate_data[job.workflow_id].append(job)
+
+    # Go through each aggregate_data list (workflow id) and find all the
+    # needed data
+    for ag_workflow_id in aggregate_data:
+        job_list = aggregate_data[ag_workflow_id]
+        stage1_jobs = list()
+        stage2_jobs = list()
+        stage3_jobs = list()
+        # sort jobs into stage1, stage2, & stage3.
+        for job in job_list:
+            if job.job_name.find('stage1') > 0:
+                stage1_jobs.append(job)
+            elif job.job_name.find('stage2') > 0:
+                stage2_jobs.append(job)
+            elif job.job_name.find('stage3') > 0:
+                stage3_jobs.append(job)
+
+        for job_list in [ stage1_jobs, stage2_jobs, stage3_jobs]:
+            if len(job_list) < 1:
+                  # No jobs in that stage this time around.
+                  continue
+
+            # Get the aggregate name.
+            ag_name = "github_libcxx_premerge_checks_"
+            if job_list[0].job_name.find('stage1') > 0:
+                ag_name = ag_name + "stage1_aggregate"
+            elif job_list[0].job_name.find('stage2') > 0:
+                ag_name = ag_name + "stage2_aggregate"
+            elif job_list[0].job_name.find('stage3') > 0:
+                ag_name = ag_name + "stage3_aggregate"
+            else:
+                ag_name = ag_name + "unknown_aggregate"
+
+            # Initialize the rest of the aggregate values
+            earliest_create = job_list[0].created_at_ns
+            earliest_start = job_list[0].started_at_ns
+            earliest_complete = job_list[0].completed_at_ns
+            latest_start = job_list[0].started_at_ns
+            latest_complete = job_list[0].completed_at_ns
+            ag_status = job_list[0].status
+
+            # Go through rest of jobs for this workflow id, updating stats
+            for job in job_list[1:]:
+                # Update the status
+                ag_status = ag_status or job.status
+                # Get the earliest & latest times
+                if job.created_at_ns < earliest_create:
+                    earliest_create = job.created_at_ns
+                if job.completed_at_ns < earliest_complete:
+                    earliest_complete = job.completed_at_ns
+                if job.started_at_ns > latest_start:
+                    latest_start = job.started_at_ns
+                if job.started_at_ns < earliest_start:
+                    earliest_start = job.started_at_ns
+                if job.completed_at_ns > latest_complete:
+                    latest_complete = job.completed_at_ns
+
+            # Compute aggregate run time (in seconds, not ns)
+            ag_run_time = (latest_complete - earliest_start) / 1000000000
+            # Compute aggregate queue time (in seconds, not ns)
+            ag_queue_time = (latest_start - earliest_create) / 1000000000
+            # Append the aggregate metrics to the workflow metrics list.
+            workflow_metrics.append(
+                AggregateMetric(
+                    ag_name, ag_queue_time, ag_run_time, ag_status,
+                    ag_workflow_id
+                )
+            )
+    return
+
+def clean_up_libcxx_job_name(old_name: str) -> str:
+    """
+    Convert libcxx job names to generically legal strings.
+
+    Take a name like 'stage1 (generic-cxx03, clang-22, clang++-22)'
+    and convert it to 'stage1_generic_cxx03__clang_22__clangxx_22'.
+    (Remove parentheses; replace commas, hyphens and spaces with
+    underscores; replace '+' with 'x'.
+    """
+    # Names should have exactly one set of parentheses, so break on that. If
+    # they don't have any parentheses, then don't update them at all.
+    if old_name.find('(') == -1:
+        return old_name
+    stage, remainder = old_name.split('(')
+    stage = stage.strip()
+    if remainder[-1] == ')':
+        remainder = remainder[:-1]
+    remainder = remainder.replace('-', '_')
+    remainder = remainder.replace(',', '_')
+    remainder = remainder.replace(' ', '_')
+    remainder = remainder.replace('+', 'x')
+    new_name = stage + '_' + remainder
+    return new_name
+
 def github_get_metrics(
     github_repo: github.Repository, last_workflows_seen_as_completed: set[int]
 ) -> tuple[list[JobMetrics], int]:
@@ -151,9 +285,14 @@ def github_get_metrics(
             break
 
         # This workflow is not interesting to us.
-        if task.name not in GITHUB_WORKFLOW_TO_TRACK:
+        if (task.name not in GITHUB_WORKFLOW_TO_TRACK
+            and task.name != "Build and Test libc++"):
             continue
 
+        libcxx_testing = False
+        if task.name == "Build and Test libc++":
+          libcxx_testing = True
+
         if task.status == "completed":
             workflow_seen_as_completed.add(task.id)
 
@@ -163,11 +302,20 @@ def github_get_metrics(
 
         name_prefix = GITHUB_WORKFLOW_TO_TRACK[task.name]
         for job in task.jobs():
+            if libcxx_testing:
+                # We're not running macos or windows libc++ tests on our
+                # infrastructure.
+                if (job.name.find("macos") != -1 or
+                    job.name.find("windows") != -1):
+                    continue
             # This job is not interesting to us.
-            if job.name not in GITHUB_JOB_TO_TRACK[name_prefix]:
+            elif job.name not in GITHUB_JOB_TO_TRACK[name_prefix]:
                 continue
 
-            name_suffix = GITHUB_JOB_TO_TRACK[name_prefix][job.name]
+            if libcxx_testing:
+                name_suffix = clean_up_libcxx_job_name(job.name)
+            else:
+                name_suffix = GITHUB_JOB_TO_TRACK[name_prefix][job.name]
             metric_name = name_prefix + "_" + name_suffix
 
             if task.status != "completed":
@@ -216,21 +364,29 @@ def github_get_metrics(
                 continue
 
             logging.info(f"Adding a job metric for job {job.id} in workflow {task.id}")
-            # The timestamp associated with the event is expected by Grafana to be
-            # in nanoseconds.
+            # The timestamp associated with the event is expected by Grafana to
+            # be in nanoseconds.
+            created_at_ns = int(created_at.timestamp()) * 10**9
+            started_at_ns = int(started_at.timestamp()) * 10**9
             completed_at_ns = int(completed_at.timestamp()) * 10**9
             workflow_metrics.append(
                 JobMetrics(
                     metric_name,
                     queue_time.seconds,
                     run_time.seconds,
                     job_result,
+                    created_at_ns,
+                    started_at_ns,
                     completed_at_ns,
                     task.id,
                     task.name,
                 )
             )
 
+    # Finished collecting the JobMetrics for all jobs; now create the
+    # aggregates for any libc++ jobs.
+    create_and_append_libcxx_aggregates(workflow_metrics)
+
     for name, value in queued_count.items():
         workflow_metrics.append(
             GaugeMetric(f"workflow_queue_size_{name}", value, time.time_ns())