[CI] Track unique contributors to LLVM over time (#534)

This change uploads additional metrics to Grafana so that we may
visualize LLVM contributor data. From newly scraped commits and stored
data from BigQuery, we can determine the number of unique LLVM
contributors per day, the number of all time unique LLVM contributors,
and the number of new contributors to LLVM per day.

Author: jriv01

llvm-ops-metrics/ops-container/process_llvm_commits.py

@@ -195,23 +195,56 @@ def query_for_reviews(
   return list(new_commits.values())
 
 
+def get_past_contributors(bq_client: bigquery.Client) -> set[str]:
+  """Get past contributors to LLVM from BigQuery dataset.
+
+  Args:
+    bq_client: The BigQuery client to use.
+
+  Returns:
+    Set of unique past contributors to LLVM.
+  """
+  results = bq_client.query("""
+      SELECT
+        DISTINCT commit_author
+      FROM %s.%s
+      WHERE commit_author IS NOT NULL
+      """ % (OPERATIONAL_METRICS_DATASET, LLVM_COMMITS_TABLE)).result()
+  return set(row.commit_author for row in results)
+
+
 def upload_daily_metrics_to_grafana(
     grafana_api_key: str,
     grafana_metrics_userid: str,
     new_commits: list[LLVMCommitInfo],
+    past_contributors: set[str],
 ) -> None:
   """Upload daily commit metrics to Grafana.
 
   Args:
     grafana_api_key: The key to make API requests with.
     grafana_metrics_userid: The user to make API requests with.
     new_commits: List of commits to process & upload to Grafana.
+    past_contributors: Set of unique past contributors to LLVM.
   """
+
+  def post_data(data: str) -> None:
+    """Helper function to post data to Grafana."""
+    response = requests.post(
+        GRAFANA_URL,
+        headers={"Content-Type": "text/plain"},
+        data=data,
+        auth=(grafana_metrics_userid, grafana_api_key),
+    )
+    if response.status_code < 200 or response.status_code >= 300:
+      logging.error("Failed to submit data to Grafana: %s", response.text)
+
   # Count each type of commit made
   approval_count = 0
   review_count = 0
   pull_request_count = 0
   push_count = 0
+  contributors = set()
   for commit in new_commits:
     if commit.is_approved:
       approval_count += 1
@@ -221,44 +254,49 @@ def upload_daily_metrics_to_grafana(
       pull_request_count += 1
     else:
       push_count += 1
+    contributors.add(commit.commit_author)
 
   # Post data via InfluxDB API call
+  # Commit data
   request_data = (
       "llvm_project_main_daily_commits"
       " approval_count={},review_count={},pull_request_count={},push_count={}"
   ).format(approval_count, review_count, pull_request_count, push_count)
-  response = requests.post(
-      GRAFANA_URL,  # Set timestamp precision to seconds
-      headers={"Content-Type": "text/plain"},
-      data=request_data,
-      auth=(grafana_metrics_userid, grafana_api_key),
-  )
+  post_data(request_data)
 
-  if response.status_code < 200 or response.status_code >= 300:
-    logging.error("Failed to submit data to Grafana: %s", response.text)
+  # Contributor data
+  request_data = (
+      "llvm_project_main"
+      " daily_unique_contributor_count={},all_time_unique_contributor_count={}"
+      .format(len(contributors), len(contributors | past_contributors))
+  )
+  post_data(request_data)
 
 
-def upload_daily_metrics_to_bigquery(new_commits: list[LLVMCommitInfo]) -> None:
+def upload_daily_metrics_to_bigquery(
+    bq_client: bigquery.Client, new_commits: list[LLVMCommitInfo]
+) -> None:
   """Upload processed commit metrics to a BigQuery dataset.
 
   Args:
+    bq_client: The BigQuery client to use.
     new_commits: List of commits to process & upload to BigQuery.
   """
-  bq_client = bigquery.Client()
   table_ref = bq_client.dataset(OPERATIONAL_METRICS_DATASET).table(
       LLVM_COMMITS_TABLE
   )
   table = bq_client.get_table(table_ref)
   commit_records = [dataclasses.asdict(commit) for commit in new_commits]
   bq_client.insert_rows(table, commit_records)
-  bq_client.close()
 
 
 def main() -> None:
   github_token = os.environ["GITHUB_TOKEN"]
   grafana_api_key = os.environ["GRAFANA_API_KEY"]
   grafana_metrics_userid = os.environ["GRAFANA_METRICS_USERID"]
 
+  bq_client = bigquery.Client()
+
   # Scrape new commits
   date_to_scrape = datetime.datetime.now(
       datetime.timezone.utc
@@ -275,13 +313,21 @@ def main() -> None:
   logging.info("Querying for reviews of new commits.")
   new_commit_info = query_for_reviews(new_commits, github_token)
 
+  logging.info("Getting set of past LLVM contributors.")
+  past_contributors = get_past_contributors(bq_client)
+
   logging.info("Uploading metrics to Grafana.")
   upload_daily_metrics_to_grafana(
-      grafana_api_key, grafana_metrics_userid, new_commit_info
+      grafana_api_key,
+      grafana_metrics_userid,
+      new_commit_info,
+      past_contributors,
   )
 
   logging.info("Uploading metrics to BigQuery.")
-  upload_daily_metrics_to_bigquery(new_commit_info)
+  upload_daily_metrics_to_bigquery(bq_client, new_commit_info)
+
+  bq_client.close()
 
 
 if __name__ == "__main__":