Add a new DAG that validates data provided by the tpu-info CLI (#1190)

This change automates a DAG that does cross-validation of TPU performance data. It ensures that the metrics reported directly from the hardware are consistent with the data ingested into the cloud monitoring pipeline. The verification suite includes metrics such as TPU Utilization, TensorCore Activity, Memory Usage, and Latency, and is designed to automatically scale as new metric strategies are added to the validation library.

Author: yuna-tzeng

dags/common/scheduling_helper/scheduling_helper.py

@@ -52,6 +52,7 @@ class DayOfWeek(enum.Enum):
         "tpu_sdk_monitoring_validation": dt.timedelta(minutes=30),
         "jobset_ttr_kill_process": dt.timedelta(minutes=90),
         "jobset_uptime_validation": dt.timedelta(minutes=90),
+        "tpu_info_metrics_verification": dt.timedelta(minutes=30),
     },
 }
 

dags/tpu_observability/interruption_validation_dag.py

@@ -162,7 +162,7 @@ def fetch_interruption_metric_records(
 
   # key: resource_name, value: EventRecord
   event_records: dict[str, EventRecord] = {}
-  response = gcp_util.query_time_series(
+  response = gcp_util.list_time_series(
       project_id=configs.project_id,
       filter_str=metric_filter,
       start_time=time_util.TimeUtil.from_unix_seconds(time_range.start),