[Device ID] visualize benchmark graph with group_key (#6593)

add group_key to combine (deviceId, and display name) if any for dp
visualizaition


rename additional_info to metadata_info

---------

Signed-off-by: Yang Wang <[email protected]>

Author: yangw-dev

torchci/clickhouse_queries/oss_ci_benchmark_llms/query.sql

@@ -63,10 +63,19 @@ WITH benchmarks AS (
             IF(
                 tupleElement(o.benchmark, 'extra_info')['failure_type'] = '',
                 '',
-                -- Default to false
+                -- Default to empty
                 tupleElement(o.benchmark, 'extra_info')['failure_type']
-            )
-        ) AS addtional_info --  additional_info for a record
+            ),
+            'device_id',
+            IF(
+                tupleElement(o.benchmark, 'extra_info')['instance_arn'] = '',
+                '',
+                -- Default to empty
+                tupleElement(o.benchmark, 'extra_info')['instance_arn']
+            ),
+            'timestamp',
+            formatDateTime(fromUnixTimestamp(o.timestamp), '%Y-%m-%dT%H:%i:%sZ')
+        ) AS metadata_info --  metadata_info for a record
     FROM
         benchmark.oss_ci_benchmark_v3 o
     WHERE
@@ -119,7 +128,7 @@ SELECT DISTINCT
     arch,
     granularity_bucket,
     extra,
-    addtional_info
+    metadata_info
 FROM
     benchmarks
 WHERE

torchci/components/benchmark/llms/components/LLMsGraphPanel.tsx

@@ -15,6 +15,7 @@ import {
   Tooltip,
   Typography,
 } from "@mui/material";
+import { deepClone } from "@mui/x-data-grid/internals";
 import {
   COMMIT_TO_WORKFLOW_ID,
   WORKFLOW_ID_TO_COMMIT,
@@ -104,10 +105,11 @@ export default function LLMsGraphPanel({
   const lWorkflowId = COMMIT_TO_WORKFLOW_ID[lBranchAndCommit.commit];
   const rWorkflowId = COMMIT_TO_WORKFLOW_ID[rBranchAndCommit.commit];
 
-  const groupByFieldName = "display";
+  const groupByFieldName = "group_key";
 
   const chartData: { [k: string]: any } = {};
   const graphSeries: { [k: string]: any } = {};
+
   metricNames.forEach((metric: string) => {
     if (
       modelName === DEFAULT_MODEL_NAME &&
@@ -210,9 +212,10 @@ export default function LLMsGraphPanel({
 
               return record;
             });
-
+    const graphItems = formGraphItem(chartData[metric]);
+    // group by timestamp to identify devices with the same timestamp
     graphSeries[metric] = seriesWithInterpolatedTimes(
-      chartData[metric],
+      graphItems,
       startTime,
       stopTime,
       granularity,
@@ -371,7 +374,7 @@ const MetricTable = ({
             return (
               <TableRow key={index}>
                 <TableCell>
-                  <span>{entry.granularity_bucket} </span>
+                  <span>{entry?.metadata_info.timestamp} </span>
                 </TableCell>
                 <TableCell sx={{ py: 0.25 }}>
                   <code>
@@ -408,3 +411,18 @@ const MetricTable = ({
     </TableContainer>
   );
 };
+
+// creates chart items to visualize in the series graph, group by device name and display name
+function formGraphItem(data: any[]) {
+  const res: any[] = [];
+  data.forEach((item) => {
+    const deviceId = item.metadata_info.device_id;
+    const displayName = item.display;
+    const group_key =
+      deviceId !== "" ? `${displayName} (${deviceId})` : displayName;
+    const seriesData = deepClone(item);
+    seriesData.group_key = group_key;
+    res.push(seriesData);
+  });
+  return res;
+}

torchci/lib/benchmark/llms/common.ts

@@ -118,7 +118,7 @@ export interface LLMsBenchmarkData {
   arch: string;
   display?: string;
   extra?: { [key: string]: string };
-  additional_info?: { [key: string]: string };
+  metadata_info?: { [key: string]: string };
 }
 
 export interface BranchAndCommitPerfData extends BranchAndCommit {

torchci/lib/benchmark/llms/utils/llmUtils.ts

@@ -401,12 +401,12 @@ const processJobLevelFailureRows = (
       if ("FAILURE_REPORT" in record) {
         const failure_record = record["FAILURE_REPORT"];
         const hasrFailure =
-          "r" in failure_record && failure_record["r"].additional_info
-            ? failure_record["r"].additional_info["failure_type"] === "GIT_JOB"
+          "r" in failure_record && failure_record["r"].metadata_info
+            ? failure_record["r"].metadata_info["failure_type"] === "GIT_JOB"
             : false;
         const haslFailure =
-          "l" in failure_record && failure_record["l"].additional_info
-            ? failure_record["l"].additional_info["failure_type"] === "GIT_JOB"
+          "l" in failure_record && failure_record["l"].metadata_info
+            ? failure_record["l"].metadata_info["failure_type"] === "GIT_JOB"
             : false;
         isJobLevelFailure = hasrFailure || haslFailure;
       }