Util UI for allocated memory (#6878)

UI fix to include the mem allocation and mem bandwidth for gpu

Signed-off-by: Yang Wang <[email protected]>

Author: yangw-dev

torchci/clickhouse_queries/oss_ci_util/oss_ci_list_util_stats/query.sql

@@ -5,15 +5,18 @@ WITH aggregate_data AS (
     max(JSONExtractFloat(json_data, 'cpu','max')) as cpu_max,
     max(JSONExtractFloat(json_data, 'memory','max')) as memory_max,
     max(arrayMax(arrayMap(x->JSONExtractFloat(x,'util_percent','max'),JSONExtractArrayRaw(json_data,'gpu_usage')))) as gpu_max,
-    max(arrayMax(arrayMap(x->JSONExtractFloat(x,'mem_util_percent','max'),JSONExtractArrayRaw(json_data,'gpu_usage')))) as gpu_mem_max,
+    max(arrayMax(arrayMap(x->JSONExtractFloat(x,'mem_util_percent','max'),JSONExtractArrayRaw(json_data,'gpu_usage')))) as gpu_mem_bandwidth_max,
+    max(arrayMax(arrayMap(x -> JSONExtractFloat(x, 'allocated_mem_percent', 'max'),JSONExtractArrayRaw(json_data, 'gpu_usage'))))as gpu_allocated_mem_max,
     avg(JSONExtractFloat(json_data, 'cpu','max')) as cpu_avg,
     avg(JSONExtractFloat(json_data, 'memory','max')) as memory_avg,
     avg(arrayAvg(arrayMap(x->JSONExtractFloat(x,'util_percent','max'),JSONExtractArrayRaw(json_data,'gpu_usage')))) as gpu_avg,
-    avg(arrayAvg(arrayMap(x->JSONExtractFloat(x,'mem_util_percent','max'),JSONExtractArrayRaw(json_data,'gpu_usage')))) as gpu_mem_avg,
+    avg(arrayAvg(arrayMap(x->JSONExtractFloat(x,'mem_util_percent','max'),JSONExtractArrayRaw(json_data,'gpu_usage')))) as gpu_mem_bandwidth_avg,
+    avg(arrayAvg(arrayFilter(x -> x IS NOT NULL,arrayMap(x->JSONExtractFloat(x,'allocated_mem_percent','max'),JSONExtractArrayRaw(json_data,'gpu_usage'))))) as gpu_allocated_mem_avg,
     quantile(0.9)(JSONExtractFloat(json_data, 'cpu','max')) AS cpu_p90,
     quantile(0.9)(JSONExtractFloat(json_data, 'memory','max')) AS memory_p90,
     quantile(0.9)(arrayMax(arrayMap(x->JSONExtractFloat(x,'util_percent','max'),JSONExtractArrayRaw(json_data,'gpu_usage')))) as gpu_p90,
-    quantile(0.9)(arrayMax(arrayMap(x->JSONExtractFloat(x,'mem_util_percent','max'),JSONExtractArrayRaw(json_data,'gpu_usage')))) as gpu_mem_p90
+    quantile(0.9)(arrayMax(arrayMap(x->JSONExtractFloat(x,'mem_util_percent','max'),JSONExtractArrayRaw(json_data,'gpu_usage')))) as gpu_mem_bandwidth_p90,
+    quantile(0.9)(arrayMax(arrayFilter(x -> x IS NOT NULL,arrayMap(x->JSONExtractFloat(x,'allocated_mem_percent','max'),JSONExtractArrayRaw(json_data,'gpu_usage'))))) as gpu_allocated_mem_p90
 FROM
     misc.oss_ci_time_series
 WHERE
@@ -39,12 +42,15 @@ SELECT
     a.memory_avg,
     a.gpu_max,
     a.gpu_avg,
-    a.gpu_mem_max,
-    a.gpu_mem_avg,
+    a.gpu_mem_bandwidth_max,
+    a.gpu_mem_bandwidth_avg,
     a.cpu_p90,
     a.memory_p90,
-    a.gpu_mem_p90,
-    a.gpu_p90
+    a.gpu_mem_bandwidth_p90,
+    a.gpu_p90,
+    a.gpu_allocated_mem_max,
+    a.gpu_allocated_mem_avg,
+    a.gpu_allocated_mem_p90
 FROM
     misc.oss_ci_utilization_metadata o
     JOIN aggregate_data a ON a.job_id = o.job_id AND a.run_attempt = o.run_attempt

torchci/components/utilization/JobUtilizationPage/JobUtilizationPage.tsx

@@ -18,7 +18,8 @@ export const lineFilters: PickerConfig[] = [
     category: "gpu",
     types: [
       { name: "gpu util", tags: ["gpu", "|util_percent"] },
-      { name: "gpu mem", tags: ["gpu", "|mem_util_percent"] },
+      { name: "gpu mem bw", tags: ["gpu", "|mem_util_percent"] },
+      { name: "gpu allocated mem", tags: ["gpu", "|allocated_mem_percent"] },
     ],
   },
   { category: "cpu", types: [{ name: "cpu", tags: ["cpu"] }] },
@@ -45,6 +46,7 @@ export const JobUtilizationPage = ({
   // this makes sense for utilization to detect potential effieciency issues, later our ui
   // can support other aggregation methods for analysis, it's very disruptive to add both in UI right now.
   const aggregateType = "max";
+  const skipLineTypes = ["_mem_value", "total_mem"];
 
   useEffect(() => {
     if (!data) {
@@ -55,9 +57,15 @@ export const JobUtilizationPage = ({
     const lines = data.ts_list;
 
     // currently we only show data that is aggregated by max value during the time interval
-    const filteredLines = lines.filter((line) =>
-      line.id.includes(aggregateType)
-    );
+    const filteredLines = lines.filter((line) => {
+      const skiplineType = skipLineTypes.find((skipLineType) => {
+        return line.id.includes(skipLineType);
+      });
+      if (skiplineType) {
+        return false;
+      }
+      return line.id.includes(aggregateType);
+    });
 
     const jobStats: StatsInfo[] = processStatsData(filteredLines);
 

torchci/components/utilization/JobUtilizationPage/helper.ts

@@ -257,7 +257,12 @@ export function calculateAverage(data: number[]) {
 function getAllGpusStats(stats: StatsInfo[]) {
   // get all gpus stats for the test
   const gpuUtils = stats.filter((item) => item.id.includes("|util_percent"));
-  const gpuMems = stats.filter((item) => item.id.includes("|mem_util_percent"));
+  const gpuMemBWs = stats.filter((item) =>
+    item.id.includes("|mem_util_percent")
+  );
+  const gpuAllocMems = stats.filter((item) =>
+    item.id.includes("|allocated_mem")
+  );
 
   if (gpuUtils.length == 0) {
     return [];
@@ -284,18 +289,40 @@ function getAllGpusStats(stats: StatsInfo[]) {
       ],
     },
     {
-      name: "gpu_mem_all",
-      id: "gpu_mem_all",
+      name: "gpu_mem_bw_all",
+      id: "gpu_mem_bw_all",
       columns: [
-        aggregateStats(gpuMems, StatType.Average, AgggregateMethod.Average),
-        aggregateStats(gpuMems, StatType.Max, AgggregateMethod.Max),
+        aggregateStats(gpuMemBWs, StatType.Average, AgggregateMethod.Average),
+        aggregateStats(gpuMemBWs, StatType.Max, AgggregateMethod.Max),
         aggregateStats(
-          gpuMems,
+          gpuMemBWs,
           StatType.SpikeFrequency,
           AgggregateMethod.Average
         ),
         aggregateStats(
-          gpuMems,
+          gpuMemBWs,
+          StatType.SpikeAvgInterval,
+          AgggregateMethod.Max
+        ),
+      ],
+    },
+    {
+      name: "gpu_allocated_mem_all",
+      id: "gpu_allocated_mem_all",
+      columns: [
+        aggregateStats(
+          gpuAllocMems,
+          StatType.Average,
+          AgggregateMethod.Average
+        ),
+        aggregateStats(gpuAllocMems, StatType.Max, AgggregateMethod.Max),
+        aggregateStats(
+          gpuAllocMems,
+          StatType.SpikeFrequency,
+          AgggregateMethod.Average
+        ),
+        aggregateStats(
+          gpuAllocMems,
           StatType.SpikeAvgInterval,
           AgggregateMethod.Max
         ),

torchci/components/utilization/WorkflowUtilizationPage/WorkflowUtilizationPage.tsx

@@ -48,6 +48,8 @@ const WorkflowUtilization = () => {
     };
   });
 
+  console.log(rows);
+
   const columns: any[] = [
     { field: "name", headerName: "Job Name", width: 400 },
     { field: "id", headerName: "Job id", width: 120 },

torchci/components/utilization/components/TestSectionView/RankTestView/RankTestView.tsx

@@ -35,8 +35,12 @@ const DefaultGpuResourceValue = [
     value: "gpus_util_all",
   },
   {
-    name: "all gpu memory",
-    value: "gpu_mem_all",
+    name: "all gpu memory bandwidth",
+    value: "gpu_mem_bw_all",
+  },
+  {
+    name: "all gpu allocated memory ",
+    value: "gpu_allocated_mem_all",
   },
 ];
 

torchci/lib/utilization/fetchListUtilizationMetadataInfo.ts

@@ -95,9 +95,24 @@ function toUtilizationStats(metadata: any) {
     gpu_max: metadata.gpu_count ? metadata.gpu_max : undefined,
     gpu_avg: metadata.gpu_count ? metadata.gpu_avg : undefined,
     gpu_p90: metadata.gpu_count ? metadata.gpu_p90 : undefined,
-    gpu_memory_max: metadata.gpu_count ? metadata.gpu_mem_max : undefined,
-    gpu_memory_avg: metadata.gpu_count ? metadata.gpu_mem_avg : undefined,
-    gpu_memmory_p90: metadata.gpu_count ? metadata.gpu_mem_p90 : undefined,
+    gpu_memory_bandwidth_max: metadata.gpu_count
+      ? metadata.gpu_mem_bandwidth_max
+      : undefined,
+    gpu_memory_bandwidth_avg: metadata.gpu_count
+      ? metadata.gpu_mem_bandwidth_avg
+      : undefined,
+    gpu_memory_bandwidth_p90: metadata.gpu_count
+      ? metadata.gpu_mem_bandwidth_p90
+      : undefined,
+    gpu_allocated_memory_max: metadata.gpu_count
+      ? metadata.gpu_allocated_mem_max
+      : undefined,
+    gpu_allocated_memory_avg: metadata.gpu_count
+      ? metadata.gpu_allocated_mem_avg
+      : undefined,
+    gpu_allocated_memory_p90: metadata.gpu_count
+      ? metadata.gpu_allocated_mem_p90
+      : undefined,
     has_gpu: metadata.gpu_count ? metadata.gpu_count > 0 : false,
   };
   return stats;

torchci/lib/utilization/fetchUtilization.ts

@@ -99,7 +99,9 @@ function getDisplayName(name: string) {
   }
   if (tags[0].toLowerCase().includes("gpu")) {
     if (name.includes("mem_util")) {
-      return `gpu_${tags[1]}_mem`;
+      return `gpu_${tags[1]}_mem_bandwidth`;
+    } else if (name.includes("allocated_mem_percent")) {
+      return `gpu_${tags[1]}_allocated_mem_percent`;
     }
     return `gpu_${tags[1]}_util`;
   }

torchci/lib/utilization/types.ts

@@ -85,10 +85,13 @@ export interface UtilizationAggreStats {
   memory_p90: number;
   gpu_max?: number;
   gpu_avg?: number;
-  gpu_memory_max?: number;
-  gpu_memory_avg?: number;
   gpu_p90?: number;
-  gpu_memmory_p90?: number;
+  gpu_memory_bandwidth_max?: number;
+  gpu_memory_bandwidth_avg?: number;
+  gpu_memory_bandwidth_p90?: number;
+  gpu_allocated_memory_max?: number;
+  gpu_allocated_memory_avg?: number;
+  gpu_allocated_memory_p90?: number;
 }
 
 /**