fix gpu bugs for cluster with no gpu

Signed-off-by: Mohammed Abdi <[email protected]>

Author: Mohammed Abdi

backend/src/routes/api/mcad-prometheus/metricsData.ts

@@ -47,7 +47,7 @@ const fetchPrometheusDataRange = async (
 
 const getMetric = async (host: string, query: string, axiosInstance: AxiosInstance) => {
   const fetchedData: any = await fetchPrometheusData(host, query, axiosInstance);
-  const valueAsDecimal = parseFloat(fetchedData[0].value[1]);
+  //const valueAsDecimal = parseFloat(fetchedData[0].value[1]);
   return fetchedData;
 };
 

backend/src/routes/api/metrics-data/metricsData.ts

@@ -46,7 +46,7 @@ const fetchPrometheusDataRange = async (
 
 const getMetric = async (host: string, query: string, axiosInstance: AxiosInstance) => {
   const fetchedData: any = await fetchPrometheusData(host, query, axiosInstance);
-  const valueAsDecimal = parseFloat(fetchedData[0].value[1]);
+  //const valueAsDecimal = parseFloat(fetchedData[0].value[1]);
   return fetchedData;
 };
 

frontend/src/pages/MCADashboard/MCADashboard.tsx

@@ -41,6 +41,7 @@ export const MCADashboardInner: React.FC<MCADashboardInnerProps> = React.memo(
           Dispatched: '-',
           Queued: '-',
           'Re-enqueued': '-',
+          Failed: '-',
           Other: '-',
         },
       },

frontend/src/pages/MCADashboard/Metrics/MetricCard.tsx

@@ -19,10 +19,15 @@ const MetricCard: React.FC<MetricCardProps> = ({
   unit,
 }: MetricCardProps): React.ReactElement => {
   const [percentage, setPercentage] = React.useState(0.0);
+  const [noGpu, setNoGpu] = React.useState("");
 
   const getData = async () => {
     const data = await getMetricData(query);
-    setPercentage(Math.round(data * 100) / 100);
+    if (data === 'No GPU In Cluster') {
+      setNoGpu(data);
+    } else {
+      setPercentage(Math.round(data * 100) / 100);
+    }
   };
 
   React.useEffect(() => {
@@ -41,8 +46,16 @@ const MetricCard: React.FC<MetricCardProps> = ({
     <Card className="metric-card">
       <CardHeader className="metric-card-header">{name}</CardHeader>
       <CardBody className="metric-card-data">
-        {Math.round(percentage * 100) / 100}
-        {unit === Unit.PERCENT && unit}
+        {noGpu === "" ? (
+          <>
+            {Math.round(percentage * 100) / 100}
+            {unit === Unit.PERCENT && unit}
+          </>
+        ) : (
+          <>
+            {noGpu}
+          </>
+        )}
       </CardBody>
     </Card>
   );

frontend/src/pages/MCADashboard/Metrics/Metrics.tsx

@@ -22,6 +22,7 @@ const emptyDataObject: Data = {
       Dispatched: '-',
       Queued: '-',
       'Re-enqueued': '-',
+      Failed: '-',
       Other: '-',
     },
   },

frontend/src/pages/MCADashboard/Metrics/Resources.tsx

@@ -22,6 +22,7 @@ const emptyDataObject: Data = {
       Dispatched: '-',
       Queued: '-',
       'Re-enqueued': '-',
+      Failed: '-',
       Other: '-',
     },
   },

frontend/src/pages/MCADashboard/Metrics/api/metricsData.ts

@@ -4,9 +4,30 @@ import React from 'react';
 import { timeStringToSeconds } from '~/pages/MCADashboard/Metrics/metrics-utils';
 
 export const getMetricData = async (query: string) => {
-  const body = { query: query };
-  const res: { data: { value: [string, number] }[] } = await axios.post('/api/metrics-data', body);
-  return res.data[0].value[1];
+  const noGpu = "No GPU In Cluster"
+  const utilizedGPUQuery = 'count(count by (UUID,GPU_I_ID) (DCGM_FI_PROF_GR_ENGINE_ACTIVE{exported_pod=~".+"})) or vector(0)'
+  const utilizedGPUMemoryQuery = 'count(count by (UUID,GPU_I_ID) (DCGM_FI_DEV_MEM_COPY_UTIL))'
+  try {
+    const body = { query: query };
+    const res: { data: { value: [string, number] }[] } = await axios.post('/api/metrics-data', body);
+    if (query === utilizedGPUQuery) { // since vector(0) in query, even if no gpu returns 0
+      const gpubody = { query: utilizedGPUMemoryQuery }; // use the utilizedGPUMemoryQuery to verify gpu is present in the cluster
+      const gpures: { data: { value: [string, number] }[] } = await axios.post('/api/metrics-data', gpubody);
+      if (gpures.data && gpures.data[0] && gpures.data[0].value && gpures.data[0].value[1] !== undefined) {
+        return res.data[0].value[1];
+      } else {
+        return noGpu;
+      }
+    } else {
+      if (res.data && res.data[0] && res.data[0].value && res.data[0].value[1] !== undefined) {
+        return res.data[0].value[1];
+      } else {
+        return noGpu;
+      }
+    }
+  } catch (error) {
+    return 0;
+  }
 };
 
 export const getMetricTableData = async (query: string) => {