Skip to content

Commit 66dffc3

Browse files
author
iGavroche
committed
Fix AMD GPU monitoring: dynamic rocm-smi CSV field detection
1 parent d82bfb6 commit 66dffc3

2 files changed

Lines changed: 55 additions & 57 deletions

File tree

ui/package-lock.json

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

ui/src/app/api/gpu/route.ts

Lines changed: 48 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -576,8 +576,8 @@ async function getRocmGpuStats(isWindows: boolean) {
576576
.map(line => line.trim())
577577
.filter(line => line.length > 0 && !line.startsWith('Exception') && !line.startsWith('Error'));
578578

579-
// Find the header line (should contain "device,GPU ID")
580-
const headerIndex = lines.findIndex(line => line.includes('device,GPU ID') || line.startsWith('device,'));
579+
// Find the header line (should start with "device,")
580+
const headerIndex = lines.findIndex(line => line.startsWith('device,'));
581581

582582
if (headerIndex === -1 || lines.length < headerIndex + 2) {
583583
return [];
@@ -604,44 +604,45 @@ async function getRocmGpuStats(isWindows: boolean) {
604604
return fields;
605605
}
606606

607+
// Parse header to dynamically find field indices (works with any rocm-smi version)
608+
const headerLine = lines[headerIndex];
609+
const headerFields = parseCSVLine(headerLine);
610+
611+
function findHeaderIndex(namePatterns: string[]): number {
612+
for (let i = 0; i < headerFields.length; i++) {
613+
const h = headerFields[i].toLowerCase().trim();
614+
for (const pattern of namePatterns) {
615+
if (h.includes(pattern.toLowerCase())) {
616+
return i;
617+
}
618+
}
619+
}
620+
return -1;
621+
}
622+
623+
const tempFieldIdx = findHeaderIndex(['temperature', '(c)', 'temp']);
624+
const mclkFieldIdx = findHeaderIndex(['mclk clock speed', 'mclk']);
625+
const sclkFieldIdx = findHeaderIndex(['sclk clock speed', 'sclk']);
626+
const powerFieldIdx = findHeaderIndex(['power (w)', 'power']);
627+
const usageFieldIdx = findHeaderIndex(['gpu use', 'gpu_use', 'gpu use (%)']);
628+
const memTotalFieldIdx = findHeaderIndex(['vram total memory', 'vram total']);
629+
const memUsedFieldIdx = findHeaderIndex(['vram total used memory', 'vram total used', 'vram used']);
630+
const cardSkuFieldIdx = findHeaderIndex(['card sku', 'sku']);
631+
const cardModelFieldIdx = findHeaderIndex(['card model', 'model']);
632+
const cardNameFieldIdx = findHeaderIndex(['device name']);
633+
const deviceIdFieldIdx = findHeaderIndex(['device id', 'gpu id']);
634+
const cardVendorFieldIdx = findHeaderIndex(['card vendor', 'vendor']);
635+
607636
// Skip header line and process data lines
608637
const gpus = lines.slice(headerIndex + 1).map((line, idx) => {
609-
// Parse CSV line - rocm-smi CSV format has changed!
610-
// New format (25 fields): device,Device Name,Device ID,Device Rev,Subsystem ID,GUID,Temperature (Sensor edge) (C),mclk clock speed:,mclk clock level:,sclk clock speed:,sclk clock level:,socclk clock speed:,socclk clock level:,Current Socket Graphics Package Power (W),GPU use (%),GPU Memory Allocated (VRAM%),Memory Activity,VRAM Total Memory (B),VRAM Total Used Memory (B),Card Series,Card Model,Card Vendor,Card SKU,Node ID,GFX Version
611-
// Old format (18 fields): device,GPU ID,Temperature,mclk clock speed,mclk level,sclk clock speed,sclk level,socclk speed,socclk level,Power,GPU use,Memory Activity,VRAM Total,VRAM Used,Card series,Card model,Card vendor,Card SKU
612638
const fields = parseCSVLine(line);
613639

614-
// Detect format based on field count
615-
const isNewFormat = fields.length >= 25;
616-
const isOldFormat = fields.length >= 18 && fields.length < 25;
617-
618-
if (!isNewFormat && !isOldFormat) {
619-
// Pad with empty strings to prevent index errors
620-
while (fields.length < 25) {
621-
fields.push('');
622-
}
623-
}
624-
625640
// Parse device name (card0, card1, etc.) to get index
626641
const deviceName = fields[0]?.trim() || '';
627642
// Extract numeric part from device name (e.g., "card0" -> 0)
628643
const deviceMatch = deviceName.match(/\d+/);
629644
const index = deviceMatch ? parseInt(deviceMatch[0]) : idx;
630645

631-
// Extract fields based on format
632-
// New format: Temperature at field 6, mclk at field 7, sclk at field 9, Power at field 13, Usage at field 14, Memory Total at field 17, Memory Used at field 18
633-
// Old format: Temperature at field 2, mclk at field 3, sclk at field 5, Power at field 9, Usage at field 10, Memory Total at field 12, Memory Used at field 13
634-
const tempFieldIdx = isNewFormat ? 6 : 2;
635-
const mclkFieldIdx = isNewFormat ? 7 : 3;
636-
const sclkFieldIdx = isNewFormat ? 9 : 5;
637-
const powerFieldIdx = isNewFormat ? 13 : 9;
638-
const usageFieldIdx = isNewFormat ? 14 : 10;
639-
const memTotalFieldIdx = isNewFormat ? 17 : 12;
640-
const memUsedFieldIdx = isNewFormat ? 18 : 13;
641-
const cardSkuFieldIdx = isNewFormat ? 22 : 17;
642-
const cardModelFieldIdx = isNewFormat ? 20 : 15;
643-
const cardNameFieldIdx = isNewFormat ? 1 : -1; // Device Name in new format
644-
645646
const tempStr = fields[tempFieldIdx]?.trim() || '';
646647
// Parse temperature - rocm-smi provides temperature in Celsius
647648
let temperature = 0;
@@ -653,8 +654,7 @@ async function getRocmGpuStats(isWindows: boolean) {
653654
}
654655
}
655656

656-
// GPU use (%) - field index depends on format
657-
const gpuUtilStr = fields[usageFieldIdx]?.trim() || '0';
657+
const gpuUtilStr = usageFieldIdx >= 0 ? (fields[usageFieldIdx]?.trim() || '0') : '0';
658658
let gpuUtil = 0;
659659
if (gpuUtilStr && gpuUtilStr !== 'N/A' && !isNaN(parseFloat(gpuUtilStr))) {
660660
const parsed = parseFloat(gpuUtilStr);
@@ -666,11 +666,8 @@ async function getRocmGpuStats(isWindows: boolean) {
666666
// rocm-smi GPU use is already a percentage, but validate and clamp to 0-100
667667
gpuUtil = Math.max(0, Math.min(100, gpuUtil));
668668

669-
// Memory values from rocm-smi are in bytes, but check if they're valid
670-
// Field indices depend on format
671-
672-
const memoryTotalStr = fields[memTotalFieldIdx]?.trim() || '0';
673-
const memoryUsedStr = fields[memUsedFieldIdx]?.trim() || '0';
669+
const memoryTotalStr = memTotalFieldIdx >= 0 ? (fields[memTotalFieldIdx]?.trim() || '0') : '0';
670+
const memoryUsedStr = memUsedFieldIdx >= 0 ? (fields[memUsedFieldIdx]?.trim() || '0') : '0';
674671
let memoryTotal = parseFloat(memoryTotalStr) || 0;
675672
let memoryUsed = parseFloat(memoryUsedStr) || 0;
676673

@@ -680,8 +677,7 @@ async function getRocmGpuStats(isWindows: boolean) {
680677
if (memoryUsed > memoryTotal) memoryUsed = memoryTotal; // Clamp used to total
681678

682679
const memoryFree = Math.max(0, memoryTotal - memoryUsed);
683-
// Power draw - field index depends on format
684-
const powerDrawStr = fields[powerFieldIdx]?.trim() || '';
680+
const powerDrawStr = powerFieldIdx >= 0 ? (fields[powerFieldIdx]?.trim() || '') : '';
685681
// Parse power draw, handle cases where it might be in different formats
686682
let powerDraw = 0;
687683
// Check if the field looks like a clock value (contains "Mhz" or "MHz") and skip it
@@ -699,11 +695,8 @@ async function getRocmGpuStats(isWindows: boolean) {
699695
}
700696
}
701697

702-
// Parse clock speeds (format: "(1000Mhz)" -> 1000)
703-
// mclk = memory clock, sclk = graphics/core clock
704-
// Field indices depend on format
705-
const mclkStr = fields[mclkFieldIdx]?.trim() || '(0Mhz)';
706-
const sclkStr = fields[sclkFieldIdx]?.trim() || '(0Mhz)';
698+
const mclkStr = mclkFieldIdx >= 0 ? (fields[mclkFieldIdx]?.trim() || '(0Mhz)') : '(0Mhz)';
699+
const sclkStr = sclkFieldIdx >= 0 ? (fields[sclkFieldIdx]?.trim() || '(0Mhz)') : '(0Mhz)';
707700

708701
// Extract numeric value from clock strings like "(1000Mhz)" or "1000Mhz"
709702
// Handle both formats: "(1472Mhz)" and raw numbers
@@ -732,13 +725,11 @@ async function getRocmGpuStats(isWindows: boolean) {
732725
clockMemory = 0;
733726
}
734727

735-
// Get GPU name from Card SKU (most descriptive), then Card model, then Device Name (new format), then fallback
736-
// Field indices depend on format
737-
const cardSku = fields.length > cardSkuFieldIdx ? (fields[cardSkuFieldIdx]?.trim() || '') : '';
738-
const cardModel = fields.length > cardModelFieldIdx ? (fields[cardModelFieldIdx]?.trim() || '') : '';
739-
const deviceNameField = cardNameFieldIdx >= 0 && fields.length > cardNameFieldIdx ? (fields[cardNameFieldIdx]?.trim() || '') : '';
740-
const cardVendor = fields.length > 16 ? (fields[16]?.trim() || '') : '';
741-
const gpuId = fields[1]?.trim() || '';
728+
const cardSku = cardSkuFieldIdx >= 0 ? (fields[cardSkuFieldIdx]?.trim() || '') : '';
729+
const cardModel = cardModelFieldIdx >= 0 ? (fields[cardModelFieldIdx]?.trim() || '') : '';
730+
const deviceNameField = cardNameFieldIdx >= 0 ? (fields[cardNameFieldIdx]?.trim() || '') : '';
731+
const cardVendor = cardVendorFieldIdx >= 0 ? (fields[cardVendorFieldIdx]?.trim() || '') : '';
732+
const gpuId = deviceIdFieldIdx >= 0 ? (fields[deviceIdFieldIdx]?.trim() || '') : '';
742733

743734
// Use Card SKU if available and not a hex ID or numeric ID, otherwise prefer Card model, then fallback
744735
let name = '';
@@ -772,21 +763,21 @@ async function getRocmGpuStats(isWindows: boolean) {
772763
name = `AMD GPU ${index}`;
773764
}
774765

775-
// Convert memory from bytes to MB (rocm-smi reports in bytes)
776-
// Check if values are already in MB/GB by checking magnitude
766+
// Convert memory to MB (rocm-smi reports in bytes, but magnitude varies)
777767
let memoryTotalMB = 0;
778768
let memoryUsedMB = 0;
779769
let memoryFreeMB = 0;
780770

781771
if (memoryTotal > 0) {
782-
// If value is very large (> 1TB), assume bytes and convert to MB
783-
// If value is reasonable (< 1000), assume already in GB and convert to MB
784-
if (memoryTotal > 1024 * 1024 * 1024) {
772+
// Values > 1M are clearly bytes (e.g., 536870912 = 512MB)
773+
// Values 10-1M are likely MB already
774+
// Values < 10 are likely GB
775+
if (memoryTotal > 1000000) {
785776
// Bytes - convert to MB
786777
memoryTotalMB = Math.round(memoryTotal / (1024 * 1024));
787778
memoryUsedMB = Math.round(memoryUsed / (1024 * 1024));
788779
memoryFreeMB = Math.round(memoryFree / (1024 * 1024));
789-
} else if (memoryTotal > 1000) {
780+
} else if (memoryTotal >= 10) {
790781
// Already in MB
791782
memoryTotalMB = Math.round(memoryTotal);
792783
memoryUsedMB = Math.round(memoryUsed);

0 commit comments

Comments
 (0)