Skip to content

Commit 8883aa9

Browse files
ssongliussongliu
committed
feat: Compatible with XPU monitoring
1 parent 6b061fa commit 8883aa9

File tree

9 files changed

+107
-52
lines changed

9 files changed

+107
-52
lines changed

‎agent/app/dto/monitor.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ type MonitorGPUSearch struct {
4444
EndTime time.Time `json:"endTime"`
4545
}
4646
type MonitorGPUData struct {
47+
GPUType string `json:"gpuType"`
4748
ProductNames []string `json:"productNames"`
4849
Date []time.Time `json:"date"`
4950
GPUValue []float64 `json:"gpuValue"`
@@ -58,8 +59,8 @@ type GPUPowerUsageHelper struct {
5859
Percent float64 `json:"percent"`
5960
}
6061
type GPUMemoryUsageHelper struct {
61-
Total int `json:"total"`
62-
Used int `json:"used"`
62+
Total float64 `json:"total"`
63+
Used float64 `json:"used"`
6364
Percent float64 `json:"percent"`
6465

6566
GPUProcesses []GPUProcess `json:"gpuProcesses"`

‎agent/app/model/monitor.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@ type MonitorGPU struct {
3939
Temperature int `json:"temperature"`
4040
PowerDraw float64 `json:"powerDraw"`
4141
MaxPowerLimit float64 `json:"maxPowerLimit"`
42-
MemUsed int `json:"memUsed"`
43-
MemTotal int `json:"memTotal"`
42+
MemUsed float64 `json:"memUsed"`
43+
MemTotal float64 `json:"memTotal"`
4444
FanSpeed int `json:"fanSpeed"`
4545
Processes string `json:"processes"`
4646
}

‎agent/app/service/monitor.go‎

Lines changed: 43 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -130,17 +130,21 @@ func (m *MonitorService) LoadGPUMonitorData(req dto.MonitorGPUSearch) (dto.Monit
130130
}
131131
if len(req.ProductName) == 0 {
132132
if gpuExist {
133+
data.GPUType = "gpu"
133134
gpuInfo, err := gpuclient.LoadGpuInfo()
134135
if err != nil || len(gpuInfo.GPUs) == 0 {
136+
global.LOG.Error("Load GPU info failed or no GPU found, err: ", err)
135137
return data, buserr.New("ErrRecordNotFound")
136138
}
137139
req.ProductName = gpuInfo.GPUs[0].ProductName
138140
for _, item := range gpuInfo.GPUs {
139141
data.ProductNames = append(data.ProductNames, item.ProductName)
140142
}
141143
} else {
144+
data.GPUType = "xpu"
142145
xpuInfo, err := xpuClient.LoadGpuInfo()
143146
if err != nil || len(xpuInfo.Xpu) == 0 {
147+
global.LOG.Error("Load XPU info failed or no XPU found, err: ", err)
144148
return data, buserr.New("ErrRecordNotFound")
145149
}
146150
req.ProductName = xpuInfo.Xpu[0].Basic.DeviceName
@@ -158,15 +162,18 @@ func (m *MonitorService) LoadGPUMonitorData(req dto.MonitorGPUSearch) (dto.Monit
158162
data.Date = append(data.Date, gpu.CreatedAt)
159163
data.GPUValue = append(data.GPUValue, gpu.GPUUtil)
160164
data.TemperatureValue = append(data.TemperatureValue, gpu.Temperature)
161-
data.PowerValue = append(data.PowerValue, dto.GPUPowerUsageHelper{
162-
Total: gpu.MaxPowerLimit,
163-
Used: gpu.PowerDraw,
164-
Percent: gpu.PowerDraw / gpu.MaxPowerLimit * 100,
165-
})
165+
powerItem := dto.GPUPowerUsageHelper{
166+
Total: gpu.MaxPowerLimit,
167+
Used: gpu.PowerDraw,
168+
}
169+
if powerItem.Total != 0 {
170+
powerItem.Percent = powerItem.Used / powerItem.Total
171+
}
172+
data.PowerValue = append(data.PowerValue, powerItem)
166173
memItem := dto.GPUMemoryUsageHelper{
167174
Total: gpu.MemTotal,
168175
Used: gpu.MemUsed,
169-
Percent: float64(gpu.MemUsed) / float64(gpu.MemTotal) * 100,
176+
Percent: gpu.MemUsed / gpu.MemTotal * 100,
170177
}
171178
var process []dto.GPUProcess
172179
if err := json.Unmarshal([]byte(gpu.Processes), &process); err == nil {
@@ -563,14 +570,13 @@ func saveGPUDataToDB() {
563570
var list []model.MonitorGPU
564571
for _, gpuItem := range gpuInfo.GPUs {
565572
item := model.MonitorGPU{
566-
ProductName: gpuItem.ProductName,
567-
GPUUtil: loadGPUInfoFloat(gpuItem.GPUUtil),
568-
Temperature: loadGPUInfoInt(gpuItem.Temperature),
569-
PowerDraw: loadGPUInfoFloat(gpuItem.PowerDraw),
570-
MaxPowerLimit: loadGPUInfoFloat(gpuItem.MaxPowerLimit),
571-
MemUsed: loadGPUInfoInt(gpuItem.MemUsed),
572-
MemTotal: loadGPUInfoInt(gpuItem.MemTotal),
573-
FanSpeed: loadGPUInfoInt(gpuItem.FanSpeed),
573+
ProductName: gpuItem.ProductName,
574+
GPUUtil: loadGPUInfoFloat(gpuItem.GPUUtil),
575+
Temperature: loadGPUInfoInt(gpuItem.Temperature),
576+
PowerDraw: loadGPUInfoFloat(gpuItem.PowerDraw),
577+
MemUsed: loadGPUInfoFloat(gpuItem.MemUsed),
578+
MemTotal: loadGPUInfoFloat(gpuItem.MemTotal),
579+
FanSpeed: loadGPUInfoInt(gpuItem.FanSpeed),
574580
}
575581
process, _ := json.Marshal(gpuItem.Processes)
576582
if len(process) != 0 {
@@ -595,25 +601,28 @@ func saveXPUDataToDB() {
595601
var list []model.MonitorGPU
596602
for _, xpuItem := range xpuInfo.Xpu {
597603
item := model.MonitorGPU{
598-
ProductName: xpuItem.Basic.DeviceName,
599-
GPUUtil: loadGPUInfoFloat(xpuItem.Stats.MemoryUtil),
600-
Temperature: loadGPUInfoInt(xpuItem.Stats.Temperature),
601-
PowerDraw: loadGPUInfoFloat(xpuItem.Stats.Power),
602-
MemUsed: loadGPUInfoInt(xpuItem.Stats.MemoryUsed),
603-
MemTotal: loadGPUInfoInt(xpuItem.Basic.Memory),
604-
}
605-
var processItem []dto.GPUProcess
606-
for _, ps := range xpuItem.Processes {
607-
processItem = append(processItem, dto.GPUProcess{
608-
Pid: fmt.Sprintf("%v", ps.PID),
609-
Type: ps.SHR,
610-
ProcessName: ps.Command,
611-
UsedMemory: ps.Memory,
612-
})
613-
}
614-
process, _ := json.Marshal(processItem)
615-
if len(process) != 0 {
616-
item.Processes = string(process)
604+
ProductName: xpuItem.Basic.DeviceName,
605+
GPUUtil: loadGPUInfoFloat(xpuItem.Stats.MemoryUtil),
606+
Temperature: loadGPUInfoInt(xpuItem.Stats.Temperature),
607+
PowerDraw: loadGPUInfoFloat(xpuItem.Stats.Power),
608+
MaxPowerLimit: float64(xpuItem.Config.PowerLimit),
609+
MemUsed: loadGPUInfoFloat(xpuItem.Stats.MemoryUsed),
610+
MemTotal: loadGPUInfoFloat(xpuItem.Basic.Memory),
611+
}
612+
if len(xpuItem.Processes) != 0 {
613+
var processItem []dto.GPUProcess
614+
for _, ps := range xpuItem.Processes {
615+
processItem = append(processItem, dto.GPUProcess{
616+
Pid: fmt.Sprintf("%v", ps.PID),
617+
Type: ps.SHR,
618+
ProcessName: ps.Command,
619+
UsedMemory: ps.Memory,
620+
})
621+
}
622+
process, _ := json.Marshal(processItem)
623+
if len(process) != 0 {
624+
item.Processes = string(process)
625+
}
617626
}
618627
list = append(list, item)
619628
}
@@ -632,6 +641,7 @@ func loadGPUInfoInt(val string) int {
632641
}
633642
func loadGPUInfoFloat(val string) float64 {
634643
valItem := strings.ReplaceAll(val, "W", "")
644+
valItem = strings.ReplaceAll(valItem, "MB", "")
635645
valItem = strings.ReplaceAll(valItem, "%", "")
636646
valItem = strings.TrimSpace(valItem)
637647
data, _ := strconv.ParseFloat(valItem, 64)

‎agent/init/migration/migrations/init.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -721,7 +721,7 @@ var UpdateDatabase = &gormigrate.Migration{
721721
}
722722

723723
var AddGPUMonitor = &gormigrate.Migration{
724-
ID: "20251119-add-gpu-monitor",
724+
ID: "20251127-add-gpu-monitor",
725725
Migrate: func(tx *gorm.DB) error {
726726
return global.GPUMonitorDB.AutoMigrate(&model.MonitorGPU{})
727727
},

‎agent/utils/ai_tools/xpu/xpu_info.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ type XpuInfo struct {
1010
type Xpu struct {
1111
Basic Basic `json:"basic"`
1212
Stats Stats `json:"stats"`
13+
Config Config `json:"config"`
1314
Processes []Process `json:"processes"`
1415
}
1516

@@ -23,6 +24,11 @@ type Basic struct {
2324
PciBdfAddress string `json:"pciBdfAddress"`
2425
}
2526

27+
type Config struct {
28+
PowerLimit int `json:"power_limit"`
29+
PowerValidRange string `json:"power_vaild_range"`
30+
}
31+
2632
type Stats struct {
2733
Power string `json:"power"`
2834
Frequency string `json:"frequency"`

‎frontend/src/api/interface/host.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,7 @@ export namespace Host {
167167
endTime: Date;
168168
}
169169
export interface MonitorGPUData {
170+
gpuType: string;
170171
productNames: Array<string>;
171172
date: Array<Date>;
172173
gpuValue: Array<number>;

‎frontend/src/lang/modules/zh.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -656,7 +656,7 @@ const message = {
656656
gpu: {
657657
gpu: 'GPU 监控',
658658
base: '基础信息',
659-
gpuHelper: '当前系统未检测到 NVIDIA-SMI或者XPU-SMI 指令，请检查后重试！',
659+
gpuHelper: '当前系统未检测到 NVIDIA-SMI 或者 XPU-SMI 指令，请检查后重试！',
660660
driverVersion: '驱动版本',
661661
cudaVersion: 'CUDA 版本',
662662
process: '进程信息',

‎frontend/src/views/ai/gpu/index.vue‎

Lines changed: 49 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,9 @@
3030
</div>
3131
</el-card>
3232
</div>
33-
<el-row :gutter="7" class="card-interval" v-if="options.length !== 0">
34-
<el-col :span="24">
35-
<el-card style="overflow: inherit">
33+
<el-row :gutter="7" v-if="options.length !== 0">
34+
<el-col v-bind="gpuType === 'gpu' ? fullWidthProps : halfWidthProps">
35+
<el-card class="card-interval" style="overflow: inherit">
3636
<template #header>
3737
<div :class="mobile ? 'flx-wrap' : 'flex justify-between'">
3838
<span class="title">{{ $t('monitor.gpuUtil') }}</span>
@@ -51,7 +51,7 @@
5151
</el-card>
5252
</el-col>
5353
<el-col :xs="24" :sm="24" :md="12" :lg="12" :xl="12">
54-
<el-card style="overflow: inherit">
54+
<el-card class="card-interval" style="overflow: inherit">
5555
<template #header>
5656
<div :class="mobile ? 'flx-wrap' : 'flex justify-between'">
5757
<span class="title">{{ $t('monitor.memoryUsage') }}</span>
@@ -70,7 +70,7 @@
7070
</el-card>
7171
</el-col>
7272
<el-col :xs="24" :sm="24" :md="12" :lg="12" :xl="12">
73-
<el-card style="overflow: inherit">
73+
<el-card class="card-interval" style="overflow: inherit">
7474
<template #header>
7575
<div :class="mobile ? 'flx-wrap' : 'flex justify-between'">
7676
<span class="title">{{ $t('monitor.powerUsage') }}</span>
@@ -89,7 +89,7 @@
8989
</el-card>
9090
</el-col>
9191
<el-col :xs="24" :sm="24" :md="12" :lg="12" :xl="12">
92-
<el-card style="overflow: inherit">
92+
<el-card class="card-interval" style="overflow: inherit">
9393
<template #header>
9494
<div>
9595
{{ $t('monitor.temperature') }}
@@ -110,8 +110,8 @@
110110
</div>
111111
</el-card>
112112
</el-col>
113-
<el-col :xs="24" :sm="24" :md="12" :lg="12" :xl="12">
114-
<el-card style="overflow: inherit">
113+
<el-col :xs="24" :sm="24" :md="12" :lg="12" :xl="12" v-if="gpuType === 'gpu'">
114+
<el-card class="card-interval" style="overflow: inherit">
115115
<template #header>
116116
<div :class="mobile ? 'flx-wrap' : 'flex justify-between'">
117117
<span class="title">{{ $t('monitor.fanSpeed') }}</span>
@@ -161,8 +161,12 @@ const mobile = computed(() => {
161161
return globalStore.isMobile();
162162
});
163163
164+
const fullWidthProps = { span: 24 };
165+
const halfWidthProps = { xs: 24, sm: 24, md: 12, lg: 12, xl: 12 };
166+
164167
const loading = ref(false);
165168
const options = ref([]);
169+
const gpuType = ref('gpu');
166170
const timeRangeGlobal = ref<[Date, Date]>([new Date(new Date().setHours(0, 0, 0, 0)), new Date()]);
167171
const chartsOption = ref({
168172
loadPowerChart: null,
@@ -189,14 +193,19 @@ const search = async () => {
189193
.then((res) => {
190194
loading.value = false;
191195
options.value = res.data.productNames || [];
196+
gpuType.value = res.data.gpuType || 'gpu';
192197
searchInfo.productName = searchInfo.productName || (options.value.length > 0 ? options.value[0] : '');
193198
let baseDate = res.data.date.length === 0 ? loadEmptyDate(timeRangeGlobal.value) : res.data.date;
194199
let date = baseDate.map(function (item: any) {
195200
return dateFormatWithoutYear(item);
196201
});
197202
initCPUCharts(date, res.data.gpuValue);
198203
initMemoryCharts(date, res.data.memoryValue);
199-
initPowerCharts(date, res.data.powerValue);
204+
if (gpuType.value === 'gpu') {
205+
initPowerCharts(date, res.data.powerValue);
206+
} else {
207+
initXpuPowerCharts(date, res.data.powerValue);
208+
}
200209
initSpeedCharts(date, res.data.speedValue);
201210
initTemperatureCharts(date, res.data.temperatureValue);
202211
})
@@ -270,6 +279,33 @@ function initPowerCharts(baseDate: any, items: any) {
270279
formatStr: '%',
271280
};
272281
}
282+
283+
function initXpuPowerCharts(baseDate: any, items: any) {
284+
let list = items.map(function (item: any) {
285+
return { value: Number(item.used.toFixed(2)), data: item };
286+
});
287+
list = list.length === 0 ? loadEmptyData2() : list;
288+
chartsOption.value['loadPowerChart'] = {
289+
xData: baseDate,
290+
yData: [
291+
{
292+
name: i18n.global.t('monitor.powerUsage'),
293+
data: list,
294+
},
295+
],
296+
tooltip: {
297+
trigger: 'axis',
298+
formatter: function (list: any) {
299+
let res = loadDate(list[0].name);
300+
for (const item of list) {
301+
res += loadSeries(item, item.data.value ? item.data.value : item.data, 'W');
302+
}
303+
return res;
304+
},
305+
},
306+
formatStr: 'W',
307+
};
308+
}
273309
function initTemperatureCharts(baseDate: any, items: any) {
274310
let temperatures = items.map(function (item: any) {
275311
return Number(item);
@@ -334,14 +370,15 @@ function withMemoryProcess(list: any) {
334370
if (!process) {
335371
return res;
336372
}
373+
let title = gpuType.value === 'gpu' ? i18n.global.t('aiTools.gpu.type') : i18n.global.t('aiTools.gpu.shr');
337374
res += `
338375
<div style="margin-top: 10px; border-bottom: 1px dashed black;"></div>
339376
<table style="border-collapse: collapse; margin-top: 20px; font-size: 12px;">
340377
<thead>
341378
<tr>
342379
<th style="padding: 6px 8px;">PID</th>
343-
<th style="padding: 6px 8px;">${i18n.global.t('aiTools.gpu.type')}</th>
344380
<th style="padding: 6px 8px;">${i18n.global.t('aiTools.gpu.processName')}</th>
381+
<th style="padding: 6px 8px;">${title}</th>
345382
<th style="padding: 6px 8px;">${i18n.global.t('aiTools.gpu.processMemoryUsage')}</th>
346383
</tr>
347384
</thead>
@@ -354,10 +391,10 @@ function withMemoryProcess(list: any) {
354391
${row.pid}
355392
</td>
356393
<td style="padding: 6px 8px; text-align: center;">
357-
${loadProcessType(row.type)}
394+
${row.processName}
358395
</td>
359396
<td style="padding: 6px 8px; text-align: center;">
360-
${row.processName}
397+
${loadProcessType(row.type)}
361398
</td>
362399
<td style="padding: 6px 8px; text-align: center;">
363400
${row.usedMemory}

‎frontend/src/views/home/status/index.vue‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -315,7 +315,7 @@
315315
<span class="input-help" v-else>{{ item.deviceName }}</span>
316316
</el-col>
317317
</template>
318-
<el-col :xs="6" :sm="6" :md="6" :lg="6" :xl="6" align="center" v-if="totalCount > 5">
318+
<el-col :xs="6" :sm="6" :md="3" :lg="3" :xl="3" align="center" v-if="totalCount > 5">
319319
<el-button v-if="!showMore" link type="primary" @click="changeShowMore(true)" class="buttonClass">
320320
{{ $t('tabs.more') }}
321321
<el-icon><Bottom /></el-icon>

0 commit comments

Comments
 (0)