[GPUHEALTH-1522] feat: add DCGM ECC and NVLink RX metrics

Signed-off-by: Jingxiang Zhang <jingzhang@nvidia.com>

Author: jingxiang-z

third_party/fleet-intelligence-sdk/components/accelerator/nvidia/dcgm/mem/component.go

@@ -418,6 +418,26 @@ func (c *component) Check() components.CheckResult {
 						"uuid": deviceData.UUID,
 						"gpu":  fmt.Sprintf("%d", deviceData.DeviceID),
 					}).Set(float64(fieldValue.Int64()))
+				case dcgm.DCGM_FI_DEV_ECC_SBE_VOL_DEV:
+					metricDCGMFIDevECCSBEVolDev.With(prometheus.Labels{
+						"uuid": deviceData.UUID,
+						"gpu":  fmt.Sprintf("%d", deviceData.DeviceID),
+					}).Set(float64(fieldValue.Int64()))
+				case dcgm.DCGM_FI_DEV_ECC_DBE_VOL_DEV:
+					metricDCGMFIDevECCDBEVolDev.With(prometheus.Labels{
+						"uuid": deviceData.UUID,
+						"gpu":  fmt.Sprintf("%d", deviceData.DeviceID),
+					}).Set(float64(fieldValue.Int64()))
+				case dcgm.DCGM_FI_DEV_ECC_SBE_AGG_DEV:
+					metricDCGMFIDevECCSBEAggDev.With(prometheus.Labels{
+						"uuid": deviceData.UUID,
+						"gpu":  fmt.Sprintf("%d", deviceData.DeviceID),
+					}).Set(float64(fieldValue.Int64()))
+				case dcgm.DCGM_FI_DEV_ECC_DBE_AGG_DEV:
+					metricDCGMFIDevECCDBEAggDev.With(prometheus.Labels{
+						"uuid": deviceData.UUID,
+						"gpu":  fmt.Sprintf("%d", deviceData.DeviceID),
+					}).Set(float64(fieldValue.Int64()))
 				case dcgm.DCGM_FI_DEV_RETIRED_PENDING:
 					metricDCGMFIDevRetiredPending.With(prometheus.Labels{"uuid": deviceData.UUID, "gpu": fmt.Sprintf("%d", deviceData.DeviceID)}).Set(float64(fieldValue.Int64()))
 				case dcgm.DCGM_FI_DEV_RETIRED_DBE:

third_party/fleet-intelligence-sdk/components/accelerator/nvidia/dcgm/mem/metrics.go

@@ -37,6 +37,10 @@ var memFields = []dcgm.Short{
 	dcgm.DCGM_FI_DEV_ECC_DBE_VOL_TOTAL,              // Total double bit volatile ECC errors
 	dcgm.DCGM_FI_DEV_ECC_SBE_AGG_TOTAL,              // Total single bit aggregate (persistent) ECC errors
 	dcgm.DCGM_FI_DEV_ECC_DBE_AGG_TOTAL,              // Total double bit aggregate (persistent) ECC errors
+	dcgm.DCGM_FI_DEV_ECC_SBE_VOL_DEV,                // Single bit volatile ECC errors detected in device memory
+	dcgm.DCGM_FI_DEV_ECC_DBE_VOL_DEV,                // Double bit volatile ECC errors detected in device memory
+	dcgm.DCGM_FI_DEV_ECC_SBE_AGG_DEV,                // Aggregate single bit ECC errors detected in device memory
+	dcgm.DCGM_FI_DEV_ECC_DBE_AGG_DEV,                // Aggregate double bit ECC errors detected in device memory
 	dcgm.DCGM_FI_DEV_RETIRED_PENDING,                // Whether pages are pending retirement
 	dcgm.DCGM_FI_DEV_RETIRED_DBE,                    // Retired DBE pages
 	dcgm.DCGM_FI_DEV_RETIRED_SBE,                    // Retired SBE pages
@@ -172,6 +176,38 @@ var (
 		[]string{pkgmetrics.MetricComponentLabelKey, "uuid", "gpu"},
 	).MustCurryWith(componentLabel)
 
+	metricDCGMFIDevECCSBEVolDev = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "dcgm_fi_dev_ecc_sbe_vol_dev",
+			Help: "Device memory single bit volatile ECC errors.",
+		},
+		[]string{pkgmetrics.MetricComponentLabelKey, "uuid", "gpu"},
+	).MustCurryWith(componentLabel)
+
+	metricDCGMFIDevECCDBEVolDev = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "dcgm_fi_dev_ecc_dbe_vol_dev",
+			Help: "Device memory double bit volatile ECC errors.",
+		},
+		[]string{pkgmetrics.MetricComponentLabelKey, "uuid", "gpu"},
+	).MustCurryWith(componentLabel)
+
+	metricDCGMFIDevECCSBEAggDev = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "dcgm_fi_dev_ecc_sbe_agg_dev",
+			Help: "Device memory single bit aggregate (persistent) ECC errors. Note: monotonically increasing.",
+		},
+		[]string{pkgmetrics.MetricComponentLabelKey, "uuid", "gpu"},
+	).MustCurryWith(componentLabel)
+
+	metricDCGMFIDevECCDBEAggDev = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "dcgm_fi_dev_ecc_dbe_agg_dev",
+			Help: "Device memory double bit aggregate (persistent) ECC errors. Note: monotonically increasing.",
+		},
+		[]string{pkgmetrics.MetricComponentLabelKey, "uuid", "gpu"},
+	).MustCurryWith(componentLabel)
+
 	metricDCGMFIDevRetiredPending = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{Name: "dcgm_fi_dev_retired_pending", Help: "Number of pages pending retirement"},
 		[]string{pkgmetrics.MetricComponentLabelKey, "uuid", "gpu"},
@@ -227,6 +263,10 @@ func init() {
 		metricDCGMFIDevECCDBEVolTotal,
 		metricDCGMFIDevECCSBEAggTotal,
 		metricDCGMFIDevECCDBAggTotal,
+		metricDCGMFIDevECCSBEVolDev,
+		metricDCGMFIDevECCDBEVolDev,
+		metricDCGMFIDevECCSBEAggDev,
+		metricDCGMFIDevECCDBEAggDev,
 		metricDCGMFIDevRetiredPending,
 		metricDCGMFIDevRetiredDBE,
 		metricDCGMFIDevRetiredSBE,

third_party/fleet-intelligence-sdk/components/accelerator/nvidia/dcgm/mem/metrics_test.go

@@ -0,0 +1,42 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mem
+
+import (
+	"testing"
+
+	dcgm "github.com/NVIDIA/go-dcgm/pkg/dcgm"
+)
+
+func TestMemFieldsIncludeExpectedECCDeviceCounters(t *testing.T) {
+	fieldSet := make(map[dcgm.Short]struct{}, len(memFields))
+	for _, field := range memFields {
+		fieldSet[field] = struct{}{}
+	}
+
+	required := []dcgm.Short{
+		dcgm.DCGM_FI_DEV_ECC_DBE_AGG_DEV,
+		dcgm.DCGM_FI_DEV_ECC_DBE_VOL_DEV,
+		dcgm.DCGM_FI_DEV_ECC_SBE_AGG_DEV,
+		dcgm.DCGM_FI_DEV_ECC_SBE_VOL_DEV,
+	}
+
+	for _, field := range required {
+		if _, ok := fieldSet[field]; !ok {
+			t.Errorf("missing expected mem field: %d", field)
+		}
+	}
+}

third_party/fleet-intelligence-sdk/components/accelerator/nvidia/dcgm/nvlink/component.go

@@ -375,6 +375,8 @@ func (c *component) Check() components.CheckResult {
 					metricDCGMFIDevC2CLinkErrorReplay.With(prometheus.Labels{"uuid": deviceData.UUID, "gpu": fmt.Sprintf("%d", deviceData.DeviceID)}).Set(float64(fieldValue.Int64()))
 				case dcgm.DCGM_FI_DEV_NVLINK_COUNT_RX_GENERAL_ERRORS:
 					metricDCGMFIDevNvlinkCountRxGeneralErrors.With(prometheus.Labels{"uuid": deviceData.UUID, "gpu": fmt.Sprintf("%d", deviceData.DeviceID)}).Set(float64(fieldValue.Int64()))
+				case dcgm.DCGM_FI_DEV_NVLINK_COUNT_RX_ERRORS:
+					metricDCGMFIDevNvlinkCountRxErrors.With(prometheus.Labels{"uuid": deviceData.UUID, "gpu": fmt.Sprintf("%d", deviceData.DeviceID)}).Set(float64(fieldValue.Int64()))
 				case dcgm.DCGM_FI_DEV_NVLINK_COUNT_RX_MALFORMED_PACKET_ERRORS:
 					metricDCGMFIDevNvlinkCountRxMalformedPacketErrors.With(prometheus.Labels{"uuid": deviceData.UUID, "gpu": fmt.Sprintf("%d", deviceData.DeviceID)}).Set(float64(fieldValue.Int64()))
 				case dcgm.DCGM_FI_DEV_NVLINK_COUNT_RX_REMOTE_ERRORS:

third_party/fleet-intelligence-sdk/components/accelerator/nvidia/dcgm/nvlink/metrics.go

@@ -34,6 +34,7 @@ var nvlinkFields = []dcgm.Short{
 	dcgm.DCGM_FI_DEV_FABRIC_MANAGER_STATUS,
 	dcgm.DCGM_FI_DEV_C2C_LINK_ERROR_REPLAY,
 	dcgm.DCGM_FI_DEV_NVLINK_COUNT_RX_GENERAL_ERRORS,
+	dcgm.DCGM_FI_DEV_NVLINK_COUNT_RX_ERRORS,
 	dcgm.DCGM_FI_DEV_NVLINK_COUNT_RX_MALFORMED_PACKET_ERRORS,
 	dcgm.DCGM_FI_DEV_NVLINK_COUNT_RX_REMOTE_ERRORS,
 	dcgm.DCGM_FI_DEV_NVLINK_COUNT_RX_SYMBOL_ERRORS,
@@ -122,6 +123,10 @@ var (
 		prometheus.GaugeOpts{Name: "dcgm_fi_dev_nvlink_count_rx_general_errors", Help: "Total number of packets Rx with header mismatch"},
 		[]string{pkgmetrics.MetricComponentLabelKey, "uuid", "gpu"},
 	).MustCurryWith(componentLabel)
+	metricDCGMFIDevNvlinkCountRxErrors = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{Name: "dcgm_fi_dev_nvlink_count_rx_errors", Help: "Total number of packets with errors Rx on a link"},
+		[]string{pkgmetrics.MetricComponentLabelKey, "uuid", "gpu"},
+	).MustCurryWith(componentLabel)
 	metricDCGMFIDevNvlinkCountRxMalformedPacketErrors = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{Name: "dcgm_fi_dev_nvlink_count_rx_malformed_packet_errors", Help: "Number of packets Rx on a link where packets are malformed"},
 		[]string{pkgmetrics.MetricComponentLabelKey, "uuid", "gpu"},
@@ -171,6 +176,7 @@ func init() {
 		metricDCGMFIDevFabricManagerStatus,
 		metricDCGMFIDevC2CLinkErrorReplay,
 		metricDCGMFIDevNvlinkCountRxGeneralErrors,
+		metricDCGMFIDevNvlinkCountRxErrors,
 		metricDCGMFIDevNvlinkCountRxMalformedPacketErrors,
 		metricDCGMFIDevNvlinkCountRxRemoteErrors,
 		metricDCGMFIDevNvlinkCountRxSymbolErrors,

third_party/fleet-intelligence-sdk/components/accelerator/nvidia/dcgm/nvlink/metrics_test.go

@@ -37,6 +37,7 @@ func TestNVLinkFieldsIncludeExpectedCounters(t *testing.T) {
 		dcgm.DCGM_FI_DEV_FABRIC_MANAGER_STATUS,
 		dcgm.DCGM_FI_DEV_C2C_LINK_ERROR_REPLAY,
 		dcgm.DCGM_FI_DEV_NVLINK_COUNT_RX_GENERAL_ERRORS,
+		dcgm.DCGM_FI_DEV_NVLINK_COUNT_RX_ERRORS,
 		dcgm.DCGM_FI_DEV_NVLINK_COUNT_RX_MALFORMED_PACKET_ERRORS,
 		dcgm.DCGM_FI_DEV_NVLINK_COUNT_RX_REMOTE_ERRORS,
 		dcgm.DCGM_FI_DEV_NVLINK_COUNT_RX_SYMBOL_ERRORS,