added more analyzer parts

Author: alexandraBara

nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py

@@ -29,7 +29,7 @@
 from nodescraper.interfaces import DataAnalyzer
 from nodescraper.models import TaskResult
 
-from .amdsmidata import AmdSmiDataModel, Fw, Partition, Processes
+from .amdsmidata import AmdSmiDataModel, AmdSmiStatic, Fw, Partition, Processes
 from .analyzer_args import AmdSmiAnalyzerArgs
 
 
@@ -41,6 +41,75 @@ class AmdSmiAnalyzer(DataAnalyzer[AmdSmiDataModel, None]):
     L0_TO_RECOVERY_COUNT_ERROR_THRESHOLD = 3
     L0_TO_RECOVERY_COUNT_WARNING_THRESHOLD = 1
 
+    def check_expected_max_power(
+        self,
+        amdsmi_static_data: list[AmdSmiStatic],
+        expected_max_power: int,
+    ):
+        """Check the max power for all GPUs. If the max power is not as expected, log an error event"""
+        incorrect_max_power_gpus: dict[int, int | str | float] = {}
+        for gpu in amdsmi_static_data:
+            if gpu.limit is None or gpu.limit.max_power is None:
+                self._log_event(
+                    category=EventCategory.PLATFORM,
+                    description=f"GPU: {gpu.gpu} has no max power limit set",
+                    priority=EventPriority.WARNING,
+                    data={"gpu": gpu.gpu},
+                )
+                continue
+            max_power_value = gpu.limit.max_power.value
+            try:
+                max_power_float = float(max_power_value)
+            except ValueError:
+                self._log_event(
+                    category=EventCategory.PLATFORM,
+                    description=f"GPU: {gpu.gpu} has an invalid max power limit",
+                    priority=EventPriority.ERROR,
+                    data={
+                        "gpu": gpu.gpu,
+                        "max_power_value": max_power_value,
+                    },
+                )
+                continue
+            if max_power_float != expected_max_power:
+                incorrect_max_power_gpus[gpu.gpu] = max_power_float
+        if incorrect_max_power_gpus:
+            self._log_event(
+                category=EventCategory.PLATFORM,
+                description="Max power mismatch",
+                priority=EventPriority.ERROR,
+                data={
+                    "gpus": list(incorrect_max_power_gpus.keys()),
+                    "max_power_values": incorrect_max_power_gpus,
+                    "expected_max_power": expected_max_power,
+                },
+            )
+
+    def check_expected_driver_version(
+        self,
+        amdsmi_static_data: list[AmdSmiStatic],
+        expected_driver_version: str,
+    ):
+        bad_driver_gpus = []
+        for gpu in amdsmi_static_data:
+            if gpu.driver.version != expected_driver_version:
+                bad_driver_gpus.append(gpu.gpu)
+        if bad_driver_gpus:
+            self._log_event(
+                category=EventCategory.PLATFORM,
+                description="Driver Version Mismatch",
+                priority=EventPriority.ERROR,
+                data={
+                    "gpus": bad_driver_gpus,
+                    "driver_version": {
+                        gpu.gpu: gpu.driver.version
+                        for gpu in amdsmi_static_data
+                        if gpu.gpu in bad_driver_gpus
+                    },
+                    "expected_driver_version": expected_driver_version,
+                },
+            )
+
     def expected_gpu_processes(
         self, processes_data: list[Processes] | None, max_num_processes: int
     ):
@@ -63,7 +132,7 @@ def expected_gpu_processes(
                 # Skip if there are no processes or the process info is a string which indicates no processes
                 continue
 
-            process_count = len(process.process_list)  # Number of processes for GPU
+            process_count = len(process.process_list)
             if process_count > max_num_processes:
                 gpu_exceeds_num_processes[process.gpu] = process_count
 
@@ -78,6 +147,85 @@ def expected_gpu_processes(
                 console_log=True,
             )
 
+    def static_consistancy_check(self, amdsmi_static_data: list[AmdSmiStatic]):
+        """Check the static data for all GPUs. If the static data is not consistent, log an error event"""
+        consistancy_data: dict[str, set[str] | set[int]] = {
+            "market_name": {gpu.asic.market_name for gpu in amdsmi_static_data},
+            "vendor_id": {gpu.asic.vendor_id for gpu in amdsmi_static_data},
+            "vendor_name": {gpu.asic.vendor_name for gpu in amdsmi_static_data},
+            "subvendor_id": {gpu.asic.subvendor_id for gpu in amdsmi_static_data},
+            "subsystem_id": {gpu.asic.subsystem_id for gpu in amdsmi_static_data},
+            "device_id": {gpu.asic.device_id for gpu in amdsmi_static_data},
+            "rev_id": {gpu.asic.rev_id for gpu in amdsmi_static_data},
+            "num_compute_units": {gpu.asic.num_compute_units for gpu in amdsmi_static_data},
+            "target_graphics_version": {
+                gpu.asic.target_graphics_version for gpu in amdsmi_static_data
+            },
+        }
+        for key, value in consistancy_data.items():
+            if len(value) > 1:
+                self._log_event(
+                    category=EventCategory.PLATFORM,
+                    description=f"{key} is not consistent across all GPUs",
+                    priority=EventPriority.ERROR,
+                    data={
+                        "field": key,
+                        "non_consistent_values": value,
+                    },
+                )
+
+    def check_static_data(
+        self,
+        amdsmi_static_data: list[AmdSmiStatic],
+        vendor_id: str | None,
+        subvendor_id: str | None,
+        device_id: tuple[str | None, str | None],
+        subsystem_id: tuple[str | None, str | None],
+        sku_name: str,
+    ):
+        mismatch_gpus: list[tuple[int, str, str]] = []
+        expected_data: dict[str, str | None] = {
+            "vendor_id": vendor_id,
+            "subvendor_id": subvendor_id,
+            "vendor_name": "Advanced Micro Devices Inc",
+            "market_name": sku_name,
+        }
+        for gpu_data in amdsmi_static_data:
+            for key in expected_data:
+                collected_data: dict[str, str] = {
+                    "vendor_id": gpu_data.asic.vendor_id,
+                    "subvendor_id": gpu_data.asic.subvendor_id,
+                    "vendor_name": gpu_data.asic.vendor_name,
+                    "market_name": sku_name,
+                }
+                if expected_data[key] is not None:
+                    if expected_data[key] not in collected_data[key]:
+                        mismatch_gpus.append((gpu_data.gpu, key, collected_data[key]))
+                        break
+            if device_id[0] is not None and device_id[1] is not None:
+                if (
+                    device_id[0].upper() not in gpu_data.asic.device_id.upper()
+                    and device_id[1].upper() not in gpu_data.asic.device_id.upper()
+                ):
+                    mismatch_gpus.append((gpu_data.gpu, "device_id", gpu_data.asic.device_id))
+            if subsystem_id[0] is not None and subsystem_id[1] is not None:
+                if (
+                    subsystem_id[0].upper() not in gpu_data.asic.subsystem_id.upper()
+                    and subsystem_id[1].upper() not in gpu_data.asic.subsystem_id.upper()
+                ):
+                    mismatch_gpus.append((gpu_data.gpu, "subsystem_id", gpu_data.asic.subsystem_id))
+        if mismatch_gpus:
+            self._log_event(
+                category=EventCategory.PLATFORM,
+                description="amd-smi static data mismatch",
+                priority=EventPriority.ERROR,
+                data={
+                    "gpus": [data[0] for data in mismatch_gpus],
+                    "key": [data[1] for data in mismatch_gpus],
+                    "collected_data": [data[2] for data in mismatch_gpus],
+                },
+            )
+
     def check_pldm_version(
         self,
         amdsmi_fw_data: list[Fw] | None,
@@ -98,9 +246,9 @@ def check_pldm_version(
         for fw_data in amdsmi_fw_data:
             gpu = fw_data.gpu
             for fw_info in fw_data.fw_list:
-                if PLDM_STRING == fw_info.fw_id and expected_pldm_version != fw_info.fw_version:
+                if PLDM_STRING == fw_info.fw_name and expected_pldm_version != fw_info.fw_version:
                     mismatched_gpus.append(gpu)
-                if PLDM_STRING == fw_info.fw_id:
+                if PLDM_STRING == fw_info.fw_name:
                     break
             else:
                 pldm_missing_gpus.append(gpu)
@@ -131,21 +279,32 @@ def check_expected_memory_partition_mode(
             )
             return
         bad_memory_partition_mode_gpus = []
-        for partition_current in partition_data.current_partition:
+        for partition_current in partition_data.memory_partition:
             if (
                 expected_memory_partition_mode is not None
-                and partition_current.memory != expected_memory_partition_mode
-            ) or (
+                and partition_current.partition_type != expected_memory_partition_mode
+            ):
+                bad_memory_partition_mode_gpus.append(
+                    {
+                        "gpu_id": partition_current.gpu_id,
+                        "memory_partition_mode": partition_current.partition_type,
+                    }
+                )
+
+        for partition_current in partition_data.compute_partition:
+            if (
                 expected_compute_partition_mode is not None
-                and partition_current.accelerator_type != expected_compute_partition_mode
+                and partition_current.partition_type != expected_compute_partition_mode
             ):
                 bad_memory_partition_mode_gpus.append(
                     {
                         "gpu_id": partition_current.gpu_id,
-                        "compute_partition_mode": partition_current.accelerator_type,
-                        "memory_partition_mode": partition_current.memory,
+                        "compute_partition_mode": partition_current.partition_type,
                     }
                 )
+
+        # accelerator currently not avaialbe in API
+
         if bad_memory_partition_mode_gpus:
             self._log_event(
                 category=EventCategory.PLATFORM,
@@ -163,13 +322,52 @@ def analyze_data(self, data: AmdSmiDataModel, args=None) -> TaskResult:
         if args is None:
             args = AmdSmiAnalyzerArgs()
 
+        if args.l0_to_recovery_count_error_threshold is None:
+            args.l0_to_recovery_count_error_threshold = self.L0_TO_RECOVERY_COUNT_ERROR_THRESHOLD
+        if args.l0_to_recovery_count_warning_threshold is None:
+            args.l0_to_recovery_count_warning_threshold = (
+                self.L0_TO_RECOVERY_COUNT_WARNING_THRESHOLD
+            )
+
         if args.expected_gpu_processes:
             self.expected_gpu_processes(data.process, args.expected_gpu_processes)
+
+        if data.static is None or len(data.static) == 0:
+            self._log_event(
+                category=EventCategory.PLATFORM,
+                description="No AMD SMI static data available",
+                priority=EventPriority.WARNING,
+                data={"amdsmi_static_data": data.static},
+            )
+        else:
+            if args.expected_max_power:
+                self.check_expected_max_power(data.static, args.expected_max_power)
+            if args.expected_driver_version:
+                self.check_expected_driver_version(data.static, args.expected_driver_version)
             if args.expected_memory_partition_mode or args.expected_compute_partition_mode:
                 self.check_expected_memory_partition_mode(
                     data.partition,
                     args.expected_memory_partition_mode,
                     args.expected_compute_partition_mode,
                 )
+            self.static_consistancy_check(data.static)
+            if (
+                self.system_info.sku
+                and args.devid_ep
+                and args.devid_ep_vf
+                and args.vendorid_ep
+                and args.check_static_data
+            ) or args.check_static_data:
+                self.check_static_data(
+                    data.static,
+                    args.vendorid_ep,
+                    args.vendorid_ep,
+                    (args.devid_ep, args.devid_ep),
+                    (args.devid_ep, args.devid_ep),
+                    sku_name=args.sku_name,
+                )
+
+        if args.expected_pldm_version:
+            self.check_pldm_version(data.firmware, args.expected_pldm_version)
 
         return self.result

nodescraper/plugins/inband/amdsmi/amdsmi_plugin.py

@@ -25,13 +25,19 @@
 ###############################################################################
 from nodescraper.base import InBandDataPlugin
 
+from .amdsmi_analyzer import AmdSmiAnalyzer
 from .amdsmi_collector import AmdSmiCollector
 from .amdsmidata import AmdSmiDataModel
+from .analyzer_args import AmdSmiAnalyzerArgs
 
 
-class AmdSmiPlugin(InBandDataPlugin[AmdSmiDataModel, None, None]):
+class AmdSmiPlugin(InBandDataPlugin[AmdSmiDataModel, None, AmdSmiAnalyzerArgs]):
     """Plugin for collection and analysis of amdsmi data"""
 
     DATA_MODEL = AmdSmiDataModel
 
     COLLECTOR = AmdSmiCollector
+
+    ANALYZER = AmdSmiAnalyzer
+
+    ANALYZER_ARGS = AmdSmiAnalyzerArgs

nodescraper/plugins/inband/amdsmi/analyzer_args.py

@@ -30,11 +30,16 @@
 
 class AmdSmiAnalyzerArgs(AnalyzerArgs):
 
-    check_static_data: bool = False
-    expected_gpu_processes: Optional[int] = None
-    expected_max_power: Optional[int] = None
-    expected_driver_version: Optional[str] = None
-    expected_memory_partition_mode: Optional[str] = None
-    expected_compute_partition_mode: Optional[str] = None
-    expected_pldm_version: Optional[str] = None
-    xgmi_speed_override: Optional[float] = None
+    check_static_data: bool = True
+    expected_gpu_processes: Optional[int] = 12
+    expected_max_power: Optional[int] = 2
+    expected_driver_version: Optional[str] = "5"
+    expected_memory_partition_mode: Optional[str] = "test"
+    expected_compute_partition_mode: Optional[str] = "test2"
+    expected_pldm_version: Optional[str] = "test3"
+    l0_to_recovery_count_error_threshold: Optional[int] = 1
+    l0_to_recovery_count_warning_threshold: Optional[int] = 2
+    vendorid_ep: Optional["str"] = "vendorid_ep"
+    vendorid_ep_vf: Optional["str"] = "vendorid_ep_vf"
+    devid_ep: Optional["str"] = "devid_ep"
+    sku_name: Optional["str"] = "sku_name"