refactor: enhance get topology

Signed-off-by: thxCode <[email protected]>

Author: thxCode

gpustack_runtime/detector/amd.py

@@ -3,6 +3,7 @@
 import contextlib
 import logging
 from functools import lru_cache
+from pathlib import Path
 
 from .. import envs
 from ..logging import debug_log_exception, debug_log_warning
@@ -102,47 +103,47 @@ def detect(self) -> Devices | None:
                 dev_index = dev_idx
 
                 dev_gpu_asic_info = pyamdsmi.amdsmi_get_gpu_asic_info(dev)
-                dev_uuid = f"GPU-{(dev_gpu_asic_info.get('asic_serial')[2:]).lower()}"
+                if dev_gpu_asic_info.get("asic_serial") != "N/A":
+                    asic_serial = dev_gpu_asic_info.get("asic_serial")
+                    dev_uuid = f"GPU-{(asic_serial[2:]).lower()}"
+                else:
+                    dev_uuid = f"GPU-{pyrocmsmi.rsmi_dev_unique_id_get(dev_idx)[2:]}"
                 dev_hsa_agent = hsa_agents.get(dev_uuid)
 
-                dev_card_id = None
-                if dev_hsa_agent:
-                    dev_card_id = dev_hsa_agent.driver_node_id
-                elif hasattr(pyamdsmi, "amdsmi_get_gpu_kfd_info"):
-                    dev_kfd_info = pyamdsmi.amdsmi_get_gpu_kfd_info(dev)
-                    dev_card_id = dev_kfd_info.get("node_id")
-                else:
+                dev_gpu_driver_info = pyamdsmi.amdsmi_get_gpu_driver_info(dev)
+                dev_driver_ver = dev_gpu_driver_info.get("driver_version")
+
+                dev_name = dev_hsa_agent.name
+                if not dev_name:
+                    dev_name = dev_gpu_asic_info.get("market_name")
+
+                dev_cc = dev_hsa_agent.compute_capability
+                if not dev_cc:
                     with contextlib.suppress(pyrocmsmi.ROCMSMIError):
-                        dev_card_id = pyrocmsmi.rsmi_dev_node_id_get(dev_idx)
+                        dev_cc = pyrocmsmi.rsmi_dev_target_graphics_version_get(dev_idx)
 
-                dev_gpudev_info = None
-                if dev_card_id is not None:
+                dev_bdf = None
+                dev_card_id = None
+                dev_renderd_id = None
+                with contextlib.suppress(pyamdsmi.AmdSmiException):
+                    dev_bdf = pyamdsmi.amdsmi_get_gpu_device_bdf(dev)
+                    dev_card_id, dev_renderd_id = _get_card_and_renderd_id(dev_bdf)
+
+                dev_cores = dev_hsa_agent.compute_units
+                dev_asic_family_id = dev_hsa_agent.asic_family_id
+                if (
+                    not dev_cores or not dev_asic_family_id
+                ) and dev_card_id is not None:
                     with contextlib.suppress(pyamdgpu.AMDGPUError):
                         _, _, dev_gpudev = pyamdgpu.amdgpu_device_initialize(
                             dev_card_id,
                         )
                         dev_gpudev_info = pyamdgpu.amdgpu_query_gpu_info(dev_gpudev)
                         pyamdgpu.amdgpu_device_deinitialize(dev_gpudev)
-
-                dev_gpu_driver_info = pyamdsmi.amdsmi_get_gpu_driver_info(dev)
-                dev_driver_ver = dev_gpu_driver_info.get("driver_version")
-
-                if dev_hsa_agent:
-                    dev_name = dev_hsa_agent.name
-                    if not dev_name:
-                        dev_name = dev_gpu_asic_info.get("market_name")
-                    dev_cc = dev_hsa_agent.compute_capability
-                else:
-                    dev_name = dev_gpu_asic_info.get("market_name")
-                    dev_cc = None
-                    with contextlib.suppress(pyrocmsmi.ROCMSMIError):
-                        dev_cc = pyrocmsmi.rsmi_dev_target_graphics_version_get(dev_idx)
-
-                dev_cores = None
-                if dev_hsa_agent:
-                    dev_cores = dev_hsa_agent.compute_units
-                elif dev_gpudev_info and hasattr(dev_gpudev_info, "cu_active_number"):
-                    dev_cores = dev_gpudev_info.cu_active_number
+                        if not dev_cores:
+                            dev_cores = dev_gpudev_info.cu_active_number
+                        if not dev_asic_family_id:
+                            dev_asic_family_id = dev_gpudev_info.family_id
 
                 dev_cores_util = None
                 dev_temp = None
@@ -201,24 +202,22 @@ def detect(self) -> Devices | None:
                     )
 
                 dev_appendix = {
-                    "arch_family": _get_arch_family(dev_gpudev_info),
+                    "arch_family": _get_arch_family(dev_asic_family_id),
                     "vgpu": dev_compute_partition is not None,
-                    "card_id": dev_card_id,
                 }
+                if dev_bdf:
+                    dev_appendix["bdf"] = dev_bdf
+                if dev_card_id is not None:
+                    dev_appendix["card_id"] = dev_card_id
+                if dev_renderd_id is not None:
+                    dev_appendix["renderd_id"] = dev_renderd_id
 
                 with contextlib.suppress(pyamdsmi.AmdSmiException):
                     dev_xgmi = pyamdsmi.amdsmi_get_xgmi_info(dev)
-                    for k in [
-                        "xgmi_lanes",
-                        "xgmi_hive_id",
-                        "xgmi_node_id",
-                    ]:
-                        if value := dev_xgmi.get(k):
-                            dev_appendix[k] = value
-
-                with contextlib.suppress(pyamdsmi.AmdSmiException):
-                    dev_bdf = pyamdsmi.amdsmi_get_gpu_device_bdf(dev)
-                    dev_appendix["bdf"] = dev_bdf
+                    if xgmi_lanes := dev_xgmi.get("xgmi_lanes", None):
+                        dev_appendix["xgmi_lanes"] = xgmi_lanes
+                        dev_appendix["xgmi_hive_id"] = dev_xgmi.get("xgmi_hive_id")
+                        dev_appendix["xgmi_node_id"] = dev_xgmi.get("xgmi_node_id")
 
                 ret.append(
                     Device(
@@ -410,15 +409,20 @@ def distance_pci_devices(bdf_a: str, bdf_b: str) -> TopologyDistanceEnum:
         return topology
 
 
-def _get_arch_family(
-    dev_gpudev_info: pyamdgpu.c_amdgpu_gpu_info | None,
-) -> str | None:
-    if not dev_gpudev_info:
-        return None
+def _get_arch_family(dev_family_id: int | None) -> str:
+    """
+    Get the architecture family name from the device family ID.
+
+    Args:
+        dev_family_id:
+            The device family ID.
+
+    Returns:
+        The architecture family as string.
 
-    family_id = dev_gpudev_info.family_id
-    if family_id is None:
-        return None
+    """
+    if dev_family_id is None:
+        return "Unknown"
 
     arch_family = {
         pyamdgpu.AMDGPU_FAMILY_SI: "Southern Islands",
@@ -439,4 +443,29 @@ def _get_arch_family(
         pyamdgpu.AMDGPU_FAMILY_GC_12_0_0: "GC 12.0.0",
     }
 
-    return arch_family.get(family_id, "Unknown")
+    return arch_family.get(dev_family_id, "Unknown")
+
+
+def _get_card_and_renderd_id(dev_bdf: str) -> tuple[int | None, int | None]:
+    """
+    Get the card ID and renderD ID for a given device bdf.
+
+    Args:
+        dev_bdf:
+            The device bdf.
+
+    Returns:
+        A tuple of (card_id, renderd_id).
+
+    """
+    card_id = None
+    renderd_id = None
+    drm_path = Path(f"/sys/module/amdgpu/drivers/pci:amdgpu/{dev_bdf}/drm")
+    if drm_path.exists():
+        for dir_path in drm_path.iterdir():
+            if dir_path.name.startswith("card"):
+                card_id = int(dir_path.name[4:])
+            elif dir_path.name.startswith("renderD"):
+                renderd_id = int(dir_path.name[7:])
+
+    return card_id, renderd_id

gpustack_runtime/detector/hygon.py

@@ -7,7 +7,7 @@
 
 from .. import envs
 from ..logging import debug_log_exception, debug_log_warning
-from . import Topology, pyhsa, pyrocmcore, pyrocmsmi
+from . import Topology, pyamdgpu, pyhsa, pyrocmcore, pyrocmsmi
 from .__types__ import Detector, Device, Devices, ManufacturerEnum, TopologyDistanceEnum
 from .__utils__ import (
     PCIDevice,
@@ -91,13 +91,14 @@ def detect(self) -> Devices | None:
             pyrocmsmi.rsmi_init()
 
             sys_driver_ver = None
-            sys_driver_ver_path = Path("/sys/module/hydcu/version")
-            if sys_driver_ver_path.exists():
-                try:
-                    with sys_driver_ver_path.open(encoding="utf-8") as f:
-                        sys_driver_ver = f.read().strip()
-                except OSError:
-                    pass
+            for path in [
+                Path("/sys/module/hycu/version"),
+                Path("/sys/module/hydcu/version"),
+            ]:
+                if path.exists():
+                    with contextlib.suppress(Exception):
+                        sys_driver_ver = path.read_text().strip()
+                    break
 
             sys_runtime_ver_original = pyrocmcore.getROCmVersion()
             sys_runtime_ver = get_brief_version(sys_runtime_ver_original)
@@ -109,18 +110,34 @@ def detect(self) -> Devices | None:
                 dev_uuid = f"GPU-{pyrocmsmi.rsmi_dev_unique_id_get(dev_idx)[2:]}"
                 dev_hsa_agent = hsa_agents.get(dev_uuid)
 
-                if dev_hsa_agent:
-                    dev_name = dev_hsa_agent.name
-                    if not dev_name:
-                        dev_name = pyrocmsmi.rsmi_dev_name_get(dev_idx)
-                    dev_cc = dev_hsa_agent.compute_capability
-                    dev_cores = dev_hsa_agent.compute_units
-                else:
+                dev_name = dev_hsa_agent.name
+                if not dev_name:
                     dev_name = pyrocmsmi.rsmi_dev_name_get(dev_idx)
-                    dev_cc = pyrocmsmi.rsmi_dev_target_graphics_version_get(dev_idx)
-                    dev_cores = None
+
+                dev_cc = dev_hsa_agent.compute_capability
+                if not dev_cc:
+                    with contextlib.suppress(pyrocmsmi.ROCMSMIError):
+                        dev_cc = pyrocmsmi.rsmi_dev_target_graphics_version_get(dev_idx)
+
+                dev_bdf = None
+                dev_card_id = None
+                dev_renderd_id = None
+                with contextlib.suppress(Exception):
+                    dev_bdf = pyrocmsmi.rsmi_dev_pci_id_get(dev_idx)
+                    dev_card_id, dev_renderd_id = _get_card_and_renderd_id(dev_bdf)
+
+                dev_cores = dev_hsa_agent.compute_units
+                if not dev_cores and dev_card_id is not None:
+                    with contextlib.suppress(pyamdgpu.AMDGPUError):
+                        _, _, dev_gpudev = pyamdgpu.amdgpu_device_initialize(
+                            dev_card_id,
+                        )
+                        dev_gpudev_info = pyamdgpu.amdgpu_query_gpu_info(dev_gpudev)
+                        pyamdgpu.amdgpu_device_deinitialize(dev_gpudev)
+                        dev_cores = dev_gpudev_info.cu_active_number
 
                 dev_cores_util = pyrocmsmi.rsmi_dev_busy_percent_get(dev_idx)
+                dev_temp = pyrocmsmi.rsmi_dev_temp_metric_get(dev_idx)
                 if dev_cores_util is None:
                     debug_log_warning(
                         logger,
@@ -136,18 +153,18 @@ def detect(self) -> Devices | None:
                     pyrocmsmi.rsmi_dev_memory_usage_get(dev_idx),
                 )
 
-                dev_temp = pyrocmsmi.rsmi_dev_temp_metric_get(dev_idx)
-
                 dev_power = pyrocmsmi.rsmi_dev_power_cap_get(dev_idx)
                 dev_power_used = pyrocmsmi.rsmi_dev_power_get(dev_idx)
 
                 dev_appendix = {
                     "vgpu": False,
                 }
-
-                with contextlib.suppress(Exception):
-                    dev_bdf = pyrocmsmi.rsmi_dev_pci_id_get(dev_idx)
+                if dev_bdf is not None:
                     dev_appendix["bdf"] = dev_bdf
+                if dev_card_id is not None:
+                    dev_appendix["card_id"] = dev_card_id
+                if dev_renderd_id is not None:
+                    dev_appendix["renderd_id"] = dev_renderd_id
 
                 ret.append(
                     Device(
@@ -316,3 +333,33 @@ def distance_pci_devices(bdf_a: str, bdf_b: str) -> TopologyDistanceEnum:
             raise
 
         return topology
+
+
+def _get_card_and_renderd_id(dev_bdf: str) -> tuple[int | None, int | None]:
+    """
+    Get the card ID and renderD ID for a given device bdf.
+
+    Args:
+        dev_bdf:
+            The device bdf.
+
+    Returns:
+        A tuple of (card_id, renderd_id).
+
+    """
+    card_id = None
+    renderd_id = None
+
+    for path in [
+        Path(f"/sys/module/hycu/drivers/pci:hycu/{dev_bdf}/drm"),
+        Path(f"/sys/module/hydcu/drivers/pci:hydcu/{dev_bdf}/drm"),
+    ]:
+        if path.exists():
+            for dir_path in path.iterdir():
+                if dir_path.name.startswith("card"):
+                    card_id = int(dir_path.name[4:])
+                elif dir_path.name.startswith("renderD"):
+                    renderd_id = int(dir_path.name[7:])
+            break
+
+    return card_id, renderd_id

gpustack_runtime/detector/pyhsa/__init__.py

@@ -3,6 +3,7 @@
 ##
 from __future__ import annotations
 
+import contextlib
 import os
 import string
 import sys
@@ -437,6 +438,12 @@ def hsa_agent_get_info_driver_node_id(agent):
     return c_driver_node_id.value
 
 
+def has_agent_get_asic_family_id(agent):
+    c_family_id = c_uint32()
+    hsa_agent_get_info(agent, HSA_AMD_AGENT_INFO_ASIC_FAMILY_ID, byref(c_family_id))
+    return c_family_id.value
+
+
 @dataclass
 class Agent:
     device_type: int
@@ -445,7 +452,7 @@ class Agent:
     name: str
     compute_capability: str
     compute_units: int
-    driver_node_id: int
+    asic_family_id: int | None
 
 
 def get_agents() -> list[Agent]:
@@ -464,7 +471,9 @@ def has_agent_callback(agent) -> int:
             agent_name = hsa_agent_get_info_product_name(agent)
             agent_compute_capability = hsa_agent_get_info_name(agent)
             agent_compute_units = hsa_agent_get_info_compute_unit_count(agent)
-            agent_driver_node_id = hsa_agent_get_info_driver_node_id(agent)
+            agent_asic_family_id = None
+            with contextlib.suppress(HSAError):
+                agent_asic_family_id = has_agent_get_asic_family_id(agent)
 
             agents.append(
                 Agent(
@@ -474,7 +483,7 @@ def has_agent_callback(agent) -> int:
                     name=agent_name,
                     compute_capability=agent_compute_capability,
                     compute_units=agent_compute_units,
-                    driver_node_id=agent_driver_node_id,
+                    asic_family_id=agent_asic_family_id,
                 )
             )
 

gpustack_runtime/detector/pyrocmsmi/__init__.py

@@ -324,15 +324,17 @@ def rsmi_dev_pci_id_get(device=0):
     ret = rocmsmiLib.rsmi_dev_pci_id_get(device, byref(c_pci_id))
     _rocmsmiCheckReturn(ret)
 
-    v_pci_id = c_pci_id.value
+    return str_bdfid(c_pci_id.value)
+
+
+def str_bdfid(bdfid: int) -> str:
     # BDFID = ((DOMAIN & 0xFFFFFFFF) << 32) | ((Partition & 0xF) << 28)
     #         | ((BUS & 0xFF) << 8) | ((DEVICE & 0x1F) <<3 )
     #         | (FUNCTION & 0x7)
-    # Extract domain, bus, device, function
-    domain = (v_pci_id >> 32) & 0xFFFFFFFF
-    bus = (v_pci_id >> 8) & 0xFF
-    device_id = (v_pci_id >> 3) & 0x1F
-    function = v_pci_id & 0x7
+    domain = (bdfid >> 32) & 0xFFFFFFFF
+    bus = (bdfid >> 8) & 0xFF
+    device_id = (bdfid >> 3) & 0x1F
+    function = bdfid & 0x7
     return f"{domain:04x}:{bus:02x}:{device_id:02x}.{function:x}"