fix: failed on detection in amd

Signed-off-by: thxCode <[email protected]>

Author: thxCode

gpustack_runtime/detector/amd.py

@@ -9,6 +9,7 @@
 from .__types__ import Detector, Device, Devices, ManufacturerEnum
 from .__utils__ import (
     PCIDevice,
+    byte_to_mebibyte,
     get_brief_version,
     get_pci_devices,
     get_utilization,
@@ -141,20 +142,48 @@ def detect(self) -> Devices | None:
                 elif dev_gpudev_info and hasattr(dev_gpudev_info, "cu_active_number"):
                     dev_cores = dev_gpudev_info.cu_active_number
 
-                dev_gpu_metrics_info = pyamdsmi.amdsmi_get_gpu_metrics_info(dev)
-                dev_cores_util = dev_gpu_metrics_info.get("average_gfx_activity", 0)
-                dev_gpu_vram_usage = pyamdsmi.amdsmi_get_gpu_vram_usage(dev)
-                dev_mem = dev_gpu_vram_usage.get("vram_total")
-                dev_mem_used = dev_gpu_vram_usage.get("vram_used")
-                dev_temp = dev_gpu_metrics_info.get("temperature_hotspot", 0)
-
-                dev_power_info = pyamdsmi.amdsmi_get_power_info(dev)
-                dev_power = dev_power_info.get("power_limit", 0) // 1000000  # uW to W
-                dev_power_used = (
-                    dev_power_info.get("current_socket_power")
-                    if dev_power_info.get("current_socket_power", "N/A") != "N/A"
-                    else dev_power_info.get("average_socket_power", 0)
-                )
+                dev_cores_util, dev_temp = None, None
+                try:
+                    dev_gpu_metrics_info = pyamdsmi.amdsmi_get_gpu_metrics_info(dev)
+                    dev_cores_util = dev_gpu_metrics_info.get("average_gfx_activity", 0)
+                    dev_temp = dev_gpu_metrics_info.get("temperature_hotspot", 0)
+                except pyamdsmi.AmdSmiException:
+                    with contextlib.suppress(pyrocmsmi.ROCMSMIError):
+                        pyrocmsmi.rsmi_init()
+                        dev_cores_util = pyrocmsmi.rsmi_dev_busy_percent_get(dev_idx)
+                        dev_temp = pyrocmsmi.rsmi_dev_temp_metric_get(dev_idx)
+
+                dev_mem, dev_mem_used = None, None
+                try:
+                    dev_gpu_vram_usage = pyamdsmi.amdsmi_get_gpu_vram_usage(dev)
+                    dev_mem = dev_gpu_vram_usage.get("vram_total")
+                    dev_mem_used = dev_gpu_vram_usage.get("vram_used")
+                except pyamdsmi.AmdSmiException:
+                    with contextlib.suppress(pyrocmsmi.ROCMSMIError):
+                        pyrocmsmi.rsmi_init()
+                        dev_mem = byte_to_mebibyte(  # byte to MiB
+                            pyrocmsmi.rsmi_dev_memory_total_get(dev_idx),
+                        )
+                        dev_mem_used = byte_to_mebibyte(  # byte to MiB
+                            pyrocmsmi.rsmi_dev_memory_usage_get(dev_idx),
+                        )
+
+                dev_power, dev_power_used = None, None
+                try:
+                    dev_power_info = pyamdsmi.amdsmi_get_power_info(dev)
+                    dev_power = (
+                        dev_power_info.get("power_limit", 0) // 1000000
+                    )  # uW to W
+                    dev_power_used = (
+                        dev_power_info.get("current_socket_power")
+                        if dev_power_info.get("current_socket_power", "N/A") != "N/A"
+                        else dev_power_info.get("average_socket_power", 0)
+                    )
+                except pyamdsmi.AmdSmiException:
+                    with contextlib.suppress(pyrocmsmi.ROCMSMIError):
+                        pyrocmsmi.rsmi_init()
+                        dev_power = pyrocmsmi.rsmi_dev_power_cap_get(dev_idx)
+                        dev_power_used = pyrocmsmi.rsmi_dev_power_get(dev_idx)
 
                 dev_compute_partition = None
                 with contextlib.suppress(pyamdsmi.AmdSmiException):

gpustack_runtime/detector/pyamdgpu/__init__.py

@@ -221,6 +221,7 @@ def _LoadAMDGPULibrary():
                 # Linux path
                 locs = [
                     "libdrm_amdgpu.so.1.0.0",
+                    "libdrm_amdgpu.so.1",
                     "libdrm_amdgpu.so",
                 ]
                 for loc in locs:

pack/Dockerfile

@@ -18,7 +18,7 @@
 # - GPUSTACK_RUNTIME_DOCKER_MIRRORED_NAME_FILTER_LABELS: Semicolon-separated list of labels to filter mirrored images when deploying mirrored deployment.
 ARG PYTHON_VERSION=3.11
 ARG GPUSTACK_RUNTIME_BASE_IMAGE=runtime
-ARG GPUSTACK_RUNTIME_ROCM_VERSION=6.2.4
+ARG GPUSTACK_RUNTIME_ROCM_VERSION=6.4.4
 ARG GPUSTACK_RUNTIME_DOCKER_MIRRORED_NAME_FILTER_LABELS
 
 FROM ubuntu:22.04@sha256:3c61d3759c2639d4b836d32a2d3c83fa0214e36f195a3421018dbaaf79cbe37f AS runtime
@@ -346,6 +346,21 @@ RUN --mount=type=bind,from=rocm-base,source=/opt/rocm/share,target=/opt/rocm/sha
     uv pip install --no-build-isolation \
         /opt/rocm/share/amd_smi
     uv pip tree
+
+    # Hack to avoid: Fail to open libdrm_amdgpu.so: libdrm_amdgpu.so: cannot open shared object file: No such file or directory
+    TARGET_DIR="/usr/lib/$(uname -m)-linux-gnu"
+    TARGET_LIB="libdrm_amdgpu.so.1.0.0"
+    TARGET_LINK="libdrm_amdgpu.so"
+    TARGET_LIB_EXISTED="true"
+    if [[ ! -e "${TARGET_DIR}/${TARGET_LIB}" ]]; then
+        TARGET_LIB_EXISTED="false"
+        touch "${TARGET_DIR}/${TARGET_LIB}"
+    fi
+    pushd "${TARGET_DIR}" \
+        && ln -sf "${TARGET_LIB}" "${TARGET_LINK}"
+    if [[ "${TARGET_LIB_EXISTED}" == "false" ]]; then
+        rm -f "${TARGET_DIR}/${TARGET_LIB}"
+    fi
 EOF
 ENV AMD_VISIBLE_DEVICES="all" \
     GPUSTACK_RUNTIME_DEPLOY_MIRRORED_DEPLOYMENT_IGNORE_VOLUMES="/opt/rocm"

pyproject.toml

@@ -46,5 +46,5 @@ dev=[
     "mkdocs-material>=9.6.18",
     "pymdown-extensions>=10.16.1",
     "dockerpty>=0.4.1",
-    "amdsmi==6.2.4",
+    "amdsmi>=6.4.4",
 ]