fix: failed on amd mixed nvidia

Signed-off-by: thxCode <[email protected]>

Author: thxCode

gpustack_runtime/detector/amd.py

@@ -2,17 +2,14 @@
 
 import contextlib
 import logging
-import os
 from functools import lru_cache
-from pathlib import Path
 
 from .. import envs
 from . import pyamdgpu, pyamdsmi, pyrocmsmi
 from .__types__ import Detector, Device, Devices, ManufacturerEnum
 from .__utils__ import (
     PCIDevice,
     get_brief_version,
-    get_device_files,
     get_pci_devices,
     get_utilization,
 )
@@ -91,24 +88,21 @@ def detect(self) -> Devices | None:
             sys_runtime_ver = get_brief_version(sys_runtime_ver_original)
 
             devs = pyamdsmi.amdsmi_get_processor_handles()
-            dev_files = get_device_files(
-                pattern=r"card(?P<number>\d+)",
-                directory="/dev/dri",
-            )
             for dev_idx, dev in enumerate(devs):
-                dev_card = None
                 dev_index = dev_idx
-                if len(dev_files) >= len(devs):
-                    dev_file = dev_files[dev_idx]
-                    if dev_file.number is not None:
-                        dev_card = dev_file.number
+
+                dev_card = None
+                if hasattr(pyamdsmi, "amdsmi_get_gpu_kfd_info"):
+                    dev_kfd_info = pyamdsmi.amdsmi_get_gpu_kfd_info(dev)
+                    dev_card = dev_kfd_info.get("node_id")
+                else:
+                    with contextlib.suppress(pyrocmsmi.ROCMSMIError):
+                        pyrocmsmi.rsmi_init()
+                        dev_card = pyrocmsmi.rsmi_dev_node_id_get(dev_idx)
 
                 dev_gpudev_info = None
                 if dev_card is not None:
-                    with (
-                        contextlib.redirect_stderr(Path(os.devnull).open("w")),
-                        contextlib.suppress(pyamdgpu.AMDGPUError),
-                    ):
+                    with contextlib.suppress(pyamdgpu.AMDGPUError):
                         _, _, dev_gpudev = pyamdgpu.amdgpu_device_initialize(
                             dev_card,
                         )

gpustack_runtime/detector/pyamdgpu/__init__.py

@@ -254,16 +254,12 @@ def amdgpu_device_initialize(card=1):
 
 
 def amdgpu_device_deinitialize(device):
-    _LoadAMDGPULibrary()
-
     fn = _amdgpuGetFunctionPointer("amdgpu_device_deinitialize")
     ret = fn(device)
     _amdgpuCheckReturn(ret)
 
 
 def amdgpu_query_gpu_info(device):
-    _LoadAMDGPULibrary()
-
     c_info = c_amdgpu_gpu_info()
     fn = _amdgpuGetFunctionPointer("amdgpu_query_gpu_info")
     ret = fn(device, byref(c_info))

gpustack_runtime/detector/pyrocmsmi/__init__.py

@@ -239,3 +239,13 @@ def rsmi_dev_power_get(device=0):
     ret = rocmsmiLib.rsmi_dev_power_get(device, byref(c_power), byref(c_power_type))
     _rocmsmiCheckReturn(ret)
     return c_power.value // 1000000
+
+
+def rsmi_dev_node_id_get(device=0):
+    if not rocmsmiLib:
+        raise ROCMSMIError(ROCMSMI_ERROR_UNINITIALIZED)
+
+    c_node_id = c_uint32()
+    ret = rocmsmiLib.rsmi_dev_node_id_get(device, byref(c_node_id))
+    _rocmsmiCheckReturn(ret)
+    return c_node_id.value

gpustack_runtime/envs.py

@@ -209,7 +209,7 @@
             "cambricon.com/devices=CAMBRICON_VISIBLE_DEVICES;"
             "hygon.com/devices=HYGON_VISIBLE_DEVICES;"
             "iluvatar.ai/devices=ILUVATAR_VISIBLE_DEVICES;"
-            "metax-tech.com/devices=METAX_VISIBLE_DEVICES;"
+            "metax-tech.com/devices=CUDA_VISIBLE_DEVICES;"
             "mthreads.com/devices=METHERDS_VISIBLE_DEVICES;"
             "nvidia.com/devices=NVIDIA_VISIBLE_DEVICES;",
         ),

pack/Dockerfile

@@ -324,8 +324,7 @@ ENV ILUVATAR_VISIBLE_DEVICES="0"
 ## - Mount /opt/maca from the host to support device detecting.
 ## - Mount /opt/mxdriver/ from the host to support device detecting.
 ## E.g. docker run --rm -it --privileged -v /opt/mxdriver:/opt/mxdriver -v /opt/maca:/opt/maca gpustack/runtime:main gpustack-runtime detect --format json
-ENV METAX_VISIBLE_DEVICES="0" \
-    LD_LIBRARY_PATH="/opt/maca/lib:/opt/maca/ompi/lib:/opt/maca/ucx/lib:/opt/mxdriver/lib:${LD_LIBRARY_PATH}"
+ENV LD_LIBRARY_PATH="/opt/maca/lib:/opt/maca/ompi/lib:/opt/maca/ucx/lib:/opt/mxdriver/lib:${LD_LIBRARY_PATH}"
 
 ## Active all MThread devices detection,
 ## works with (default) MThread container runtime and privileged mode.