osdev: use multiple bits in the osdev-specific type

Most GPUs are also CoProcs.
Most CoProcs are also GPUs.
Memory may also be Storage when non-volatile.
OFED is also Network.

hwloc_obj_type_snprintf() only prints one type when in shortnames mode.

Signed-off-by: Brice Goglin <[email protected]>

Author: bgoglin

NEWS

@@ -32,8 +32,12 @@ Version 3.0.0
     in PCI object attributes.
   + The OS device "type" attribute is now a bitmask.
     - "Block" OS devices are now "Storage" (for actual Block devices)
-      or "Memory" (for Linux DAX and CXL devices, etc).
+      and/or "Memory" (for Linux DAX and CXL devices, etc).
+    - Some "CoProcessors" are also "GPUs" (e.g. CUDA devices),
+      and some GPUs are also CoProcessors (e.g. RSMI).
+    - "OpenFabrics" devices are now also "Network".
     - BXI interfaces are now "Network" instead of "OpenFabrics".
+    - See "OS devices" in the documentation for details.
   + The "Backend" info attribute of OS devices is now in the topology
     root object together with other backends.
   + The array and count of info attributes are now embedded in the new

doc/examples/gpu.c

@@ -3,7 +3,7 @@
  * - getting CUDA and OpenCL attributes
  * - displaying the locality of the GPU
  *
- * Copyright © 2009-2019 Inria.  All rights reserved.
+ * Copyright © 2009-2023 Inria.  All rights reserved.
  * Copyright © 2009-2011,2017 Université Bordeaux
  * Copyright © 2009-2010 Cisco Systems, Inc.  All rights reserved.
  * See COPYING in top-level directory.
@@ -36,7 +36,7 @@ int main(void)
       obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_OS_DEVICE, i);
       printf("%s:\n", obj->name);
 
-      /* obj->attr->osdev.type is HWLOC_OBJ_OSDEV_COPROC */
+      /* obj->attr->osdev.type is HWLOC_OBJ_OSDEV_COPROC|HWLOC_OBJ_OSDEV_GPU */
 
       s = hwloc_obj_get_info_by_name(obj, "Backend");
       /* obj->subtype also contains CUDA or OpenCL since v2.0 */

doc/hwloc.doxy

@@ -1652,7 +1652,7 @@ components when I/O discovery is enabled and supported.
  <li><em>sda</em> or <em>nvme0n1</em> (Linux component)</li>
  </ul>
 </li>
-<li>Non-volatile, special-purpose or CXL memory devices (::HWLOC_OBJ_OSDEV_MEMORY)
+<li>Non-volatile, high-bandwidth, or special-purpose or CXL memory devices (::HWLOC_OBJ_OSDEV_MEMORY, with also ::HWLOC_OBJ_OSDEV_STORAGE if non-volatile)
  <ul>
  <li><em>dax2.0</em> or <em>mem0</em> (Linux component)</li>
  </ul>
@@ -1661,35 +1661,37 @@ components when I/O discovery is enabled and supported.
  <li><em>eth0</em>, <em>wlan0</em>, <em>ib0</em> (Linux component)</li>
  <li><em>hsn0</em> with "Slingshot" subtype for HPE Cray HSNs (Linux component).</li>
  <li><em>bxi0</em> with "BXI" subtype for Atos/Bull BXI HCAs (Linux component).</li>
+ <li>OpenFabrics devices as explained below.
  </ul>
 </li>
-<li>OpenFabrics (InfiniBand, Omni-Path, usNIC, etc) HCAs (::HWLOC_OBJ_OSDEV_OPENFABRICS)
+<li>OpenFabrics (InfiniBand, Omni-Path, usNIC, etc) HCAs (::HWLOC_OBJ_OSDEV_OPENFABRICS and ::HWLOC_OBJ_OSDEV_NETWORK)
  <ul>
  <li><em>mlx5_0</em>, <em>hfi1_0</em>, <em>qib0</em>, <em>usnic_0</em> (Linux component)</li>
  </ul>
 </li>
-<li>GPUs (::HWLOC_OBJ_OSDEV_GPU)
+<li>GPUs (::HWLOC_OBJ_OSDEV_GPU and possibly ::HWLOC_OBJ_OSDEV_COPROC)
  <ul>
- <li><em>rsmi0</em> for the first RSMI device
-  ("RSMI" subtype, from the RSMI component, using the AMD ROCm SMI library)</li>
- <li><em>nvml0</em> for the first NVML device
-  ("NVML" subtype, from the NVML component, using the NVIDIA Management Library)</li>
  <li><em>:0.0</em> for the first display
   ("Display" subtype, from the GL component, using the NV-CONTROL X extension library, NVCtrl)</li>
  <li><em>card0</em> and <em>renderD128</em> for DRM device files
   (from the Linux component, filtered-out by default because considered non-important)</li>
+ <li>Any GPU that may be used as a co-processor as explained below.
  </ul>
 </li>
-<li>Co-Processors (::HWLOC_OBJ_OSDEV_COPROC)
+<li>Co-Processors (::HWLOC_OBJ_OSDEV_COPROC and possibly ::HWLOC_OBJ_OSDEV_GPU)
  <ul>
  <li><em>opencl0d0</em> for the first device of the first OpenCL platform,
   <em>opencl1d3</em> for the fourth device of the second OpenCL platform
   ("OpenCL" subtype, from the  OpenCL component)</li>
  <li><em>ze0</em> for the first Level Zero device
   ("LevelZero" subtype, from the levelzero component, using the oneAPI Level Zero library),
   and <em>ze0.1</em> for its second subdevice (if any).</li>
+ <li><em>rsmi0</em> for the first RSMI device
+  ("RSMI" subtype, from the RSMI component, using the AMD ROCm SMI library)</li>
  <li><em>cuda0</em> for the first NVIDIA CUDA device
   ("CUDA" subtype, from the CUDA component, using the NVIDIA CUDA Library)</li>
+ <li><em>nvml0</em> for the first NVML device
+  ("NVML" subtype, from the NVML component, using the NVIDIA Management Library)</li>
  <li><em>ve0</em> for the first NEC Vector Engine device
   ("VectorEngine" subtype, from the Linux component)</li>
  </ul>
@@ -1911,8 +1913,10 @@ for instance their <tt>logical_index</tt> or <tt>os_index</tt>
 
 The kind of object is first described by the <tt>obj->type</tt>
 generic attribute (an integer).
-OS devices also have a specific <tt>obj->attr->osdev.type</tt> integer
+OS devices also have a specific <tt>obj->attr->osdev.type</tt> bitmask
 for distinguishing between NICs, GPUs, etc.
+For instance a GPU coprocessor has both ::HWLOC_OBJ_OSDEV_COPROC and
+::HWLOC_OBJ_OSDEV_GPU bits in this bitmask (see also \ref iodevices_osdev).
 
 Objects may also have an optional <tt>obj->subtype</tt> pointing
 to a better description string (displayed by lstopo either
@@ -1933,8 +1937,8 @@ subtype <tt>Cluster</tt>, <tt>Module</tt>, <tt>Tile</tt>, <tt>Compute Unit</tt>,
 </li>
 <li>OS devices (see also \ref iodevices_osdev):
 <ul>
-<li>Co-processor: subtype <tt>OpenCL</tt>, <tt>LevelZero</tt>, <tt>CUDA</tt>, or <tt>VectorEngine</tt>.</li>
-<li>GPU: subtype <tt>RSMI</tt> (AMD GPU) or <tt>NVML</tt> (NVIDIA GPU).</li>
+<li>subtype <tt>OpenCL</tt>, <tt>LevelZero</tt>, <tt>CUDA</tt>, <tt>VectorEngine</tt>,
+    <tt>RSMI</tt> (AMD GPU) or <tt>NVML</tt> (NVIDIA GPU) for GPU and/or CoProcessor OS devices.</li>
 <li>Network: subtype <tt>Slingshot</tt> (HPE Cray Slingshot Cassini HSN) or <tt>BXI</tt> (Bull/Atos BXI HCA).</li>
 <li>Storage: subtype <tt>Disk</tt>, <tt>Tape</tt>, or <tt>Removable Media Device</tt>.</li>
 <li>Memory: <tt>NVM</tt> (non-volatile memory), <tt>SPM</tt> (specific-purpose memory), <tt>CXLMem</tt> (CXL volatile ou persistent memory).</li>
@@ -2128,7 +2132,7 @@ and SectorSize (in bytes).
  in a CXL Type-3 device.
  Sizes are in KiB (1024 bytes).
 </dd>
-<dt>GPUVendor, GPUModel (GPU or Co-Processor OS devices)</dt>
+<dt>GPUVendor, GPUModel (GPU and/or Co-Processor OS devices)</dt>
 <dd>The vendor and model names of the GPU device.
 </dd>
 <dt>OpenCLDeviceType, OpenCLPlatformIndex,</dt>

hwloc/topology-cuda.c

@@ -103,7 +103,7 @@ hwloc_cuda_discover(struct hwloc_backend *backend, struct hwloc_disc_status *dst
     snprintf(cuda_name, sizeof(cuda_name), "cuda%d", i);
     cuda_device->name = strdup(cuda_name);
     cuda_device->depth = HWLOC_TYPE_DEPTH_UNKNOWN;
-    cuda_device->attr->osdev.type = HWLOC_OBJ_OSDEV_COPROC;
+    cuda_device->attr->osdev.type = HWLOC_OBJ_OSDEV_COPROC | HWLOC_OBJ_OSDEV_GPU;
 
     cuda_device->subtype = strdup("CUDA");
     hwloc_obj_add_info(cuda_device, "GPUVendor", "NVIDIA Corporation");

hwloc/topology-levelzero.c

@@ -707,7 +707,7 @@ hwloc_levelzero_discover(struct hwloc_backend *backend, struct hwloc_disc_status
       snprintf(buffer, sizeof(buffer), "ze%u", zeidx); // ze0d0 ?
       osdev->name = strdup(buffer);
       osdev->depth = HWLOC_TYPE_DEPTH_UNKNOWN;
-      osdev->attr->osdev.type = HWLOC_OBJ_OSDEV_COPROC;
+      osdev->attr->osdev.type = HWLOC_OBJ_OSDEV_COPROC | HWLOC_OBJ_OSDEV_GPU;
       osdev->subtype = strdup("LevelZero");
 
       snprintf(buffer, sizeof(buffer), "%u", i);
@@ -735,7 +735,7 @@ hwloc_levelzero_discover(struct hwloc_backend *backend, struct hwloc_disc_status
             snprintf(tmp, sizeof(tmp), "ze%u.%u", zeidx, k);
             subosdevs[k]->name = strdup(tmp);
             subosdevs[k]->depth = HWLOC_TYPE_DEPTH_UNKNOWN;
-            subosdevs[k]->attr->osdev.type = HWLOC_OBJ_OSDEV_COPROC;
+            subosdevs[k]->attr->osdev.type = HWLOC_OBJ_OSDEV_COPROC | HWLOC_OBJ_OSDEV_GPU;
             subosdevs[k]->subtype = strdup("LevelZero");
             snprintf(tmp, sizeof(tmp), "%u", k);
             hwloc_obj_add_info(subosdevs[k], "LevelZeroSubdeviceID", tmp);

hwloc/topology-linux.c

@@ -3856,7 +3856,13 @@ annotate_dax_parent(hwloc_obj_t obj, const char *name, int fsroot_fd)
   }
 
   /* we'll convert SPM (specific-purpose memory) into a HBM subtype later by looking at memattrs */
-  type = strstr(begin, "ndbus") ? "NVM" : "SPM";
+  if (strstr(begin, "ndbus")) {
+    type = "NVM";
+    if (obj->type == HWLOC_OBJ_OS_DEVICE)
+      obj->attr->osdev.type |= HWLOC_OBJ_OSDEV_STORAGE;
+  } else {
+    type = "SPM";
+  }
   hwloc_obj_add_info(obj, "DAXType", type);
 
   /* try to get some CXL info from the region */
@@ -6355,7 +6361,7 @@ hwloc_linuxfs_lookup_dax_class(struct hwloc_backend *backend, unsigned osdev_fla
 	continue;
 
       obj = hwloc_linux_add_os_device(backend, parent, HWLOC_OBJ_OSDEV_MEMORY, dirent->d_name);
-
+      /* annotate_dax_parent() will add HWLOC_OBJ_OSDEV_STORAGE if NVM */
       annotate_dax_parent(obj, dirent->d_name, root_fd);
 
       hwloc_linuxfs_dax_class_fillinfos(backend, root_fd, obj, path);
@@ -6561,7 +6567,7 @@ hwloc_linuxfs_lookup_infiniband_class(struct hwloc_backend *backend, unsigned os
     if (!parent)
       continue;
 
-    obj = hwloc_linux_add_os_device(backend, parent, HWLOC_OBJ_OSDEV_OPENFABRICS, dirent->d_name);
+    obj = hwloc_linux_add_os_device(backend, parent, HWLOC_OBJ_OSDEV_NETWORK | HWLOC_OBJ_OSDEV_OPENFABRICS, dirent->d_name);
 
     hwloc_linuxfs_infiniband_class_fillinfos(root_fd, obj, path);
   }
@@ -6849,6 +6855,7 @@ hwloc_linuxfs_cxlmem_fillinfos(int root_fd,
       snprintf(tmp, sizeof(tmp), "%llu", value / 1024);
       hwloc_obj_add_info(obj, "CXLPMEMSize", tmp);
     }
+    obj->attr->osdev.type |= HWLOC_OBJ_OSDEV_STORAGE;
   }
 
   snprintf(path, sizeof(path), "%s/serial", osdevpath);
@@ -6883,7 +6890,7 @@ hwloc_linuxfs_lookup_cxlmem(struct hwloc_backend *backend, unsigned osdev_flags)
 	continue;
 
       obj = hwloc_linux_add_os_device(backend, parent, HWLOC_OBJ_OSDEV_MEMORY, dirent->d_name);
-
+      /* hwloc_linuxfs_cxlmem_fillinfos() will add HWLOC_OBJ_OSDEV_STORAGE if PMEM */
       hwloc_linuxfs_cxlmem_fillinfos(root_fd, obj, path);
     }
     closedir(dir);

hwloc/topology-nvml.c

@@ -222,7 +222,7 @@ hwloc_nvml_discover(struct hwloc_backend *backend, struct hwloc_disc_status *dst
     osdev->name = strdup(buffer);
     osdev->subtype = strdup("NVML");
     osdev->depth = HWLOC_TYPE_DEPTH_UNKNOWN;
-    osdev->attr->osdev.type = HWLOC_OBJ_OSDEV_GPU;
+    osdev->attr->osdev.type = HWLOC_OBJ_OSDEV_COPROC | HWLOC_OBJ_OSDEV_GPU;
 
     hwloc_obj_add_info(osdev, "GPUVendor", "NVIDIA Corporation");
 

hwloc/topology-opencl.c

@@ -120,19 +120,23 @@ hwloc_opencl_discover(struct hwloc_backend *backend, struct hwloc_disc_status *d
       snprintf(buffer, sizeof(buffer), "opencl%ud%u", j, i);
       osdev->name = strdup(buffer);
       osdev->depth = HWLOC_TYPE_DEPTH_UNKNOWN;
-      osdev->attr->osdev.type = HWLOC_OBJ_OSDEV_COPROC;
 
       osdev->subtype = strdup("OpenCL");
 
       /* in theory, we should handle cases such GPU|Accelerator|CPU for strange platforms/devices */
-      if (type & CL_DEVICE_TYPE_GPU)
+      if (type & CL_DEVICE_TYPE_GPU) {
 	hwloc_obj_add_info(osdev, "OpenCLDeviceType", "GPU");
-      else if (type & CL_DEVICE_TYPE_ACCELERATOR)
+        osdev->attr->osdev.type = HWLOC_OBJ_OSDEV_COPROC | HWLOC_OBJ_OSDEV_GPU;
+      } else if (type & CL_DEVICE_TYPE_ACCELERATOR) {
 	hwloc_obj_add_info(osdev, "OpenCLDeviceType", "Accelerator");
-      else if (type & HWLOC_CL_DEVICE_TYPE_CUSTOM)
+        osdev->attr->osdev.type = HWLOC_OBJ_OSDEV_COPROC;
+      } else if (type & HWLOC_CL_DEVICE_TYPE_CUSTOM) {
 	hwloc_obj_add_info(osdev, "OpenCLDeviceType", "Custom"); /* Custom cannot be combined with any other type */
-      else
+        osdev->attr->osdev.type = HWLOC_OBJ_OSDEV_COPROC;
+      } else {
 	hwloc_obj_add_info(osdev, "OpenCLDeviceType", "Unknown");
+        osdev->attr->osdev.type = HWLOC_OBJ_OSDEV_COPROC;
+      }
 
       buffer[0] = '\0';
       clGetDeviceInfo(device_ids[i], CL_DEVICE_VENDOR, sizeof(buffer), buffer, NULL);

hwloc/topology-rsmi.c

@@ -294,7 +294,7 @@ hwloc_rsmi_discover(struct hwloc_backend *backend, struct hwloc_disc_status *dst
     osdev->name = strdup(buffer);
     osdev->subtype = strdup("RSMI");
     osdev->depth = HWLOC_TYPE_DEPTH_UNKNOWN;
-    osdev->attr->osdev.type = HWLOC_OBJ_OSDEV_GPU;
+    osdev->attr->osdev.type = HWLOC_OBJ_OSDEV_GPU | HWLOC_OBJ_OSDEV_COPROC;
 
     hwloc_obj_add_info(osdev, "GPUVendor", "AMD");
 

hwloc/topology-xml.c

@@ -848,28 +848,51 @@ hwloc__xml_import_object(hwloc_topology_t topology,
     unsigned long oldtype = obj->attr->osdev.type;
     switch (oldtype) {
     case 0: /* v2 Block */
-      obj->attr->osdev.type = HWLOC_OBJ_OSDEV_STORAGE;
-      if ((obj->name && !strncmp(obj->name, "dax", 3))
-          || (obj->subtype && !strcmp(obj->subtype, "CXLMem")))
+      if ((obj->name && !strncmp(obj->name, "dax", 3))) {
+        /* DAX is MEMORY, and STORAGE if NVM */
         obj->attr->osdev.type = HWLOC_OBJ_OSDEV_MEMORY;
+        if (obj->subtype && !strcmp(obj->subtype, "NVM"))
+          obj->attr->osdev.type |= HWLOC_OBJ_OSDEV_STORAGE;
+      } else if (obj->subtype && !strcmp(obj->subtype, "CXLMem")) {
+        /* CXL is MEMORY, and STORAGE if there's some PMEM */
+        const char *info = hwloc_obj_get_info_by_name(obj, "CXLPMEMSize");
+        obj->attr->osdev.type = HWLOC_OBJ_OSDEV_MEMORY;
+        if (info)
+          obj->attr->osdev.type |= HWLOC_OBJ_OSDEV_STORAGE;
+      } else {
+        /* anything else is STORAGE */
+        obj->attr->osdev.type = HWLOC_OBJ_OSDEV_STORAGE;
+      }
       break;
     case 1: /* v2 GPU */
       obj->attr->osdev.type = HWLOC_OBJ_OSDEV_GPU;
+      if (obj->name && (!strncmp(obj->name, "rsmi", 4) || !strncmp(obj->name, "nvml", 4) /* no RSMI/NVML subtype for v2.5 */))
+        obj->attr->osdev.type |= HWLOC_OBJ_OSDEV_COPROC;
       break;
     case 2: /* v2 Net */
       obj->attr->osdev.type = HWLOC_OBJ_OSDEV_NETWORK;
       break;
     case 3: /* v2 OFED */
-      /* BXI which became NET only */
-      obj->attr->osdev.type = HWLOC_OBJ_OSDEV_OPENFABRICS;
-      if (obj->subtype && !strcmp(obj->subtype, "BXI"))
-        obj->attr->osdev.type = HWLOC_OBJ_OSDEV_NETWORK;
+      /* everything is NET and OFED, except BXI which became NET only */
+      obj->attr->osdev.type = HWLOC_OBJ_OSDEV_NETWORK;
+      if (!obj->subtype || strcmp(obj->subtype, "BXI"))
+        obj->attr->osdev.type |= HWLOC_OBJ_OSDEV_OPENFABRICS;
       break;
     case 4: /* v2 DMA */
       obj->attr->osdev.type = HWLOC_OBJ_OSDEV_DMA;
       break;
     case 5: /* v2 COPROC */
       obj->attr->osdev.type = HWLOC_OBJ_OSDEV_COPROC;
+      if (obj->subtype) {
+        /* CUDA and L0 are also GPU, and OpenCL GPU obviously */
+        if (!strcmp(obj->subtype, "CUDA") || !strcmp(obj->subtype, "LevelZero"))
+          obj->attr->osdev.type |= HWLOC_OBJ_OSDEV_GPU;
+        else if (!strcmp(obj->subtype, "OpenCL")) {
+          const char *info = hwloc_obj_get_info_by_name(obj, "OpenCLDeviceType");
+          if (info && !strcmp(info, "GPU"))
+            obj->attr->osdev.type |= HWLOC_OBJ_OSDEV_GPU;
+        }
+      }
       break;
     default:
       /* unrecognized, no type */
@@ -2247,18 +2270,21 @@ hwloc__xml_export_object_contents (hwloc__xml_export_state_t state, hwloc_topolo
     if (v2export) {
       if (obj->attr->osdev.type & (HWLOC_OBJ_OSDEV_STORAGE|HWLOC_OBJ_OSDEV_MEMORY)) {
         state->new_prop(state, "osdev_type", "0"); /* v2 Block */
-      } else if (obj->attr->osdev.type == HWLOC_OBJ_OSDEV_OPENFABRICS) {
+      } else if (obj->attr->osdev.type & HWLOC_OBJ_OSDEV_OPENFABRICS) {
         state->new_prop(state, "osdev_type", "3"); /* v2 OFED */
-      } else if (obj->attr->osdev.type == HWLOC_OBJ_OSDEV_NETWORK) {
+      } else if (obj->attr->osdev.type & HWLOC_OBJ_OSDEV_NETWORK) {
         if (obj->subtype && !strcmp(obj->subtype, "BXI"))
           state->new_prop(state, "osdev_type", "3"); /* v2 OFED */
         else
           state->new_prop(state, "osdev_type", "2"); /* v2 Net */
-      } else if (obj->attr->osdev.type == HWLOC_OBJ_OSDEV_DMA) {
+      } else if (obj->attr->osdev.type & HWLOC_OBJ_OSDEV_DMA) {
         state->new_prop(state, "osdev_type", "4"); /* v2 DMA */
-      } else if (obj->attr->osdev.type == HWLOC_OBJ_OSDEV_COPROC) {
-        state->new_prop(state, "osdev_type", "5"); /* v2 CoProc */
-      } else if (obj->attr->osdev.type == HWLOC_OBJ_OSDEV_GPU) {
+      } else if (obj->attr->osdev.type & HWLOC_OBJ_OSDEV_COPROC) {
+        if (obj->name && (!strncmp(obj->name, "nvml", 4) || !strncmp(obj->name, "rsmi", 4)))
+          state->new_prop(state, "osdev_type", "1"); /* v2 GPU */
+        else
+          state->new_prop(state, "osdev_type", "5"); /* v2 CoProc */
+      } else if (obj->attr->osdev.type & HWLOC_OBJ_OSDEV_GPU) {
         state->new_prop(state, "osdev_type", "1"); /* v2 GPU */
       }
     } else {