Merge pull request #569 from bgoglin/cxldax

CXLDevice attributes in DAX and NUMA nodes

Author: bgoglin

doc/hwloc.doxy

@@ -2127,6 +2127,11 @@ and GID #1 of port #3.
 These info attributes are attached to objects specified in parentheses.
 
 <dl>
+<dt>CXLDevice (NUMA Nodes or DAX Memory OS devices)</dt>
+<dd>The PCI/CXL bus ID of a device whose CXL Type-3 memory is exposed here.
+There may be multiple instances of this attributes if multiple device memories
+are interleaved.
+</dd>
 <dt>DAXDevice (NUMA Nodes)</dt>
 <dd>The name of the Linux DAX device that was used to expose a non-volatile
 memory region as a volatile NUMA node.

hwloc/topology-linux.c

@@ -3690,6 +3690,69 @@ read_node_mscaches(struct hwloc_topology *topology,
   return 0;
 }
 
+static int
+annotate_cxl_dax(hwloc_obj_t obj, unsigned region, int root_fd)
+{
+  char path[300];
+  unsigned i;
+
+  for(i=0; ; i++) {
+    char decoder[20]; /* "decoderX.Y" */
+    char decoderpath[256], *endpoint;
+    char uportpath[256], *pcirootbus, *pcibdf;
+    unsigned pcidomain, pcibus, pcidevice, pcifunc;
+    char *slash, *end;
+    int err;
+
+    /* read the i-th decoder name from file target<i> */
+    snprintf(path, sizeof(path), "/sys/bus/cxl/devices/region%u/target%u", region, i);
+    if (hwloc_read_path_by_length(path, decoder, sizeof(decoder), root_fd) < 0)
+      break;
+    end = strchr(decoder, '\n');
+    if (end)
+      *end = '\0';
+    hwloc_debug("hwloc/dax/cxl: found decoder `%s' for region#%u target#%u\n", decoder, region, i);
+
+    /* get the endpoint symlink which ends with "/portT/endpointX/decoderY.X/" */
+    snprintf(path, sizeof(path), "/sys/bus/cxl/devices/%s", decoder);
+    err = hwloc_readlink(path, decoderpath, sizeof(decoderpath), root_fd);
+    if (err < 0)
+      break;
+    endpoint = strstr(decoderpath, "endpoint");
+    if (!endpoint)
+      break;
+    slash = strchr(endpoint, '/');
+    if (!slash)
+      break;
+    *slash = '\0';
+    hwloc_debug("hwloc/dax/cxl: found endpoint `%s'\n", endpoint);
+
+    /* get the PCI in the endpointX/uport symlink "../../../pci<busid>/<BDFs>../memX" */
+    snprintf(path, sizeof(path), "/sys/bus/cxl/devices/%s/uport", endpoint);
+    err = hwloc_readlink(path, uportpath, sizeof(uportpath), root_fd);
+    if (err < 0)
+      break;
+    hwloc_debug("hwloc/dax/cxl: lookind for BDF at the end of uport `%s'\n", uportpath);
+    pcirootbus = strstr(uportpath, "/pci");
+    if (!pcirootbus)
+      break;
+    slash = pcirootbus + 11; /* "/pciXXXX:YY/" */
+    if (*slash != '/')
+      break;
+    pcibdf = NULL;
+    while (sscanf(slash, "/%x:%x:%x.%x/", &pcidomain, &pcibus, &pcidevice, &pcifunc) == 4) {
+      pcibdf = slash+1;
+      slash += 13;
+    }
+    *slash = '\0';
+    if (pcibdf) {
+      hwloc_obj_add_info(obj, "CXLDevice", pcibdf);
+    }
+  }
+
+  return 0;
+}
+
 static int
 dax_is_kmem(const char *name, int fsroot_fd)
 {
@@ -3705,7 +3768,7 @@ annotate_dax_parent(hwloc_obj_t obj, const char *name, int fsroot_fd)
 {
   char daxpath[300];
   char link[PATH_MAX];
-  char *begin, *end;
+  char *begin, *end, *region;
   const char *type;
   int err;
 
@@ -3719,7 +3782,11 @@ annotate_dax_parent(hwloc_obj_t obj, const char *name, int fsroot_fd)
    * ../../../devices/LNXSYSTM:00/LNXSYBUS:00/ACPI0012:00/ndbus0/region2/dax2.0/dax2.0/ for NVDIMMs
    * ../../../devices/platform/e820_pmem/ndbus0/region0/dax0.0/dax0.0/ for fake NVM (memmap=size!start kernel parameter)
    * ../../../devices/platform/hmem.0/dax0.0/ for "soft-reserved" specific-purpose memory
+   * ../../../devices/platform/ACPI0017:00/root0/decoder0.0/region0/dax_region0/dax0.0/ for CXL RAM
+   * ../../../devices/platform/ACPI0017:00/root0/nvdimm-bridge0/ndbus0/region0/dax0.0/dax0.0/ for CXL PMEM
    */
+
+  /* remove beginning and end of link to populate DAXParent */
   begin = link;
   /* remove the starting ".." (likely multiple) */
   while (!strncmp(begin, "../", 3))
@@ -3729,9 +3796,9 @@ annotate_dax_parent(hwloc_obj_t obj, const char *name, int fsroot_fd)
     begin += 8;
   if (!strncmp(begin, "platform/", 9))
     begin += 9;
-  /* remove the ending "daxX.Y" (either one or two) */
+  /* stop at the ending "/daxX.Y" */
   end = strstr(begin, name);
-  if (end) {
+  if (end && end != begin && end[-1] == '/') {
     *end = '\0';
     if (end != begin && end[-1] == '/')
       end[-1] = '\0';
@@ -3741,6 +3808,15 @@ annotate_dax_parent(hwloc_obj_t obj, const char *name, int fsroot_fd)
   type = strstr(begin, "ndbus") ? "NVM" : "SPM";
   hwloc_obj_add_info(obj, "DAXType", type);
 
+  /* try to get some CXL info from the region */
+  region = strstr(begin, "/region");
+  if (region) {
+    unsigned i = strtoul(region+7, &end, 10);
+    if (end != region+7)
+      annotate_cxl_dax(obj, i, fsroot_fd);
+  }
+
+  /* insert DAXParent last because it's likely less useful than others */
   hwloc_obj_add_info(obj, "DAXParent", begin);
 
   /*