memattrs: add "node0_is_dram" option to HWLOC_MEMTIERS_GUESS envvar

NUMA node P#0 is usually DRAM (except of HBM-only CPUs).

Signed-off-by: Brice Goglin <[email protected]>

Author: bgoglin

doc/hwloc.doxy

@@ -1123,9 +1123,9 @@ following environment variables.
   By default, hwloc only uses heuristics that are likely correct
   and disables those that are unlikely.
   </dd>
-<!-- not stable yet, hence not documented
- HWLOC_MEMTIERS_GUESS=spm_is_hbm
-  assume all SPM nodes are HBM
+<!-- since 2.10, not stable yet, hence not documented
+ HWLOC_MEMTIERS_GUESS=spm_is_hbm,node0_is_dram
+  assume all SPM nodes are HBM, assume node0 is in the DRAM tier
 -->
 
 <dt>HWLOC_GROUPING=1</dt>

hwloc/memattrs.c

@@ -1465,13 +1465,14 @@ hwloc__group_memory_tiers(hwloc_topology_t topology,
 }
 
 enum hwloc_guess_memtiers_flag {
-  HWLOC_GUESS_MEMTIERS_FLAG_SPM_IS_HBM = 1<<0
+  HWLOC_GUESS_MEMTIERS_FLAG_NODE0_IS_DRAM = 1<<0,
+  HWLOC_GUESS_MEMTIERS_FLAG_SPM_IS_HBM = 1<<1
 };
 
 static int
 hwloc__guess_dram_hbm_tiers(struct hwloc_memory_tier_s *tier1,
                             struct hwloc_memory_tier_s *tier2,
-                            unsigned long flags __hwloc_attribute_unused)
+                            unsigned long flags)
 {
   struct hwloc_memory_tier_s *tmp;
 
@@ -1498,6 +1499,13 @@ hwloc__guess_dram_hbm_tiers(struct hwloc_memory_tier_s *tier1,
   }
   /* tier2 BW is >2x tier1 */
 
+  if ((flags & HWLOC_GUESS_MEMTIERS_FLAG_NODE0_IS_DRAM)
+      && hwloc_bitmap_isset(tier2->nodeset, 0)) {
+    /* node0 is not DRAM, and we assume that's not possible */
+    hwloc_debug("    node0 shouldn't have HBM BW\n");
+    return -1;
+  }
+
   /* assume tier1 == DRAM and tier2 == HBM */
   tier1->type = HWLOC_MEMORY_TIER_DRAM;
   tier2->type = HWLOC_MEMORY_TIER_HBM;
@@ -1529,6 +1537,10 @@ hwloc__guess_memory_tiers_types(hwloc_topology_t topology __hwloc_attribute_unus
       hwloc_debug("Assuming SPM-tier is HBM, ignore bandwidth\n");
       flags |= HWLOC_GUESS_MEMTIERS_FLAG_SPM_IS_HBM;
     }
+    if (strstr(env, "node0_is_dram")) {
+      hwloc_debug("Assuming node0 is DRAM\n");
+      flags |= HWLOC_GUESS_MEMTIERS_FLAG_NODE0_IS_DRAM;
+    }
   }
 
   if (nr_tiers == 1)
@@ -1586,6 +1598,17 @@ hwloc__guess_memory_tiers_types(hwloc_topology_t topology __hwloc_attribute_unus
       }
   }
 
+  if (flags & HWLOC_GUESS_MEMTIERS_FLAG_NODE0_IS_DRAM) {
+    /* force mark node0's tier as DRAM if we couldn't guess it */
+    for(i=0; i<nr_tiers; i++)
+      if (hwloc_bitmap_isset(tiers[i].nodeset, 0)
+          && tiers[i].type == HWLOC_MEMORY_TIER_UNKNOWN) {
+        hwloc_debug("Forcing node0 tier to DRAM");
+        tiers[i].type = HWLOC_MEMORY_TIER_DRAM;
+        break;
+      }
+  }
+
   return 0;
 }
 

tests/hwloc/memtiers.c

@@ -129,6 +129,17 @@ main(void)
   check_subtypes(new, "NVM", NULL, "HBM", NULL, "HBM");
   hwloc_topology_destroy(new);
 
+  printf("checking HBM subtypes are set on XML reload if HWLOC_MEMTIERS_GUESS=node0_is_dram,spm_is_hbm\n");
+  putenv((char*)"HWLOC_MEMTIERS_GUESS=node0_is_dram,spm_is_hbm");
+  err = hwloc_topology_init(&new);
+  assert(!err);
+  err = hwloc_topology_set_xmlbuffer(new, xmlbuffer, buflen);
+  assert(!err);
+  err = hwloc_topology_load(new);
+  assert(!err);
+  check_subtypes(new, "NVM", "DRAM", "HBM", "DRAM", "HBM");
+  hwloc_topology_destroy(new);
+
   free(xmlbuffer);
 
   printf("adding correct BW values\n");