Skip to content

Commit e4333bf

Browse files
bgoglinbgoglin
committed
memattrs: sort memory tiers by BW and expose MemoryTier info attr
Add MemoryTier=0 to node in first tier, etc. Check tiers in the memtiers test. Signed-off-by: Brice Goglin <[email protected]>
1 parent 1da57da commit e4333bf

10 files changed

+155
-68
lines changed

‎doc/hwloc.doxy

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2278,6 +2278,11 @@ and GID #1 of port #3.
22782278
These info attributes are attached to objects specified in parentheses.
22792279

22802280
<dl>
2281+
<dt>MemoryTier (NUMA Nodes)</dt>
2282+
<dd>The rank of the memory tier of this node.
2283+
Ranks start from 0 for highest bandwidth nodes.
2284+
The attribute is only set if multiple tiers are found.
2285+
</dd>
22812286
<dt>CXLDevice (NUMA Nodes or DAX Memory OS devices)</dt>
22822287
<dd>The PCI/CXL bus ID of a device whose CXL Type-3 memory is exposed here.
22832288
If multiple devices are interleaved, their bus IDs are separated by commas,

‎hwloc/memattrs.c

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1221,7 +1221,7 @@ hwloc_get_local_numanode_objs(hwloc_topology_t topology,
12211221
*/
12221222

12231223
enum hwloc_memory_tier_type_e {
1224-
/* WARNING: the order is important */
1224+
/* WARNING: keep higher BW types first for compare_tiers_by_bw_and_type() when BW info is missing */
12251225
HWLOC_MEMORY_TIER_HBM = 1UL<<0,
12261226
HWLOC_MEMORY_TIER_DRAM = 1UL<<1,
12271227
HWLOC_MEMORY_TIER_GPU = 1UL<<2,
@@ -1306,6 +1306,22 @@ static int compare_node_infos_by_type_and_bw(const void *_a, const void *_b)
13061306
return 0;
13071307
}
13081308

1309+
static int compare_tiers_by_bw_and_type(const void *_a, const void *_b)
1310+
{
1311+
const struct hwloc_memory_tier_s *a = _a, *b = _b;
1312+
/* sort by (average) BW first */
1313+
if (a->local_bw_min && b->local_bw_min) {
1314+
if (a->local_bw_min + a->local_bw_max > b->local_bw_min + b->local_bw_max)
1315+
return -1;
1316+
else if (a->local_bw_min + a->local_bw_max < b->local_bw_min + b->local_bw_max)
1317+
return 1;
1318+
}
1319+
/* then by tier type */
1320+
if (a->type != b->type)
1321+
return a->type - b->type;
1322+
return 0;
1323+
}
1324+
13091325
static struct hwloc_memory_tier_s *
13101326
hwloc__group_memory_tiers(hwloc_topology_t topology,
13111327
unsigned *nr_tiers_p)
@@ -1781,6 +1797,11 @@ hwloc__apply_memory_tiers_subtypes(hwloc_topology_t topology,
17811797
} else
17821798
hwloc_debug(" node L#%u P#%u already marked as %s, not setting %s\n",
17831799
node->logical_index, node->os_index, node->subtype, subtype);
1800+
if (nr_tiers > 1) {
1801+
char tmp[20];
1802+
snprintf(tmp, sizeof(tmp), "%u", j);
1803+
hwloc__replace_infos(&node->infos, "MemoryTier", tmp);
1804+
}
17841805
break; /* each node is in a single tier */
17851806
}
17861807
}
@@ -1814,6 +1835,10 @@ hwloc_internal_memattrs_guess_memory_tiers(hwloc_topology_t topology)
18141835

18151836
hwloc__guess_memory_tiers_types(topology, nr_tiers, tiers);
18161837

1838+
/* sort tiers by BW first, then by type */
1839+
hwloc_debug("Sorting memory tiers...\n");
1840+
qsort(tiers, nr_tiers, sizeof(*tiers), compare_tiers_by_bw_and_type);
1841+
18171842
ready:
18181843
#ifdef HWLOC_DEBUG
18191844
for(i=0; i<nr_tiers; i++) {

‎tests/hwloc/linux/64intel64-fakeKNL-SNC4-hybrid-msc.console

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ Machine (P#0 total=12GiB ClusterMode=SNC4 MemoryMode=Hybrid50)
22
Package L#0 (P#0 total=12GiB CPUVendor=GenuineIntel CPUFamilyNumber=6 CPUModelNumber=87 CPUModel=06/57 CPUStepping=0)
33
Group0(Cluster) L#0 (total=3072MiB)
44
MemCache L#0 (total=1024MiB size=2048MiB linesize=64 ways=1 Inclusive=1)
5-
NUMANode(DRAM) L#0 (P#0 local=1024MiB total=1024MiB)
6-
NUMANode(MCDRAM) L#1 (P#7 local=2048MiB total=2048MiB)
5+
NUMANode(DRAM) L#0 (P#0 local=1024MiB total=1024MiB MemoryTier=1)
6+
NUMANode(MCDRAM) L#1 (P#7 local=2048MiB total=2048MiB MemoryTier=0)
77
L2Cache L#0 (size=4096KiB linesize=64)
88
L1dCache L#0 (size=32KiB linesize=64)
99
L1iCache L#0 (size=32KiB linesize=64)
@@ -36,8 +36,8 @@ Machine (P#0 total=12GiB ClusterMode=SNC4 MemoryMode=Hybrid50)
3636
PU L#15 (P#51)
3737
Group0(Cluster) L#1 (total=3072MiB)
3838
MemCache L#1 (total=1024MiB size=2048MiB linesize=64 ways=1 Inclusive=1)
39-
NUMANode(DRAM) L#2 (P#1 local=1024MiB total=1024MiB)
40-
NUMANode(MCDRAM) L#3 (P#4 local=2048MiB total=2048MiB)
39+
NUMANode(DRAM) L#2 (P#1 local=1024MiB total=1024MiB MemoryTier=1)
40+
NUMANode(MCDRAM) L#3 (P#4 local=2048MiB total=2048MiB MemoryTier=0)
4141
L2Cache L#2 (size=4096KiB linesize=64)
4242
L1dCache L#4 (size=32KiB linesize=64)
4343
L1iCache L#4 (size=32KiB linesize=64)
@@ -70,8 +70,8 @@ Machine (P#0 total=12GiB ClusterMode=SNC4 MemoryMode=Hybrid50)
7070
PU L#31 (P#55)
7171
Group0(Cluster) L#2 (total=3072MiB)
7272
MemCache L#2 (total=1024MiB size=2048MiB linesize=64 ways=1 Inclusive=1)
73-
NUMANode(DRAM) L#4 (P#2 local=1024MiB total=1024MiB)
74-
NUMANode(MCDRAM) L#5 (P#5 local=2048MiB total=2048MiB)
73+
NUMANode(DRAM) L#4 (P#2 local=1024MiB total=1024MiB MemoryTier=1)
74+
NUMANode(MCDRAM) L#5 (P#5 local=2048MiB total=2048MiB MemoryTier=0)
7575
L2Cache L#4 (size=4096KiB linesize=64)
7676
L1dCache L#8 (size=32KiB linesize=64)
7777
L1iCache L#8 (size=32KiB linesize=64)
@@ -104,8 +104,8 @@ Machine (P#0 total=12GiB ClusterMode=SNC4 MemoryMode=Hybrid50)
104104
PU L#47 (P#59)
105105
Group0(Cluster) L#3 (total=3072MiB)
106106
MemCache L#3 (total=1024MiB size=2048MiB linesize=64 ways=1 Inclusive=1)
107-
NUMANode(DRAM) L#6 (P#3 local=1024MiB total=1024MiB)
108-
NUMANode(MCDRAM) L#7 (P#6 local=2048MiB total=2048MiB)
107+
NUMANode(DRAM) L#6 (P#3 local=1024MiB total=1024MiB MemoryTier=1)
108+
NUMANode(MCDRAM) L#7 (P#6 local=2048MiB total=2048MiB MemoryTier=0)
109109
L2Cache L#6 (size=4096KiB linesize=64)
110110
L1dCache L#12 (size=32KiB linesize=64)
111111
L1iCache L#12 (size=32KiB linesize=64)

‎tests/hwloc/linux/64intel64-fakeKNL-SNC4-hybrid.console

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
Machine (P#0 total=12GiB ClusterMode=SNC4 MemoryMode=Hybrid50)
22
Package L#0 (P#0 total=12GiB CPUVendor=GenuineIntel CPUFamilyNumber=6 CPUModelNumber=87 CPUModel=06/57 CPUStepping=0)
33
L3Cache(MemorySideCache) L#0 (total=3072MiB size=2048MiB linesize=64 ways=1 Inclusive=1)
4-
NUMANode(DRAM) L#0 (P#0 local=1024MiB total=1024MiB)
5-
NUMANode(MCDRAM) L#1 (P#7 local=2048MiB total=2048MiB)
4+
NUMANode(DRAM) L#0 (P#0 local=1024MiB total=1024MiB MemoryTier=1)
5+
NUMANode(MCDRAM) L#1 (P#7 local=2048MiB total=2048MiB MemoryTier=0)
66
L2Cache L#0 (size=4096KiB linesize=64)
77
L1dCache L#0 (size=32KiB linesize=64)
88
L1iCache L#0 (size=32KiB linesize=64)
@@ -34,8 +34,8 @@ Machine (P#0 total=12GiB ClusterMode=SNC4 MemoryMode=Hybrid50)
3434
PU L#14 (P#35)
3535
PU L#15 (P#51)
3636
L3Cache(MemorySideCache) L#1 (total=3072MiB size=2048MiB linesize=64 ways=1 Inclusive=1)
37-
NUMANode(DRAM) L#2 (P#1 local=1024MiB total=1024MiB)
38-
NUMANode(MCDRAM) L#3 (P#4 local=2048MiB total=2048MiB)
37+
NUMANode(DRAM) L#2 (P#1 local=1024MiB total=1024MiB MemoryTier=1)
38+
NUMANode(MCDRAM) L#3 (P#4 local=2048MiB total=2048MiB MemoryTier=0)
3939
L2Cache L#2 (size=4096KiB linesize=64)
4040
L1dCache L#4 (size=32KiB linesize=64)
4141
L1iCache L#4 (size=32KiB linesize=64)
@@ -67,8 +67,8 @@ Machine (P#0 total=12GiB ClusterMode=SNC4 MemoryMode=Hybrid50)
6767
PU L#30 (P#39)
6868
PU L#31 (P#55)
6969
L3Cache(MemorySideCache) L#2 (total=3072MiB size=2048MiB linesize=64 ways=1 Inclusive=1)
70-
NUMANode(DRAM) L#4 (P#2 local=1024MiB total=1024MiB)
71-
NUMANode(MCDRAM) L#5 (P#5 local=2048MiB total=2048MiB)
70+
NUMANode(DRAM) L#4 (P#2 local=1024MiB total=1024MiB MemoryTier=1)
71+
NUMANode(MCDRAM) L#5 (P#5 local=2048MiB total=2048MiB MemoryTier=0)
7272
L2Cache L#4 (size=4096KiB linesize=64)
7373
L1dCache L#8 (size=32KiB linesize=64)
7474
L1iCache L#8 (size=32KiB linesize=64)
@@ -100,8 +100,8 @@ Machine (P#0 total=12GiB ClusterMode=SNC4 MemoryMode=Hybrid50)
100100
PU L#46 (P#43)
101101
PU L#47 (P#59)
102102
L3Cache(MemorySideCache) L#3 (total=3072MiB size=2048MiB linesize=64 ways=1 Inclusive=1)
103-
NUMANode(DRAM) L#6 (P#3 local=1024MiB total=1024MiB)
104-
NUMANode(MCDRAM) L#7 (P#6 local=2048MiB total=2048MiB)
103+
NUMANode(DRAM) L#6 (P#3 local=1024MiB total=1024MiB MemoryTier=1)
104+
NUMANode(MCDRAM) L#7 (P#6 local=2048MiB total=2048MiB MemoryTier=0)
105105
L2Cache L#6 (size=4096KiB linesize=64)
106106
L1dCache L#12 (size=32KiB linesize=64)
107107
L1iCache L#12 (size=32KiB linesize=64)

‎tests/hwloc/linux/fakeheteromemtiers.console

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ Machine (P#0 total=6090MiB DMIProductName="Standard PC (i440FX + PIIX, 1996)" DM
22
Package L#0 (P#0 total=6090MiB CPUVendor=GenuineIntel CPUFamilyNumber=15 CPUModelNumber=107 CPUModel="QEMU Virtual CPU version 2.5+" CPUStepping=1)
33
L3Cache L#0 (P#0 total=6090MiB size=16MiB linesize=64 ways=16)
44
Group0 L#0 (total=3319MiB)
5-
NUMANode(DRAM) L#0 (P#0 local=2935MiB total=2935MiB)
6-
NUMANode(NVM) L#1 (P#8 local=384MiB total=384MiB DAXDevice=dax7.0 DAXType=NVM DAXParent=LNXSYSTM:00/LNXSYBUS:00/ACPI0012:00/ndbus0/region7/dax7.1)
5+
NUMANode(DRAM) L#0 (P#0 local=2935MiB total=2935MiB MemoryTier=1)
6+
NUMANode(NVM) L#1 (P#8 local=384MiB total=384MiB DAXDevice=dax7.0 DAXType=NVM DAXParent=LNXSYSTM:00/LNXSYBUS:00/ACPI0012:00/ndbus0/region7/dax7.1 MemoryTier=2)
77
L2Cache L#0 (P#0 size=4096KiB linesize=64 ways=16)
88
L1dCache L#0 (P#0 size=32KiB linesize=64 ways=8)
99
L1iCache L#0 (P#0 size=32KiB linesize=64 ways=8)
@@ -16,9 +16,9 @@ Machine (P#0 total=6090MiB DMIProductName="Standard PC (i440FX + PIIX, 1996)" DM
1616
PU L#1 (P#1)
1717
OSDev[Memory](SPM) L#0 (DAXType=SPM DAXParent=hmem.0 Size=524288 LinuxDeviceID=253:0) "dax0.0"
1818
Group0 L#1 (total=1875MiB)
19-
NUMANode(DRAM) L#2 (P#1 local=979MiB total=979MiB)
20-
NUMANode(HBM) L#3 (P#4 local=512MiB total=512MiB DAXDevice=dax1.0 DAXType=SPM DAXParent=hmem.1)
21-
NUMANode(NVM) L#4 (P#6 local=384MiB total=384MiB DAXDevice=dax5.0 DAXType=NVM DAXParent=LNXSYSTM:00/LNXSYBUS:00/ACPI0012:00/ndbus0/region5/dax5.1)
19+
NUMANode(DRAM) L#2 (P#1 local=979MiB total=979MiB MemoryTier=1)
20+
NUMANode(HBM) L#3 (P#4 local=512MiB total=512MiB DAXDevice=dax1.0 DAXType=SPM DAXParent=hmem.1 MemoryTier=0)
21+
NUMANode(NVM) L#4 (P#6 local=384MiB total=384MiB DAXDevice=dax5.0 DAXType=NVM DAXParent=LNXSYSTM:00/LNXSYBUS:00/ACPI0012:00/ndbus0/region5/dax5.1 MemoryTier=2)
2222
L2Cache L#2 (P#2 size=4096KiB linesize=64 ways=16)
2323
L1dCache L#2 (P#2 size=32KiB linesize=64 ways=8)
2424
L1iCache L#2 (P#2 size=32KiB linesize=64 ways=8)
@@ -31,8 +31,8 @@ Machine (P#0 total=6090MiB DMIProductName="Standard PC (i440FX + PIIX, 1996)" DM
3131
PU L#3 (P#3)
3232
OSDev[Memory](SPM) L#1 (DAXType=SPM DAXParent=hmem.2 Size=524288 LinuxDeviceID=253:2) "dax2.0"
3333
Group0 L#2 (total=896MiB)
34-
NUMANode(HBM) L#5 (P#2 local=512MiB total=512MiB DAXDevice=dax3.0 DAXType=SPM DAXParent=hmem.3)
35-
NUMANode(NVM) L#6 (P#9 local=384MiB total=384MiB DAXDevice=dax8.0 DAXType=NVM DAXParent=LNXSYSTM:00/LNXSYBUS:00/ACPI0012:00/ndbus0/region8/dax8.1)
34+
NUMANode(HBM) L#5 (P#2 local=512MiB total=512MiB DAXDevice=dax3.0 DAXType=SPM DAXParent=hmem.3 MemoryTier=0)
35+
NUMANode(NVM) L#6 (P#9 local=384MiB total=384MiB DAXDevice=dax8.0 DAXType=NVM DAXParent=LNXSYSTM:00/LNXSYBUS:00/ACPI0012:00/ndbus0/region8/dax8.1 MemoryTier=2)
3636
L2Cache L#4 (P#4 size=4096KiB linesize=64 ways=16)
3737
L1dCache L#4 (P#4 size=32KiB linesize=64 ways=8)
3838
L1iCache L#4 (P#4 size=32KiB linesize=64 ways=8)

‎tests/hwloc/linux/fakememinitiators-1np2c+1npp+gi.console

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,28 @@
11
Machine (P#0 total=1861GiB)
22
Package L#0 (P#0 total=929GiB CPUModel="Intel(R) Xeon(R) Gold 6230 CPU @ 2.10GHz")
3-
NUMANode(NVM) L#2 (P#7 local=742GiB total=742GiB DAXDevice=dax0.0 DAXType=NVM DAXParent=LNXSYSTM:00/LNXSYBUS:00/ACPI0012:00/ndbus0/region0/dax0.1)
3+
NUMANode(NVM) L#2 (P#7 local=742GiB total=742GiB DAXDevice=dax0.0 DAXType=NVM DAXParent=LNXSYSTM:00/LNXSYBUS:00/ACPI0012:00/ndbus0/region0/dax0.1 MemoryTier=1)
44
Group0 L#0 (total=93GiB)
5-
NUMANode L#0 (P#5 local=93GiB total=93GiB)
5+
NUMANode L#0 (P#5 local=93GiB total=93GiB MemoryTier=0)
66
Core L#0 (P#0)
77
PU L#0 (P#0)
88
Core L#1 (P#0)
99
PU L#1 (P#1)
1010
Group0 L#1 (total=94GiB)
11-
NUMANode L#1 (P#6 local=94GiB total=94GiB)
11+
NUMANode L#1 (P#6 local=94GiB total=94GiB MemoryTier=0)
1212
Core L#2 (P#0)
1313
PU L#2 (P#4)
1414
Core L#3 (P#0)
1515
PU L#3 (P#5)
1616
Package L#1 (P#1 total=931GiB CPUModel="Intel(R) Xeon(R) Gold 6230 CPU @ 2.10GHz")
17-
NUMANode(NVM) L#5 (P#10 local=744GiB total=744GiB DAXDevice=dax1.0 DAXType=NVM DAXParent=LNXSYSTM:00/LNXSYBUS:00/ACPI0012:00/ndbus0/region1/dax1.1)
17+
NUMANode(NVM) L#5 (P#10 local=744GiB total=744GiB DAXDevice=dax1.0 DAXType=NVM DAXParent=LNXSYSTM:00/LNXSYBUS:00/ACPI0012:00/ndbus0/region1/dax1.1 MemoryTier=1)
1818
Group0 L#2 (total=93GiB)
19-
NUMANode L#3 (P#8 local=93GiB total=93GiB)
19+
NUMANode L#3 (P#8 local=93GiB total=93GiB MemoryTier=0)
2020
Core L#4 (P#1)
2121
PU L#4 (P#2)
2222
Core L#5 (P#1)
2323
PU L#5 (P#3)
2424
Group0 L#3 (total=94GiB)
25-
NUMANode L#4 (P#9 local=94GiB total=94GiB)
25+
NUMANode L#4 (P#9 local=94GiB total=94GiB MemoryTier=0)
2626
Core L#6 (P#1)
2727
PU L#6 (P#6)
2828
Core L#7 (P#1)

‎tests/hwloc/linux/nvidiagpunumanodes-kept.console

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
Machine (P#0 total=342GiB PlatformName=PowerNV PlatformModel="PowerNV 8335-GTW")
22
Package L#0 (P#0 total=169GiB CPUModel="POWER9, altivec supported" CPURevision="2.1 (pvr 004e 1201)")
3-
NUMANode L#0 (P#0 local=124GiB total=124GiB)
4-
NUMANode(GPUMemory) L#1 (P#253 local=15GiB total=15GiB PCIBusID=0006:00:00.0)
5-
NUMANode(GPUMemory) L#2 (P#254 local=15GiB total=15GiB PCIBusID=0004:06:00.0)
6-
NUMANode(GPUMemory) L#3 (P#255 local=15GiB total=15GiB PCIBusID=0004:05:00.0)
3+
NUMANode L#0 (P#0 local=124GiB total=124GiB MemoryTier=0)
4+
NUMANode(GPUMemory) L#1 (P#253 local=15GiB total=15GiB PCIBusID=0006:00:00.0 MemoryTier=1)
5+
NUMANode(GPUMemory) L#2 (P#254 local=15GiB total=15GiB PCIBusID=0004:06:00.0 MemoryTier=1)
6+
NUMANode(GPUMemory) L#3 (P#255 local=15GiB total=15GiB PCIBusID=0004:05:00.0 MemoryTier=1)
77
L3Cache L#0 (size=10MiB linesize=0)
88
L2Cache L#0 (size=512KiB linesize=0)
99
L1dCache L#0 (size=32KiB linesize=128 ways=32)
@@ -43,10 +43,10 @@ Machine (P#0 total=342GiB PlatformName=PowerNV PlatformModel="PowerNV 8335-GTW")
4343
HostBridge L#2 (buses=0006:[00-00])
4444
PCI L#2 (busid=0006:00:00.0 id=0000:0000 class=0300(VGA))
4545
Package L#1 (P#8 total=173GiB CPUModel="POWER9, altivec supported" CPURevision="2.1 (pvr 004e 1201)")
46-
NUMANode L#4 (P#8 local=128GiB total=128GiB)
47-
NUMANode(GPUMemory) L#5 (P#250 local=15GiB total=15GiB PCIBusID=0035:05:00.0)
48-
NUMANode(GPUMemory) L#6 (P#251 local=15GiB total=15GiB PCIBusID=0035:04:00.0)
49-
NUMANode(GPUMemory) L#7 (P#252 local=15GiB total=15GiB PCIBusID=0007:00:00.0)
46+
NUMANode L#4 (P#8 local=128GiB total=128GiB MemoryTier=0)
47+
NUMANode(GPUMemory) L#5 (P#250 local=15GiB total=15GiB PCIBusID=0035:05:00.0 MemoryTier=1)
48+
NUMANode(GPUMemory) L#6 (P#251 local=15GiB total=15GiB PCIBusID=0035:04:00.0 MemoryTier=1)
49+
NUMANode(GPUMemory) L#7 (P#252 local=15GiB total=15GiB PCIBusID=0007:00:00.0 MemoryTier=1)
5050
L3Cache L#2 (size=10MiB linesize=0)
5151
L2Cache L#2 (size=512KiB linesize=0)
5252
L1dCache L#4 (size=32KiB linesize=128 ways=32)

0 commit comments

Comments
 (0)