add L3 shared cache info

Change-Id: I302d35bde3d4f725327d1dc4827957e369ca5f1b

Author: rdementi

src/cpucounters.cpp

@@ -1101,6 +1101,7 @@ bool PCM::discoverSystemTopology()
     uint32 smtMaskWidth = 0;
     uint32 coreMaskWidth = 0;
     uint32 l2CacheMaskShift = 0;
+    uint32 l3CacheMaskShift = 0;
 
     struct domain
     {
@@ -1111,7 +1112,7 @@ bool PCM::discoverSystemTopology()
     {
         TemporalThreadAffinity aff0(0);
 
-        if (initCoreMasks(smtMaskWidth, coreMaskWidth, l2CacheMaskShift) == false)
+        if (initCoreMasks(smtMaskWidth, coreMaskWidth, l2CacheMaskShift, l3CacheMaskShift) == false)
         {
             std::cerr << "ERROR: Major problem? No leaf 0 under cpuid function 11.\n";
             return false;
@@ -1151,20 +1152,18 @@ bool PCM::discoverSystemTopology()
             for (size_t l = 0; l < topologyDomains.size(); ++l)
             {
                 topologyDomainMap[topologyDomains[l].type] = topologyDomains[l];
-#if 0
-                std::cerr << "Topology level: " << l <<
-                                      " type: " << topologyDomains[l].type <<
-                                      " (" << TopologyEntry::getDomainTypeStr(topologyDomains[l].type) << ")" <<
-                                      " width: " << topologyDomains[l].width <<
-                                      " levelShift: " << topologyDomains[l].levelShift <<
-                                      " nextLevelShift: " << topologyDomains[l].nextLevelShift << "\n";
-#endif
+                DBG(1 , "Topology level: " , l ,
+                                      " type: " , topologyDomains[l].type ,
+                                      " (" , TopologyEntry::getDomainTypeStr(topologyDomains[l].type) , ")" ,
+                                      " width: " , topologyDomains[l].width ,
+                                      " levelShift: " , topologyDomains[l].levelShift ,
+                                      " nextLevelShift: " , topologyDomains[l].nextLevelShift);
             }
         }
     }
 
 #ifndef __APPLE__
-    auto populateEntry = [&topologyDomainMap,&smtMaskWidth, &coreMaskWidth, &l2CacheMaskShift](TopologyEntry& entry)
+    auto populateEntry = [&topologyDomainMap,&smtMaskWidth, &coreMaskWidth, &l2CacheMaskShift, &l3CacheMaskShift](TopologyEntry& entry)
     {
         auto getAPICID = [&](const uint32 leaf)
         {
@@ -1218,6 +1217,7 @@ bool PCM::discoverSystemTopology()
         {
             fillEntry(entry, smtMaskWidth, coreMaskWidth, l2CacheMaskShift, getAPICID(0xb));
         }
+        entry.l3_cache_id = extract_bits_32(getAPICID(0xb), l3CacheMaskShift, 31);
     };
 #endif
 
@@ -3231,7 +3231,7 @@ void PCM::printDetailedSystemTopology(const int detailLevel)
         std::cerr << "Tile_Id         ";
         if (detailLevel > 0) std::cerr << "Die_Id          Die_Group_Id    ";
         std::cerr << "Package_Id      Core_Type       Native_CPU_Model\n";
-        std::map<uint32, std::vector<uint32> > os_id_by_core, os_id_by_tile, core_id_by_socket;
+        std::map<uint32, std::vector<uint32> > os_id_by_core, os_id_by_tile, core_id_by_socket, os_id_by_l3_cache;
         size_t counter = 0;
         for (auto it = topology.begin(); it != topology.end(); ++it)
         {
@@ -3252,6 +3252,7 @@ void PCM::printDetailedSystemTopology(const int detailLevel)
             // add socket offset to distinguish cores and tiles from different sockets
             os_id_by_core[(it->socket_id << 15) + it->core_id].push_back(it->os_id);
             os_id_by_tile[(it->socket_id << 15) + it->tile_id].push_back(it->os_id);
+            os_id_by_l3_cache[(it->socket_id << 15) + it->l3_cache_id].push_back(it->os_id);
 
             ++counter;
         }
@@ -3288,6 +3289,16 @@ void PCM::printDetailedSystemTopology(const int detailLevel)
             }
             std::cerr << ")";
         }
+        std::cerr << "\nL3$ ";
+        for (auto core = os_id_by_l3_cache.begin(); core != os_id_by_l3_cache.end(); ++core)
+        {
+            auto os_id = core->second.begin();
+            std::cerr << "(" << *os_id;
+            for (++os_id; os_id != core->second.end(); ++os_id) {
+                std::cerr << "," << *os_id;
+            }
+            std::cerr << ")";
+        }
         std::cerr << "\n";
         std::cerr << "\n";
     }

src/topologyentry.h

@@ -4,6 +4,7 @@
 #pragma once
 
 #include "types.h"
+#include "debug.h"
 
 namespace pcm
 {
@@ -25,6 +26,7 @@ struct PCM_API TopologyEntry // describes a core
     int32 die_grp_id;
     int32 socket_id;
     int32 socket_unique_core_id;
+    int32 l3_cache_id = -1;
     int32 native_cpu_model = -1;
     enum DomainTypeID
     {
@@ -103,7 +105,7 @@ inline void fillEntry(TopologyEntry & entry, const uint32 & smtMaskWidth, const
     entry.socket_unique_core_id = entry.core_id;
 }
 
-inline bool initCoreMasks(uint32 & smtMaskWidth, uint32 & coreMaskWidth, uint32 & l2CacheMaskShift)
+inline bool initCoreMasks(uint32 & smtMaskWidth, uint32 & coreMaskWidth, uint32 & l2CacheMaskShift, uint32 & l3CacheMaskShift)
 {
     // init constants for CPU topology leaf 0xB
     // adapted from Topology Enumeration Reference code for Intel 64 Architecture
@@ -154,24 +156,62 @@ inline bool initCoreMasks(uint32 & smtMaskWidth, uint32 & coreMaskWidth, uint32
 
         (void) coreMaskWidth; // to suppress warnings on MacOS (unused vars)
 
-    #ifdef PCM_DEBUG_TOPOLOGY
-        uint32 threadsSharingL2;
-    #endif
-        uint32 l2CacheMaskWidth;
+        uint32 threadsSharingL2 = 0;
+        uint32 l2CacheMaskWidth = 0;
 
         pcm_cpuid(0x4, 2, cpuid_args); // get ID for L2 cache
         l2CacheMaskWidth = 1 + extract_bits_32(cpuid_args.array[0],14,25); // number of APIC IDs sharing L2 cache
-    #ifdef PCM_DEBUG_TOPOLOGY
         threadsSharingL2 = l2CacheMaskWidth;
-    #endif
         for( ; l2CacheMaskWidth > 1; l2CacheMaskWidth >>= 1)
         {
             l2CacheMaskShift++;
         }
-    #ifdef PCM_DEBUG_TOPOLOGY
-        std::cerr << "DEBUG: Number of threads sharing L2 cache = " << threadsSharingL2
-                << " [the most significant bit = " << l2CacheMaskShift << "]\n";
-    #endif
+        DBG(1, "Number of threads sharing L2 cache = " , threadsSharingL2, " [the most significant bit = " , l2CacheMaskShift , "]");
+
+        uint32 threadsSharingL3 = 0;
+        uint32 l3CacheMaskWidth = 0;
+
+        pcm_cpuid(0x4, 3, cpuid_args); // get ID for L3 cache
+        l3CacheMaskWidth = 1 + extract_bits_32(cpuid_args.array[0], 14, 25); // number of APIC IDs sharing L3 cache
+        threadsSharingL3 = l3CacheMaskWidth;
+        for( ; l3CacheMaskWidth > 1; l3CacheMaskWidth >>= 1)
+        {
+            l3CacheMaskShift++;
+        }
+        DBG(1, "Number of threads sharing L3 cache = " , threadsSharingL3, " [the most significant bit = " , l3CacheMaskShift , "]");
+
+        uint32 it = 0;
+
+        for (int i = 0; i < 100; ++i)
+        {
+            uint32 threadsSharingCache = 0;
+            uint32 CacheMaskWidth = 0;
+            uint32 CacheMaskShift = 0;
+            pcm_cpuid(0x4, it, cpuid_args);
+            const auto cacheType = extract_bits_32(cpuid_args.array[0], 0, 4);
+            if (cacheType == 0)
+            {
+                break; // no more caches
+            }
+            const char * cacheTypeStr = nullptr;
+            switch (cacheType)
+            {
+                case 1: cacheTypeStr = "data"; break;
+                case 2: cacheTypeStr = "instruction"; break;
+                case 3: cacheTypeStr = "unified"; break;
+                default: cacheTypeStr = "unknown"; break;
+            }
+            const auto level = extract_bits_32(cpuid_args.array[0], 5, 7);
+            CacheMaskWidth = 1 + extract_bits_32(cpuid_args.array[0], 14, 25); // number of APIC IDs sharing cache
+            threadsSharingCache = CacheMaskWidth;
+            for( ; CacheMaskWidth > 1; CacheMaskWidth >>= 1)
+            {
+                CacheMaskShift++;
+            }
+            DBG(1, "Max number of threads sharing L" , level , " " , cacheTypeStr , " cache = " , threadsSharingCache, " [the most significant bit = " , CacheMaskShift , "]",
+                " shift = " , CacheMaskShift);
+            ++it;
+        }
     }
     return true;
 }