monitor all threads of the process

Author: rdementi

src/cpucounters.cpp

@@ -66,6 +66,7 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 #include <sys/time.h>
 #ifdef __linux__
 #include <sys/mman.h>
+#include <dirent.h>
 #endif
 #endif
 
@@ -426,6 +427,12 @@ bool PCM::isFixedCounterSupported(unsigned c)
 
 bool PCM::isHWTMAL1Supported() const
 {
+    #ifdef PCM_USE_PERF
+    if (perfEventTaskHandle.empty() == false)
+    {
+       return false; // per PID/task perf collection does not support HW TMA L1
+    }
+    #endif
     static int supported = -1;
     if (supported < 0)
     {
@@ -2144,8 +2151,7 @@ PCM::PCM() :
 
 #ifdef PCM_USE_PERF
     canUsePerf = true;
-    std::vector<int> dummy(PERF_MAX_COUNTERS, -1);
-    perfEventHandle.resize(num_cores, dummy);
+    perfEventHandle.resize(num_cores, std::vector<int>(PERF_MAX_COUNTERS, -1));
 #endif
 
     for (int32 i = 0; i < num_cores; ++i)
@@ -2442,7 +2448,7 @@ PCM::ErrorCode PCM::program(const PCM::ProgramMode mode_, const void * parameter
             if (!silent) std::cerr << "Can not use Linux perf because OffcoreResponse usage is not supported. Falling-back to direct PMU programming.\n";
         }
     }
-    if (isHWTMAL1Supported() == true && perfSupportsTopDown() == false)
+    if (isHWTMAL1Supported() == true && perfSupportsTopDown() == false && pid == -1)
     {
         canUsePerf = false;
         if (!silent) std::cerr << "Installed Linux kernel perf does not support hardware top-down level-1 counters. Using direct PMU programming instead.\n";
@@ -2695,6 +2701,45 @@ PCM::ErrorCode PCM::program(const PCM::ProgramMode mode_, const void * parameter
         return PCM::UnknownError;
     }
 
+    std::vector<int> tids{};
+    #ifdef PCM_USE_PERF
+    if (pid != -1)
+    {
+        const auto strDir = std::string("/proc/") +  std::to_string(pid) + "/task/";
+        DIR * tidDir = opendir(strDir.c_str());
+        if (tidDir)
+        {
+            struct dirent * entry{nullptr};
+            while ((entry = readdir(tidDir)) != nullptr)
+            {
+                assert(entry->d_name);
+                const auto tid = atoi(entry->d_name);
+                if (tid)
+                {
+                    tids.push_back(tid);
+                    // std::cerr << "Detected task " << tids.back() << "\n";
+                }
+            }
+            closedir(tidDir);
+        }
+        else
+        {
+            std::cerr << "ERROR: Can't open " << strDir << "\n";
+            return PCM::UnknownError;
+        }
+    }
+    if (tids.empty() == false)
+    {
+        if (isHWTMAL1Supported())
+        {
+            if (!silent) std::cerr << "INFO: TMA L1 metrics are not supported in PID collection mode\n";
+        }
+        if (!silent) std::cerr << "INFO: collecting core metrics for " << tids.size() << " threads in process " << pid << "\n";
+        PerfEventHandleContainer _1(num_cores, std::vector<int>(PERF_MAX_COUNTERS, -1));
+        perfEventTaskHandle.resize(tids.size(), _1);
+    }
+    #endif
+
     programmed_pmu = true;
 
     lastProgrammedCustomCounters.clear();
@@ -2708,11 +2753,11 @@ PCM::ErrorCode PCM::program(const PCM::ProgramMode mode_, const void * parameter
     {
         if (isCoreOnline(i) == false) continue;
 
-        std::packaged_task<void()> task([this, i, mode_, pExtDesc, &programmingStatuses, &pid]() -> void
+        std::packaged_task<void()> task([this, i, mode_, pExtDesc, &programmingStatuses, &tids]() -> void
             {
                 TemporalThreadAffinity tempThreadAffinity(i, false); // speedup trick for Linux
 
-                programmingStatuses[i] = programCoreCounters(i, mode_, pExtDesc, lastProgrammedCustomCounters[i], pid);
+                programmingStatuses[i] = programCoreCounters(i, mode_, pExtDesc, lastProgrammedCustomCounters[i], tids);
             });
         asyncCoreResults.push_back(task.get_future());
         coreTaskQueues[i]->push(task);
@@ -2813,9 +2858,9 @@ PCM::ErrorCode PCM::programCoreCounters(const int i /* core */,
     const PCM::ProgramMode mode_,
     const ExtendedCustomCoreEventDescription * pExtDesc,
     std::vector<EventSelectRegister> & result,
-    const int pid)
+    const std::vector<int> & tids)
 {
-    (void) pid; // to silence uused param warning on non Linux OS
+    (void) tids; // to silence uused param warning on non Linux OS
     // program core counters
 
     result.clear();
@@ -2847,27 +2892,51 @@ PCM::ErrorCode PCM::programCoreCounters(const int i /* core */,
     };
 #ifdef PCM_USE_PERF
     int leader_counter = -1;
-    auto programPerfEvent = [this, &leader_counter, &i, &pid](perf_event_attr & e, const int eventPos, const std::string & eventName) -> bool
-    {
-        // if (i == 0) std::cerr << "DEBUG: programming event "<< std::hex << e.config << std::dec << "\n";
-        if ((perfEventHandle[i][eventPos] = syscall(SYS_perf_event_open, &e, pid,
-            i /* core id */, leader_counter /* group leader */, 0)) <= 0)
-        {
-            std::lock_guard<std::mutex> _(printErrorMutex);
-            std::cerr << "Linux Perf: Error when programming " << eventName << ", error: " << strerror(errno) <<
-               " with config 0x" << std::hex << e.config <<
-               " config1 0x" << e.config1 << std::dec << "\n";
-            if (24 == errno)
+    auto programPerfEvent = [this, &leader_counter, &i, &tids](perf_event_attr e, const int eventPos, const std::string & eventName) -> bool
+    {
+        auto programPerfEventHelper = [&i]( PerfEventHandleContainer & perfEventHandle,
+                                            perf_event_attr & e,
+                                            const int eventPos,
+                                            const std::string & eventName,
+                                            const int leader_counter,
+                                            const int tid) -> bool
+        {
+            // if (i == 0) std::cerr << "DEBUG: programming event "<< std::hex << e.config << std::dec << "\n";
+            if ((perfEventHandle[i][eventPos] = syscall(SYS_perf_event_open, &e, tid,
+                i /* core id */, leader_counter /* group leader */, 0)) <= 0)
             {
-                std::cerr << "try executing 'ulimit -n 20000' to increase the limit on the number of open files.\n";
+                std::lock_guard<std::mutex> _(printErrorMutex);
+                std::cerr << "Linux Perf: Error when programming " << eventName << ", error: " << strerror(errno) <<
+                " with config 0x" << std::hex << e.config <<
+                " config1 0x" << e.config1 << std::dec << " for tid " << tid << " leader " << leader_counter << "\n";
+                if (24 == errno)
+                {
+                    std::cerr << PCM_ULIMIT_RECOMMENDATION;
+                }
+                else
+                {
+                    std::cerr << "try running with environment variable PCM_NO_PERF=1\n";
+                }
+                return false;
             }
-            else
+            return true;
+        };
+        if (tids.empty() == false)
+        {
+            e.inherit = 1;
+            e.read_format = 0; // 'inherit' does not work for combinations of read format (e.g. PERF_FORMAT_GROUP)
+            auto handleIt = perfEventTaskHandle.begin();
+            for (const auto & tid: tids)
             {
-                std::cerr << "try running with environment variable PCM_NO_PERF=1\n";
+                if (programPerfEventHelper(*handleIt, e, eventPos, eventName, -1, tid) == false)
+                {
+                    return false;
+                }
+                ++handleIt;
             }
-            return false;
+            return true;
         }
-        return true;
+        return programPerfEventHelper(perfEventHandle, e, eventPos, eventName, leader_counter, -1);
     };
     if (canUsePerf)
     {
@@ -3736,16 +3805,27 @@ void PCM::cleanupPMU(const bool silent)
 #ifdef PCM_USE_PERF
     if (canUsePerf)
     {
-      for (int i = 0; i < num_cores; ++i)
-        for(int c = 0; c < PERF_MAX_COUNTERS; ++c)
+        auto cleanOne = [this](PerfEventHandleContainer & cont)
+        {
+            for (int i = 0; i < num_cores; ++i)
+            {
+                for(int c = 0; c < PERF_MAX_COUNTERS; ++c)
+                {
+                    auto & h = cont[i][c];
+                    if (h != -1) ::close(h);
+                    h = -1;
+                }
+            }
+        };
+        cleanOne(perfEventHandle);
+        for (auto & cont : perfEventTaskHandle)
         {
-            auto & h = perfEventHandle[i][c];
-            if (h != -1) ::close(h);
-            h = -1;
+            cleanOne(cont);
         }
+        perfEventTaskHandle.clear();
 
-      if (!silent) std::cerr << " Closed perf event handles\n";
-      return;
+        if (!silent) std::cerr << " Closed perf event handles\n";
+        return;
     }
 #endif
 
@@ -4055,6 +4135,31 @@ CoreCounterState getCoreCounterState(uint32 core)
 #ifdef PCM_USE_PERF
 void PCM::readPerfData(uint32 core, std::vector<uint64> & outData)
 {
+    if (perfEventTaskHandle.empty() == false)
+    {
+        std::fill(outData.begin(), outData.end(), 0);
+        for (const auto & handleArray : perfEventTaskHandle)
+        {
+            for (size_t ctr = 0; ctr < PERF_MAX_COUNTERS; ++ctr)
+            {
+                const int fd = handleArray[core][ctr];
+                if (fd != -1)
+                {
+                    uint64 result{0ULL};
+                    const int status = ::read(fd, &result, sizeof(result));
+                    if (status != sizeof(result))
+                    {
+                        std::cerr << "PCM Error: failed to read from Linux perf handle " << fd <<  "\n";
+                    }
+                    else
+                    {
+                        outData[ctr] += result;
+                    }
+                }
+            }
+        }
+        return;
+    }
     auto readPerfDataHelper = [this](const uint32 core, std::vector<uint64>& outData, const uint32 leader, const uint32 num_counters)
     {
         if (perfEventHandle[core][leader] < 0)
@@ -6330,7 +6435,7 @@ class PerfVirtualControlRegister : public HWRegister
         {
             std::cerr << "Linux Perf: Error on programming PMU " << pmuID << ":  " << strerror(errno) << "\n";
             std::cerr << "config: 0x" << std::hex << event.config << " config1: 0x" << event.config1 << " config2: 0x" << event.config2 << std::dec << "\n";
-            if (errno == 24) std::cerr << "try executing 'ulimit -n 20000' to increase the limit on the number of open files.\n";
+            if (errno == 24) std::cerr << PCM_ULIMIT_RECOMMENDATION;
             return;
         }
     }

src/cpucounters.h

@@ -859,7 +859,9 @@ class PCM_API PCM
 
     bool canUsePerf;
 #ifdef PCM_USE_PERF
-    std::vector<std::vector<int> > perfEventHandle;
+    typedef std::vector<std::vector<int> > PerfEventHandleContainer;
+    PerfEventHandleContainer perfEventHandle;
+    std::vector<PerfEventHandleContainer> perfEventTaskHandle;
     void readPerfData(uint32 core, std::vector<uint64> & data);
 
     enum {
@@ -894,7 +896,7 @@ class PCM_API PCM
     std::vector<std::vector<EventSelectRegister> > lastProgrammedCustomCounters;
     uint32 checkCustomCoreProgramming(std::shared_ptr<SafeMsrHandle> msr);
     ErrorCode programCoreCounters(int core, const PCM::ProgramMode mode, const ExtendedCustomCoreEventDescription * pExtDesc,
-        std::vector<EventSelectRegister> & programmedCustomCounters, const int pid);
+        std::vector<EventSelectRegister> & programmedCustomCounters, const std::vector<int> & tids);
 
     bool PMUinUse();
     void cleanupPMU(const bool silent = false);

src/pci.cpp

@@ -384,7 +384,7 @@ int openHandle(uint32 groupnr_, uint32 bus, uint32 device, uint32 function)
     int handle = ::open(path.str().c_str(), O_RDWR);
     if (handle < 0)
     {
-       if (errno == 24) std::cerr << "ERROR: try executing 'ulimit -n 20000' to increase the limit on the number of open files.\n";
+       if (errno == 24) std::cerr << "ERROR: " << PCM_ULIMIT_RECOMMENDATION;
        handle = ::open((std::string("/pcm") + path.str()).c_str(), O_RDWR);
     }
     return handle;

src/resctrl.cpp

@@ -123,7 +123,7 @@ namespace pcm
                 std::cerr << "Error reading " << f << ". Error: " << strerror(errno) << "\n";
                 if (errno == 24)
                 {
-                    std::cerr << "try executing 'ulimit -n 20000' to increase the limit on the number of open files.\n";
+                    std::cerr << PCM_ULIMIT_RECOMMENDATION;
                 }
             }
         }

src/types.h

@@ -40,6 +40,7 @@ typedef signed long long int64;
 typedef unsigned int uint32;
 typedef signed int int32;
 
+#define PCM_ULIMIT_RECOMMENDATION ("try executing 'ulimit -n 1000000' to increase the limit on the number of open files.\n")
 
 /*
         MSR addreses from