Extend process monitoring (#204)

Co-authored-by: Kavaldrin <[email protected]>

Author: awegrzyn

README.md

@@ -149,10 +149,25 @@ This feature provides basic performance status of the process. Note that is runs
 ```cpp
 enableProcessMonitoring([interval in seconds]);
 ```
-The `processPerformance` metric is generated every interval with following values:
-  + **cpu_used_pct** - percentage of a core usage over time interval
-  + **involuntary_context_switches** - involuntary context switches over time interval
-  + **memory_used_pct** - ratio of the process's resident set size  to the physical memory on the machine, expressed as a percentage (Linux only)
+Following metrics are generated every time interval:
+CPU measurements:
+ + **cpuUsedPercentage** - percentage of a core usage (kernel + user mode) over time interval
+ + **involuntaryContextSwitches** - involuntary context switches over time interval
+ + **cpuUsedAbsolute** - amount of time spent on process execution (in user and kernel mode) over time interval (expressed in microseconds)
+
+Memory measurements: (Linux only)
+ + **memoryUsagePercentage** - ratio of the process's virtual memory to memory available on the machine
+ + **virtualMemorySize** - virtual memory reserved by process (expressed in kB)
+ + **residentSetSize** - resident set size reserved by process (expressed in kB)
+
+Additional metrics are generated at the end of process execution:
+CPU measurements:
+ + **cpuTimeConsumedByProcess** - total amount of time spent on process execution (in user and kernel mode) (expressed in microseconds)
+ + **averageCpuUsedPercentage** - average percentage of a core usage over time interval
+
+Memory measurements: (Linux only)
+ + **averageResidentSetSize** - average resident set size used by process (expressed in kB)
+ + **averageVirtualMemorySize** - average virtual memory used by process (expressed in kB)
 
 ### StdOut backend output format
 ```

include/Monitoring/ProcessMonitor.h

@@ -36,37 +36,73 @@ class ProcessMonitor
 {
   friend class Monitoring;
 
+ public:
+  enum {
+    MEMORY_USAGE_PERCENTAGE = 0,
+    VIRTUAL_MEMORY_SIZE,
+    RESIDENT_SET_SIZE,
+    CPU_USED_PERCENTAGE,
+    INVOLUNTARY_CONTEXT_SWITCHING,
+    CPU_USED_ABSOLUTE,
+    AVG_RESIDENT_SET_SIZE,
+    AVG_VIRTUAL_MEMORY_SIZE,
+    AVG_CPU_USED_PERCENTAGE,
+    ACCUMULATED_CPU_TIME,
+    AVAILABLE_METRICS_SIZE
+  };
+
+  static std::vector<std::string> getAvailableMetricsNames();
+  std::vector<Metric> getPerformanceMetrics();
+
  public:
   /// Prepares externam software commands (ps)
   ProcessMonitor();
 
   /// Default destructor
   ~ProcessMonitor() = default;
 
-  /// Return performance metrics
-  Metric getPerformanceMetrics();
+  void init();
+
+ private:
+  double splitStatusLineAndRetriveValue(const std::string& line) const;
+
+  /// Retrievs total memory size from /proc/meminfo
+  void setTotalMemory();
 
  private:
+  static constexpr const char* metricsNames[] = {"memoryUsagePercentage", "virtualMemorySize", "residentSetSize",
+                                                 "cpuUsedPercentage", "involuntaryContextSwitches", "cpuUsedAbsolute",
+                                                 "averageResidentSetSize", "averageVirtualMemorySize", "averageCpuUsedPercentage",
+                                                 "cpuTimeConsumedByProcess"};
+
+  static constexpr unsigned int VM_SIZE_INDEX = 18;
+  static constexpr unsigned int VM_RSS_INDEX = 22;
+
   /// PIDs that are monitored
   unsigned int mPid;
 
   /// Total memory size
   unsigned int mTotalMemory;
 
-  /// Retrievs total memory size from /proc/meminfo
-  void setTotalMemory();
-
   /// 'getrusage' values from last execution
   struct rusage mPreviousGetrUsage;
 
+  ///each measurement will be saved to compute average/accumulation usage
+  std::vector<double> mVmSizeMeasurements;
+  std::vector<double> mVmRssMeasurements;
+  std::vector<uint64_t> mCpuMicroSeconds;
+  std::vector<double> mCpuPerctange;
+
   /// Timestamp when process monitoring was executed last time
   std::chrono::high_resolution_clock::time_point mTimeLastRun;
 
-  /// Retrieves memory usage (%)
-  double getMemoryUsage();
+  /// Retrieves virtual memory and resident set size usage
+  std::vector<Metric> getMemoryUsage();
 
   /// Retrieves CPU usage (%) and number of context switches during the interval
-  Metric getCpuAndContexts();
+  std::vector<Metric> getCpuAndContexts();
+
+  std::vector<Metric> makeLastMeasurementAndGetMetrics();
 };
 
 } // namespace monitoring

src/Backends/InfluxDB.cxx

@@ -63,6 +63,9 @@ void InfluxDB::send(std::vector<Metric>&& metrics)
     influxMetrics += toInfluxLineProtocol(metric);
     influxMetrics += "\n";
   }
+  //remove last \n
+  if (influxMetrics.size() > 0) influxMetrics.pop_back();
+
 
   try {
     mTransport->send(std::move(influxMetrics));

src/Monitoring.cxx

@@ -83,6 +83,7 @@ void Monitoring::enableProcessMonitoring(const unsigned int interval)
 {
   mProcessMonitoringInterval = interval;
   if (!mMonitorRunning) {
+    mProcessMonitor->init();
     mMonitorRunning = true;
     mMonitorThread = std::thread(&Monitoring::pushLoop, this);
   }
@@ -121,6 +122,7 @@ Monitoring::~Monitoring()
   mMonitorRunning = false;
   if (mMonitorThread.joinable()) {
     mMonitorThread.join();
+    transmit(mProcessMonitor->makeLastMeasurementAndGetMetrics());
   }
   if (mBuffering) {
     flushBuffer();

src/ProcessMonitor.cxx

@@ -11,6 +11,7 @@
 #include <cmath>
 #include <fstream>
 #include <utility>
+#include <numeric>
 
 namespace o2
 {
@@ -21,68 +22,135 @@ namespace monitoring
 ProcessMonitor::ProcessMonitor()
 {
   mPid = static_cast<unsigned int>(::getpid());
-  getrusage(RUSAGE_SELF, &mPreviousGetrUsage);
   mTimeLastRun = std::chrono::high_resolution_clock::now();
+  getrusage(RUSAGE_SELF, &mPreviousGetrUsage);
 #ifdef O2_MONITORING_OS_LINUX
   setTotalMemory();
 #endif
 }
 
-Metric ProcessMonitor::getPerformanceMetrics()
+void ProcessMonitor::init()
 {
-  auto metric = getCpuAndContexts();
-#ifdef O2_MONITORING_OS_LINUX
-  metric.addValue(getMemoryUsage(), "memory_pct");
-#endif
-  return metric;
+  mTimeLastRun = std::chrono::high_resolution_clock::now();
+  getrusage(RUSAGE_SELF, &mPreviousGetrUsage);
 }
 
 void ProcessMonitor::setTotalMemory()
 {
   std::ifstream memInfo("/proc/meminfo");
   std::string totalString;
   std::getline(memInfo, totalString);
-  std::istringstream iss(totalString);
-  std::vector<std::string> tokens{std::istream_iterator<std::string>{iss},
-                                  std::istream_iterator<std::string>{}};
-  mTotalMemory = std::stoi(tokens[1]);
+  mTotalMemory = splitStatusLineAndRetriveValue(totalString);
 }
 
-double ProcessMonitor::getMemoryUsage()
+std::vector<Metric> ProcessMonitor::getMemoryUsage()
 {
+  std::vector<Metric> metrics;
   std::ifstream statusStream("/proc/self/status");
   std::string rssString;
   rssString.reserve(50);
 
+  // Scan for VmSize
+  for (unsigned i = 0; i < VM_SIZE_INDEX; ++i) {
+    std::getline(statusStream, rssString);
+  }
+  auto vmSize = splitStatusLineAndRetriveValue(rssString);
+  mVmSizeMeasurements.push_back(vmSize);
+
+  metrics.emplace_back((vmSize * 100) / mTotalMemory, metricsNames[MEMORY_USAGE_PERCENTAGE]);
+  metrics.emplace_back(vmSize, metricsNames[VIRTUAL_MEMORY_SIZE]);
+
   // Scan for VmRSS
-  for (int i = 0; i < 18; i++) {
+  for (unsigned i = 0; i < VM_RSS_INDEX - VM_SIZE_INDEX; ++i) {
     std::getline(statusStream, rssString);
   }
-  std::istringstream iss(rssString);
-  std::vector<std::string> tokens{std::istream_iterator<std::string>{iss},
-                                  std::istream_iterator<std::string>{}};
-  return (std::stod(tokens[1]) * 100) / mTotalMemory;
+
+  auto vmRSS = splitStatusLineAndRetriveValue(rssString);
+  metrics.emplace_back(vmRSS, metricsNames[RESIDENT_SET_SIZE]);
+  mVmRssMeasurements.push_back(vmRSS);
+
+  return metrics;
 }
 
-Metric ProcessMonitor::getCpuAndContexts()
+std::vector<Metric> ProcessMonitor::getCpuAndContexts()
 {
+  std::vector<Metric> metrics;
   struct rusage currentUsage;
   getrusage(RUSAGE_SELF, &currentUsage);
   auto timeNow = std::chrono::high_resolution_clock::now();
   double timePassed = std::chrono::duration_cast<std::chrono::microseconds>(timeNow - mTimeLastRun).count();
   if (timePassed < 950) {
     MonLogger::Get() << "[WARN] Do not invoke Process Monitor more frequent then every 1s" << MonLogger::End();
-    return {"processPerformance"};
+    metrics.emplace_back("processPerformance");
+    return metrics;
   }
-  double fractionCpuUsed = (currentUsage.ru_utime.tv_sec * 1000000.0 + currentUsage.ru_utime.tv_usec - (mPreviousGetrUsage.ru_utime.tv_sec * 1000000.0 + mPreviousGetrUsage.ru_utime.tv_usec) + currentUsage.ru_stime.tv_sec * 1000000.0 + currentUsage.ru_stime.tv_usec - (mPreviousGetrUsage.ru_stime.tv_sec * 1000000.0 + mPreviousGetrUsage.ru_stime.tv_usec)) / timePassed;
 
-  Metric metric{"processPerformance"};
-  metric.addValue(static_cast<double>(std::round(fractionCpuUsed * 100.0 * 100.0) / 100.0), "cpu_used_pct");
-  metric.addValue(static_cast<uint64_t>(currentUsage.ru_nivcsw - mPreviousGetrUsage.ru_nivcsw), "involuntary_context_switches");
+  uint64_t cpuUsedInMicroSeconds = currentUsage.ru_utime.tv_sec * 1000000.0 + currentUsage.ru_utime.tv_usec - (mPreviousGetrUsage.ru_utime.tv_sec * 1000000.0 + mPreviousGetrUsage.ru_utime.tv_usec) + currentUsage.ru_stime.tv_sec * 1000000.0 + currentUsage.ru_stime.tv_usec - (mPreviousGetrUsage.ru_stime.tv_sec * 1000000.0 + mPreviousGetrUsage.ru_stime.tv_usec);
+  double fractionCpuUsed = cpuUsedInMicroSeconds / timePassed;
+
+  double cpuUsedPerctange = std::round(fractionCpuUsed * 100.0 * 100.0) / 100.0;
+  mCpuPerctange.push_back(cpuUsedPerctange);
+  mCpuMicroSeconds.push_back(cpuUsedInMicroSeconds);
+
+  metrics.emplace_back(Metric{cpuUsedPerctange, metricsNames[CPU_USED_PERCENTAGE]});
+  metrics.emplace_back(Metric{
+    static_cast<uint64_t>(currentUsage.ru_nivcsw - mPreviousGetrUsage.ru_nivcsw), metricsNames[INVOLUNTARY_CONTEXT_SWITCHING]});
+  metrics.emplace_back(cpuUsedInMicroSeconds, metricsNames[CPU_USED_ABSOLUTE]);
 
   mTimeLastRun = timeNow;
   mPreviousGetrUsage = currentUsage;
-  return metric;
+  return metrics;
+}
+
+double ProcessMonitor::splitStatusLineAndRetriveValue(const std::string& line) const
+{
+  std::istringstream iss(line);
+  std::vector<std::string> tokens{std::istream_iterator<std::string>{iss},
+                                  std::istream_iterator<std::string>{}};
+  return std::stod(tokens[1]);
+}
+
+std::vector<Metric> ProcessMonitor::getPerformanceMetrics()
+{
+  auto metrics = getCpuAndContexts();
+#ifdef O2_MONITORING_OS_LINUX
+  auto memoryMetrics = getMemoryUsage();
+  std::move(memoryMetrics.begin(), memoryMetrics.end(), std::back_inserter(metrics));
+#endif
+  return metrics;
+}
+
+std::vector<Metric> ProcessMonitor::makeLastMeasurementAndGetMetrics()
+{
+  std::vector<Metric> metrics;
+  getCpuAndContexts();
+#ifdef O2_MONITORING_OS_LINUX
+  getMemoryUsage();
+
+  auto avgVmRSS = std::accumulate(mVmRssMeasurements.begin(), mVmRssMeasurements.end(), 0.0) /
+                  mVmRssMeasurements.size();
+
+  metrics.emplace_back(avgVmRSS, metricsNames[AVG_RESIDENT_SET_SIZE]);
+
+  auto avgVmSize = std::accumulate(mVmSizeMeasurements.begin(), mVmSizeMeasurements.end(), 0.0) /
+                   mVmSizeMeasurements.size();
+  metrics.emplace_back(avgVmSize, metricsNames[AVG_VIRTUAL_MEMORY_SIZE]);
+#endif
+
+  auto avgCpuUsage = std::accumulate(mCpuPerctange.begin(), mCpuPerctange.end(), 0.0) /
+                     mCpuPerctange.size();
+  uint64_t accumulationOfCpuTimeConsumption = std::accumulate(mCpuMicroSeconds.begin(),
+                                                              mCpuMicroSeconds.end(), 0UL);
+
+  metrics.emplace_back(avgCpuUsage, metricsNames[AVG_CPU_USED_PERCENTAGE]);
+  metrics.emplace_back(accumulationOfCpuTimeConsumption, metricsNames[ACCUMULATED_CPU_TIME]);
+
+  return metrics;
+}
+
+std::vector<std::string> ProcessMonitor::getAvailableMetricsNames()
+{
+  return {std::begin(metricsNames), std::end(metricsNames)};
 }
 
 } // namespace monitoring

test/testProcessMonitor.cxx

@@ -35,6 +35,13 @@ BOOST_AUTO_TEST_CASE(monitorProcess)
   std::this_thread::sleep_for(std::chrono::milliseconds(2100));
 }
 
+BOOST_AUTO_TEST_CASE(monitorProcessMetricName)
+{
+  auto vec = o2::monitoring::ProcessMonitor::getAvailableMetricsNames();
+  BOOST_CHECK_EQUAL(vec.size(), o2::monitoring::ProcessMonitor::AVAILABLE_METRICS_SIZE);
+  BOOST_CHECK_EQUAL(vec[o2::monitoring::ProcessMonitor::AVG_CPU_USED_PERCENTAGE], "averageCpuUsedPercentage");
+}
+
 } // namespace Test
 } // namespace monitoring
 } // namespace o2