[OMON-593] Use smaps_rollup on CS8 and add API to enable each PM measurement (#299)

awegrzyn · web-flow · commit b6f2190ab2f3 · 2022-09-07T15:51:18.000+02:00
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -31,7 +31,7 @@ endif()
 
 # Define project
 project(Monitoring
-  VERSION 3.12.11
+  VERSION 3.13.0
   DESCRIPTION "O2 Monitoring library"
   LANGUAGES CXX
 )
@@ -178,6 +178,20 @@ endif()
 if (UNIX AND NOT APPLE)
   message(STATUS "Detected Linux: Process monitor enabled")
   set(LINUX true)
+  # Detecto CC7 or CS8
+  if(EXISTS "/etc/os-release")
+    file(STRINGS "/etc/os-release" OS_RELEASE)
+    foreach(KV ${OS_RELEASE})
+      if(KV MATCHES "^VERSION=\"8\"")
+        set(CS8 true)
+        message(STATUS "  Detected CS8")
+      endif()
+      if(KV MATCHES "^VERSION=\"7")
+        set(CC7 true)
+        message(STATUS "  Detected CC7")
+      endif()
+    endforeach()
+  endif()
 endif()
 
 if (APPLE)
@@ -193,6 +207,8 @@ target_compile_definitions(Monitoring
   PRIVATE
     $<$<BOOL:${APPLE}>:O2_MONITORING_OS_MAC>
     $<$<BOOL:${LINUX}>:O2_MONITORING_OS_LINUX>
+    $<$<BOOL:${CC7}>:O2_MONITORING_OS_CC7>
+    $<$<BOOL:${CS8}>:O2_MONITORING_OS_CS8>
     $<$<BOOL:${ApMon_FOUND}>:O2_MONITORING_WITH_APPMON>
     $<$<BOOL:${RdKafka_FOUND}>:O2_MONITORING_WITH_KAFKA>
     $<$<BOOL:${CURL_FOUND}>:O2_MONITORING_WITH_CURL>
diff --git a/README.md b/README.md
@@ -151,22 +151,32 @@ See how it works in the example: [examples/4-RateDerivedMetric.cxx](examples/4-R
 
 ### Process monitoring
 
-This feature provides basic performance status of the process. Note that is runs in separate thread (without mutex).
+This feature provides basic performance status of the process. Note that is runs in separate thread.
 
 ```cpp
-enableProcessMonitoring([interval in seconds]);
+enableProcessMonitoring([interval in seconds, {Measurement list}]);
 ```
+List of valid measurement lists:
+- `Monitor::Cpu`
+- `Monitor::Mem`
+- `Monitor::Smaps` - Beware. Enabling this will trigger kernel to run `smaps_account` periodically.
+
 Following metrics are generated every time interval:
-CPU measurements:
+`Monitor::Cpu`:
  + **cpuUsedPercentage** - percentage of a core usage (kernel + user mode) over time interval
  + **involuntaryContextSwitches** - involuntary context switches over time interval
  + **cpuUsedAbsolute** - amount of time spent on process execution (in user and kernel mode) over time interval (expressed in microseconds)
 
-Memory measurements: (Linux only)
+`Monitor::Mem`: (Linux only)
  + **memoryUsagePercentage** - ratio of the process's virtual memory to memory available on the machine
  + **virtualMemorySize** - virtual memory reserved by process (expressed in kB)
  + **residentSetSize** - resident set size reserved by process (expressed in kB)
 
+`Monitor::Smaps`: (Linux only)
++ **proportionalSetSize** - count of pages it has in memory, where each page is divided by the number of processes sharing it
++ **memoryPrivateClean** - unmodified private pages
++ **memoryPrivateDirty** - modified private pages
+
 Additional metrics are generated at the end of process execution:
 CPU measurements:
  + **cpuTimeConsumedByProcess** - total amount of time spent on process execution (in user and kernel mode) (expressed in microseconds)
diff --git a/examples/5-Benchmark.cxx b/examples/5-Benchmark.cxx
@@ -67,7 +67,7 @@ int main(int argc, char* argv[])
 
   auto monitoring = MonitoringFactory::Get(vm["url"].as<std::string>());
   if (vm["monitor"].as<bool>()) {
-    monitoring->enableProcessMonitoring(1);
+    monitoring->enableProcessMonitoring(1, {Monitor::Cpu, Monitor::Mem, Monitor::Smaps});
   }
   if (vm["multiple"].as<bool>()) {
     for (int j = 1; j <= count; j++) {
diff --git a/include/Monitoring/Monitoring.h b/include/Monitoring/Monitoring.h
@@ -70,7 +70,8 @@ class Monitoring
 
   /// Enables process monitoring
   /// \param interval		refresh interval
-  void enableProcessMonitoring(const unsigned int interval = 5);
+  /// \param enabledMeasurements vector of monitor measurements, eg. Monitor::Cpu
+  void enableProcessMonitoring(const unsigned int interval = 5, std::vector<Monitor> enabledMeasurements = {Monitor::Cpu, Monitor::Mem});
 
   /// Flushes metric buffer (this can also happen when buffer is full)
   void flushBuffer();
diff --git a/include/Monitoring/ProcessMonitor.h b/include/Monitoring/ProcessMonitor.h
@@ -31,6 +31,12 @@ namespace o2
 namespace monitoring
 {
 
+enum class Monitor : short {
+  Cpu,
+  Mem,
+  Smaps
+};
+
 /// Monitors current process and/or other processes running at the same machien
 class ProcessMonitor
 {
@@ -58,22 +64,27 @@ class ProcessMonitor
   static std::vector<std::string> getAvailableMetricsNames();
   std::vector<Metric> getPerformanceMetrics();
 
- public:
-  /// Prepares externam software commands (ps)
+  /// Sets PID and total memory
   ProcessMonitor();
 
   /// Default destructor
   ~ProcessMonitor() = default;
 
+  /// Set initial variables for CPU usage calculations
   void init();
 
+  /// Enable given measurement
+  void enable(Monitor measurement);
+
  private:
+  /// States which measurements are enabled
+  std::array<bool, 3> mEnabledMeasurements;
+
   double splitStatusLineAndRetriveValue(const std::string& line) const;
 
   /// Retrievs total memory size from /proc/meminfo
   void setTotalMemory();
 
- private:
   static constexpr const char* metricsNames[] = {"memoryUsagePercentage", "virtualMemorySize", "residentSetSize",
                                                  "cpuUsedPercentage", "involuntaryContextSwitches", "voluntaryContextSwitches", "cpuUsedAbsolute",
                                                  "averageResidentSetSize", "averageVirtualMemorySize", "averageCpuUsedPercentage",
diff --git a/src/Monitoring.cxx b/src/Monitoring.cxx
@@ -40,7 +40,6 @@ Monitoring::Monitoring()
   mDerivedHandler = std::make_unique<DerivedMetrics>();
   mBuffering = false;
   mProcessMonitoringInterval = 0;
-  //mAutoPushInterval = 0;
   mMonitorRunning = false;
 }
 
@@ -83,9 +82,12 @@ void Monitoring::flushBuffer(const short index)
   mStorage[index].clear();
 }
 
-void Monitoring::enableProcessMonitoring(const unsigned int interval)
+void Monitoring::enableProcessMonitoring(const unsigned int interval, std::vector<Monitor> enabledMeasurements)
 {
   mProcessMonitoringInterval = interval;
+  for (const auto& measurement : enabledMeasurements) {
+    mProcessMonitor->enable(measurement);
+  }
   if (!mMonitorRunning) {
     mProcessMonitor->init();
     mMonitorRunning = true;
diff --git a/src/ProcessMonitor.cxx b/src/ProcessMonitor.cxx
@@ -30,6 +30,12 @@ namespace o2
 namespace monitoring
 {
 
+#ifdef O2_MONITORING_OS_CS8
+static constexpr auto SMAPS_FILE = "/proc/self/smaps_rollup";
+#else
+static constexpr auto SMAPS_FILE = "/proc/self/smaps";
+#endif
+
 ProcessMonitor::ProcessMonitor()
 {
   mPid = static_cast<unsigned int>(::getpid());
@@ -38,6 +44,7 @@ ProcessMonitor::ProcessMonitor()
 #ifdef O2_MONITORING_OS_LINUX
   setTotalMemory();
 #endif
+  mEnabledMeasurements.fill(false);
 }
 
 void ProcessMonitor::init()
@@ -46,6 +53,11 @@ void ProcessMonitor::init()
   getrusage(RUSAGE_SELF, &mPreviousGetrUsage);
 }
 
+void ProcessMonitor::enable(Monitor measurement)
+{
+  mEnabledMeasurements[static_cast<short>(measurement)] = true;
+}
+
 void ProcessMonitor::setTotalMemory()
 {
   std::ifstream memInfo("/proc/meminfo");
@@ -85,7 +97,7 @@ std::vector<Metric> ProcessMonitor::getMemoryUsage()
 
 std::vector<Metric> ProcessMonitor::getSmaps()
 {
-  std::ifstream statusStream("/proc/self/smaps");
+  std::ifstream statusStream(SMAPS_FILE);
   double pssTotal = 0;
   double cleanTotal = 0;
   double dirtyTotal = 0;
@@ -147,41 +159,53 @@ double ProcessMonitor::splitStatusLineAndRetriveValue(const std::string& line) c
 
 std::vector<Metric> ProcessMonitor::getPerformanceMetrics()
 {
-  auto metrics = getCpuAndContexts();
+  std::vector<Metric> metrics;
+  metrics.reserve(12);
+  if (mEnabledMeasurements.at(static_cast<short>(Monitor::Cpu))) {
+    auto cpuMetrics = getCpuAndContexts();
+    std::move(cpuMetrics.begin(), cpuMetrics.end(), std::back_inserter(metrics));
+  }
 #ifdef O2_MONITORING_OS_LINUX
-  auto memoryMetrics = getMemoryUsage();
-  std::move(memoryMetrics.begin(), memoryMetrics.end(), std::back_inserter(metrics));
-  auto smapMetrics = getSmaps();
-  std::move(smapMetrics.begin(), smapMetrics.end(), std::back_inserter(metrics));
+  if (mEnabledMeasurements.at(static_cast<short>(Monitor::Mem))) {
+    auto memoryMetrics = getMemoryUsage();
+    std::move(memoryMetrics.begin(), memoryMetrics.end(), std::back_inserter(metrics));
+  }
+  if (mEnabledMeasurements.at(static_cast<short>(Monitor::Smaps))) {
+    auto smapMetrics = getSmaps();
+    std::move(smapMetrics.begin(), smapMetrics.end(), std::back_inserter(metrics));
+  }
 #endif
   return metrics;
 }
 
 std::vector<Metric> ProcessMonitor::makeLastMeasurementAndGetMetrics()
 {
   std::vector<Metric> metrics;
-  getCpuAndContexts();
 #ifdef O2_MONITORING_OS_LINUX
-  getMemoryUsage();
+  if (mEnabledMeasurements.at(static_cast<short>(Monitor::Mem))) {
+    getMemoryUsage();
 
-  auto avgVmRSS = std::accumulate(mVmRssMeasurements.begin(), mVmRssMeasurements.end(), 0.0) /
-                  mVmRssMeasurements.size();
+    auto avgVmRSS = std::accumulate(mVmRssMeasurements.begin(), mVmRssMeasurements.end(), 0.0) /
+                    mVmRssMeasurements.size();
 
-  metrics.emplace_back(avgVmRSS, metricsNames[AVG_RESIDENT_SET_SIZE]);
+    metrics.emplace_back(avgVmRSS, metricsNames[AVG_RESIDENT_SET_SIZE]);
 
-  auto avgVmSize = std::accumulate(mVmSizeMeasurements.begin(), mVmSizeMeasurements.end(), 0.0) /
-                   mVmSizeMeasurements.size();
-  metrics.emplace_back(avgVmSize, metricsNames[AVG_VIRTUAL_MEMORY_SIZE]);
+    auto avgVmSize = std::accumulate(mVmSizeMeasurements.begin(), mVmSizeMeasurements.end(), 0.0) /
+                     mVmSizeMeasurements.size();
+    metrics.emplace_back(avgVmSize, metricsNames[AVG_VIRTUAL_MEMORY_SIZE]);
+  }
 #endif
+  if (mEnabledMeasurements.at(static_cast<short>(Monitor::Cpu))) {
+    getCpuAndContexts();
 
-  auto avgCpuUsage = std::accumulate(mCpuPerctange.begin(), mCpuPerctange.end(), 0.0) /
-                     mCpuPerctange.size();
-  uint64_t accumulationOfCpuTimeConsumption = std::accumulate(mCpuMicroSeconds.begin(),
-                                                              mCpuMicroSeconds.end(), 0UL);
-
-  metrics.emplace_back(avgCpuUsage, metricsNames[AVG_CPU_USED_PERCENTAGE]);
-  metrics.emplace_back(accumulationOfCpuTimeConsumption, metricsNames[ACCUMULATED_CPU_TIME]);
+    auto avgCpuUsage = std::accumulate(mCpuPerctange.begin(), mCpuPerctange.end(), 0.0) /
+                       mCpuPerctange.size();
+    uint64_t accumulationOfCpuTimeConsumption = std::accumulate(mCpuMicroSeconds.begin(),
+                                                                mCpuMicroSeconds.end(), 0UL);
 
+    metrics.emplace_back(avgCpuUsage, metricsNames[AVG_CPU_USED_PERCENTAGE]);
+    metrics.emplace_back(accumulationOfCpuTimeConsumption, metricsNames[ACCUMULATED_CPU_TIME]);
+  }
   return metrics;
 }
 
diff --git a/test/testProcessMonitor.cxx b/test/testProcessMonitor.cxx
@@ -38,15 +38,58 @@ void disableRedirect()
 }
 
 
-BOOST_AUTO_TEST_CASE(monitorProcess)
+
+BOOST_AUTO_TEST_CASE(monitorProcessDefaultCount)
+{
+  {
+    auto monitoring = o2::monitoring::MonitoringFactory::Get("influxdb-stdout://");
+    monitoring->enableProcessMonitoring(1);
+    enableRedirect();
+    std::this_thread::sleep_for(std::chrono::milliseconds(200));
+  }
+
+  std::istringstream returned(coutRedirect.str());
+  disableRedirect();
+  unsigned short int countMetrics = 0;
+  for (std::string line; std::getline(returned, line); ) {
+    countMetrics++;
+  }
+  // On linux 11 (without Smaps) and macOS 6
+  BOOST_CHECK(countMetrics == 11 || countMetrics == 6);
+}
+
+
+BOOST_AUTO_TEST_CASE(monitorProcessCpuOnly)
+{
+  std::array<std::string, 6> names = {"cpuUsedPercentage", "involuntaryContextSwitches", "voluntaryContextSwitches", "cpuUsedAbsolute",
+                                      "averageCpuUsedPercentage", "cpuTimeConsumedByProcess"};
+  {
+    auto monitoring = o2::monitoring::MonitoringFactory::Get("influxdb-stdout://");
+    monitoring->enableProcessMonitoring(1, {Monitor::Cpu});
+    enableRedirect();
+    std::this_thread::sleep_for(std::chrono::milliseconds(200));
+  }
+
+  std::istringstream returned(coutRedirect.str());
+  disableRedirect();
+  unsigned short int countMetrics = 0;
+  for (std::string line; std::getline(returned, line); ) {
+    BOOST_CHECK(std::find(names.begin(), names.end(), line.substr(0, line.find(','))) != names.end());
+    countMetrics++;
+  }
+  // On linux and macOS 6
+  BOOST_CHECK_EQUAL(countMetrics, 6);
+}
+
+BOOST_AUTO_TEST_CASE(monitorProcessAll)
 {
   std::array<std::string, 14> names = {"memoryUsagePercentage", "virtualMemorySize", "residentSetSize",
                                        "cpuUsedPercentage", "involuntaryContextSwitches", "voluntaryContextSwitches", "cpuUsedAbsolute",
                                        "averageResidentSetSize", "averageVirtualMemorySize", "averageCpuUsedPercentage",
                                        "cpuTimeConsumedByProcess", "proportionalSetSize", "memoryPrivateClean", "memoryPrivateDirty"};
   {
     auto monitoring = o2::monitoring::MonitoringFactory::Get("influxdb-stdout://");
-    monitoring->enableProcessMonitoring(1);
+    monitoring->enableProcessMonitoring(1, {Monitor::Cpu, Monitor::Smaps, Monitor::Mem});
     enableRedirect();
     std::this_thread::sleep_for(std::chrono::milliseconds(200));
   }

Original file line number	Diff line number	Diff line change
`@@ -67,7 +67,7 @@ int main(int argc, char* argv[])`
`67`	`67`
`68`	`68`	`auto monitoring = MonitoringFactory::Get(vm["url"].as<std::string>());`
`69`	`69`	`if (vm["monitor"].as<bool>()) {`
`70`		`- monitoring->enableProcessMonitoring(1);`
	`70`	`+ monitoring->enableProcessMonitoring(1, {Monitor::Cpu, Monitor::Mem, Monitor::Smaps});`
`71`	`71`	`}`
`72`	`72`	`if (vm["multiple"].as<bool>()) {`
`73`	`73`	`for (int j = 1; j <= count; j++) {`
Original file line number	Diff line number	Diff line change
`@@ -40,7 +40,6 @@ Monitoring::Monitoring()`
`40`	`40`	`mDerivedHandler = std::make_unique<DerivedMetrics>();`
`41`	`41`	`mBuffering = false;`
`42`	`42`	`mProcessMonitoringInterval = 0;`
`43`		`- //mAutoPushInterval = 0;`
`44`	`43`	`mMonitorRunning = false;`
`45`	`44`	`}`
`46`	`45`
`@@ -83,9 +82,12 @@ void Monitoring::flushBuffer(const short index)`
`83`	`82`	`mStorage[index].clear();`
`84`	`83`	`}`
`85`	`84`
`86`		`-void Monitoring::enableProcessMonitoring(const unsigned int interval)`
	`85`	`+void Monitoring::enableProcessMonitoring(const unsigned int interval, std::vector<Monitor> enabledMeasurements)`
`87`	`86`	`{`
`88`	`87`	`mProcessMonitoringInterval = interval;`
	`88`	`+ for (const auto& measurement : enabledMeasurements) {`
	`89`	`+ mProcessMonitor->enable(measurement);`
	`90`	`+ }`
`89`	`91`	`if (!mMonitorRunning) {`
`90`	`92`	`mProcessMonitor->init();`
`91`	`93`	`mMonitorRunning = true;`