implement PCM_KEEP_NMI_WATCHDOG mode

Change-Id: Icb88d7ab2a62619913ae49d25a27a4e1ffbcbad2

Author: rdementi

doc/ENVVAR_README.md

@@ -7,3 +7,5 @@
 `PCM_USE_RESCTRL=1` : use Linux resctrl driver for RDT metrics
 
 `PCM_PRINT_TOPOLOGY=1` : print detailed CPU topology
+
+`PCM_KEEP_NMI_WATCHDOG=1` : don't disable NMI watchdog (reducing the core metrics set)

src/cpucounters.cpp

@@ -352,6 +352,11 @@ void pcm_cpuid(const unsigned leaf, const unsigned subleaf, PCM_CPUID_INFO & inf
     #endif
 }
 
+#ifdef __linux__
+bool isNMIWatchdogEnabled(const bool silent);
+bool keepNMIWatchdogEnabled();
+#endif
+
 void PCM::readCoreCounterConfig(const bool complainAboutMSR)
 {
     if (max_cpuid >= 0xa)
@@ -412,6 +417,11 @@ void PCM::readCoreCounterConfig(const bool complainAboutMSR)
             std::cerr << "INFO: Reducing the number of programmable counters to 3 to workaround the fixed cycle counter virtualization issue on AWS.\n";
             std::cerr << "      You can disable the workaround by setting PCM_NO_AWS_WORKAROUND=1 environment variable\n";
         }
+        if (isNMIWatchdogEnabled(true) && keepNMIWatchdogEnabled())
+        {
+            --core_gen_counter_num_max;
+            std::cerr << "INFO: Reducing the number of programmable counters to " << core_gen_counter_num_max  << " because NMI watchdog is enabled.\n";
+        }
 #endif
     }
 }
@@ -1980,6 +1990,18 @@ void PCM::initUncorePMUsPerf()
 
 #ifdef __linux__
 
+const char * keepNMIWatchdogEnabledEnvStr = "PCM_KEEP_NMI_WATCHDOG";
+
+bool keepNMIWatchdogEnabled()
+{
+    static int keep = -1;
+    if (keep < 0)
+    {
+        keep = (safe_getenv(keepNMIWatchdogEnabledEnvStr) == std::string("1")) ? 1 : 0;
+    }
+    return keep == 1;
+}
+
 #define PCM_NMI_WATCHDOG_PATH "/proc/sys/kernel/nmi_watchdog"
 
 bool isNMIWatchdogEnabled(const bool silent)
@@ -1995,7 +2017,11 @@ bool isNMIWatchdogEnabled(const bool silent)
 
 void disableNMIWatchdog(const bool silent)
 {
-    if (!silent) std::cerr << "Disabling NMI watchdog since it consumes one hw-PMU counter.\n";
+    if (!silent)
+    {
+        std::cerr << "Disabling NMI watchdog since it consumes one hw-PMU counter. To keep NMU watchdog set environment variable "
+                  << keepNMIWatchdogEnabledEnvStr << "=1 (this reduces the core metrics set)\n";
+    }
     writeSysFS(PCM_NMI_WATCHDOG_PATH, "0");
 }
 
@@ -2423,7 +2449,7 @@ perf_event_attr PCM_init_perf_event_attr(bool group = true)
 PCM::ErrorCode PCM::program(const PCM::ProgramMode mode_, const void * parameter_, const bool silent, const int pid)
 {
 #ifdef __linux__
-    if (isNMIWatchdogEnabled(silent))
+    if (isNMIWatchdogEnabled(silent) && (keepNMIWatchdogEnabled() == false))
     {
         disableNMIWatchdog(silent);
         needToRestoreNMIWatchdog = true;
@@ -2725,6 +2751,13 @@ PCM::ErrorCode PCM::program(const PCM::ProgramMode mode_, const void * parameter
         std::cerr << "PCM ERROR: pid monitoring is only supported with Linux perf_event driver\n";
         return PCM::UnknownError;
     }
+#ifdef __linux__
+    if (isNMIWatchdogEnabled(silent) && (canUsePerf == false))
+    {
+        std::cerr << "PCM ERROR: Unsupported mode. NMI watchdog is enabled and Linux perf_event driver is not used\n";
+        return PCM::UnknownError;
+    }
+#endif
 
     std::vector<int> tids{};
     #ifdef PCM_USE_PERF

tests/test.sh

@@ -4,6 +4,9 @@ export BIN_DIR="build/bin"
 
 pushd $BIN_DIR
 
+# enable NMI watchdog
+echo 1 > /proc/sys/kernel/nmi_watchdog
+
 PCM_NO_PERF=1 ./pcm -r -- sleep 1
 if [ "$?" -ne "0" ]; then
    echo "Error in pcm"
@@ -32,6 +35,12 @@ if [ "$?" -ne "0" ]; then
 fi
 kill $test_pid
 
+PCM_KEEP_NMI_WATCHDOG=1 ./pcm -r -- sleep 1
+if [ "$?" -ne "0" ]; then
+   echo "Error in pcm"
+   exit 1
+fi
+
 ./pcm -r 0.1 -csv=pcm.csv -- sleep 5
 if [ "$?" -ne "0" ]; then
    echo "Error in pcm"