Merge pull request #404 from opcm/push-2022-05-10

Push 2022 05 10

Author: opcm

doc/ENVVAR_README.md

@@ -7,3 +7,5 @@
 `PCM_USE_RESCTRL=1` : use Linux resctrl driver for RDT metrics
 
 `PCM_PRINT_TOPOLOGY=1` : print detailed CPU topology
+
+`PCM_KEEP_NMI_WATCHDOG=1` : don't disable NMI watchdog (reducing the core metrics set)

src/cpucounters.cpp

@@ -352,6 +352,11 @@ void pcm_cpuid(const unsigned leaf, const unsigned subleaf, PCM_CPUID_INFO & inf
     #endif
 }
 
+#ifdef __linux__
+bool isNMIWatchdogEnabled(const bool silent);
+bool keepNMIWatchdogEnabled();
+#endif
+
 void PCM::readCoreCounterConfig(const bool complainAboutMSR)
 {
     if (max_cpuid >= 0xa)
@@ -412,6 +417,11 @@ void PCM::readCoreCounterConfig(const bool complainAboutMSR)
             std::cerr << "INFO: Reducing the number of programmable counters to 3 to workaround the fixed cycle counter virtualization issue on AWS.\n";
             std::cerr << "      You can disable the workaround by setting PCM_NO_AWS_WORKAROUND=1 environment variable\n";
         }
+        if (isNMIWatchdogEnabled(true) && keepNMIWatchdogEnabled())
+        {
+            --core_gen_counter_num_max;
+            std::cerr << "INFO: Reducing the number of programmable counters to " << core_gen_counter_num_max  << " because NMI watchdog is enabled.\n";
+        }
 #endif
     }
 }
@@ -1980,6 +1990,18 @@ void PCM::initUncorePMUsPerf()
 
 #ifdef __linux__
 
+const char * keepNMIWatchdogEnabledEnvStr = "PCM_KEEP_NMI_WATCHDOG";
+
+bool keepNMIWatchdogEnabled()
+{
+    static int keep = -1;
+    if (keep < 0)
+    {
+        keep = (safe_getenv(keepNMIWatchdogEnabledEnvStr) == std::string("1")) ? 1 : 0;
+    }
+    return keep == 1;
+}
+
 #define PCM_NMI_WATCHDOG_PATH "/proc/sys/kernel/nmi_watchdog"
 
 bool isNMIWatchdogEnabled(const bool silent)
@@ -1995,7 +2017,11 @@ bool isNMIWatchdogEnabled(const bool silent)
 
 void disableNMIWatchdog(const bool silent)
 {
-    if (!silent) std::cerr << "Disabling NMI watchdog since it consumes one hw-PMU counter.\n";
+    if (!silent)
+    {
+        std::cerr << " Disabling NMI watchdog since it consumes one hw-PMU counter. To keep NMU watchdog set environment variable "
+                  << keepNMIWatchdogEnabledEnvStr << "=1 (this reduces the core metrics set)\n";
+    }
     writeSysFS(PCM_NMI_WATCHDOG_PATH, "0");
 }
 
@@ -2423,7 +2449,7 @@ perf_event_attr PCM_init_perf_event_attr(bool group = true)
 PCM::ErrorCode PCM::program(const PCM::ProgramMode mode_, const void * parameter_, const bool silent, const int pid)
 {
 #ifdef __linux__
-    if (isNMIWatchdogEnabled(silent))
+    if (isNMIWatchdogEnabled(silent) && (keepNMIWatchdogEnabled() == false))
     {
         disableNMIWatchdog(silent);
         needToRestoreNMIWatchdog = true;
@@ -2590,7 +2616,16 @@ PCM::ErrorCode PCM::program(const PCM::ProgramMode mode_, const void * parameter
                 coreEventDesc[2].umask_value = SKL_MEM_LOAD_RETIRED_L2_MISS_UMASK;
                 coreEventDesc[3].event_number = SKL_MEM_LOAD_RETIRED_L2_HIT_EVTNR;
                 coreEventDesc[3].umask_value = SKL_MEM_LOAD_RETIRED_L2_HIT_UMASK;
-                if (core_gen_counter_num_max == 3)
+                if (core_gen_counter_num_max == 2)
+                {
+                    L3CacheHitRatioAvailable = true;
+                    L3CacheMissesAvailable = true;
+                    L3CacheHitsSnoopAvailable = true;
+                    L3CacheHitsAvailable = true;
+                    core_gen_counter_num_used = 2;
+                    break;
+                }
+                else if (core_gen_counter_num_max == 3)
                 {
                     L3CacheHitRatioAvailable = true;
                     L3CacheMissesAvailable = true;
@@ -2725,6 +2760,13 @@ PCM::ErrorCode PCM::program(const PCM::ProgramMode mode_, const void * parameter
         std::cerr << "PCM ERROR: pid monitoring is only supported with Linux perf_event driver\n";
         return PCM::UnknownError;
     }
+#ifdef __linux__
+    if (isNMIWatchdogEnabled(silent) && (canUsePerf == false))
+    {
+        std::cerr << "PCM ERROR: Unsupported mode. NMI watchdog is enabled and Linux perf_event driver is not used\n";
+        return PCM::UnknownError;
+    }
+#endif
 
     std::vector<int> tids{};
     #ifdef PCM_USE_PERF

tests/test.sh

@@ -4,24 +4,31 @@ export BIN_DIR="build/bin"
 
 pushd $BIN_DIR
 
+echo Enable NMI watchdog
+echo 1 > /proc/sys/kernel/nmi_watchdog
+
+echo Testing pcm with PCM_NO_PERF=1
 PCM_NO_PERF=1 ./pcm -r -- sleep 1
 if [ "$?" -ne "0" ]; then
    echo "Error in pcm"
    exit 1
 fi
 
+echo Testing pcm with PCM_USE_UNCORE_PERF=1
 PCM_USE_UNCORE_PERF=1 ./pcm -r -- sleep 1
 if [ "$?" -ne "0" ]; then
    echo "Error in pcm"
    exit 1
 fi
 
+echo Testing pcm w/o env vars
 ./pcm -r -- sleep 1
 if [ "$?" -ne "0" ]; then
    echo "Error in pcm"
    exit 1
 fi
 
+echo Testing pcm with -pid
 perl -e ' do {} until (0)' &
 test_pid="$!"
 ./pcm -pid $test_pid -- sleep 1
@@ -32,102 +39,126 @@ if [ "$?" -ne "0" ]; then
 fi
 kill $test_pid
 
+echo Testing pcm with PCM_KEEP_NMI_WATCHDOG=1
+PCM_KEEP_NMI_WATCHDOG=1 ./pcm -r -- sleep 1
+if [ "$?" -ne "0" ]; then
+   echo "Error in pcm"
+   exit 1
+fi
+
+echo Testing pcm with -csv
 ./pcm -r 0.1 -csv=pcm.csv -- sleep 5
 if [ "$?" -ne "0" ]; then
    echo "Error in pcm"
    exit 1
 fi
 
+echo Testing pcm-memory
 ./pcm-memory -- sleep 1
 if [ "$?" -ne "0" ]; then
     echo "Error in pcm-memory"
     exit 1
 fi
 
+echo Testing pcm-memory with -rank
 ./pcm-memory -rank=1 -- sleep 1
 if [ "$?" -ne "0" ]; then
     echo "Error in pcm-memory"
     exit 1
 fi
 
+echo Testing pcm-memory with -rank and -csv
 ./pcm-memory -rank=1 -csv -- sleep 1
 if [ "$?" -ne "0" ]; then
     echo "Error in pcm-memory"
     exit 1
 fi
 
+echo Testing pcm-iio
 ./pcm-iio -i=1
 if [ "$?" -ne "0" ]; then
     echo "Error in pcm-iio"
     exit 1
 fi
 
+echo Testing pcm-raw
 ./pcm-raw -e core/config=0x30203,name=LD_BLOCKS.STORE_FORWARD/ -e cha/config=0,name=UNC_CHA_CLOCKTICKS/ -e imc/fixed,name=DRAM_CLOCKS -e thread_msr/config=0x10,config1=1 -e thread_msr/config=0x19c,config1=0 -- sleep 1
 if [ "$?" -ne "0" ]; then
     echo "Error in pcm-raw"
     exit 1
 fi
 
+echo Testing pcm-mmio
 ./pcm-mmio 0x0
 if [ "$?" -ne "0" ]; then
     echo "Error in pcm-mmio"
     exit 1
 fi
 
+echo Testing pcm-pcicfg
 ./pcm-pcicfg 0 0 0 0 0
 if [ "$?" -ne "0" ]; then
     echo "Error in pcm-pcicfg"
     exit 1
 fi
 
+echo Testing pcm-numa
 ./pcm-numa -- sleep 1
 if [ "$?" -ne "0" ]; then
     echo "Error in pcm-numa"
     exit 1
 fi
 
+echo Testing pcm-core
 ./pcm-core -e cpu/umask=0x01,event=0x0e,name=UOPS_ISSUED.STALL_CYCLES/ -- sleep 1
 if [ "$?" -ne "0" ]; then
     echo "Error in pcm-core"
     exit 1
 fi
 
+echo Testing c_example
 ./examples/c_example
 if [ "$?" -ne "0" ]; then
     echo "Error in c_example"
     exit 1
 fi
 
+echo Testing c_example_shlib
 ./examples/c_example_shlib
 if [ "$?" -ne "0" ]; then
     echo "Error in c_example_shlib"
     exit 1
 fi
 
+echo Testing pcm-msr \(read only\)
 ./pcm-msr -a 0x30A
 if [ "$?" -ne "0" ]; then
     echo "Error in pcm-msr"
     exit 1
 fi
 
+echo Testing pcm-power
 ./pcm-power -- sleep 1
 if [ "$?" -ne "0" ]; then
     echo "Error in pcm-power"
     exit 1
 fi
 
+echo Testing pcm-pcie
 ./pcm-pcie -- sleep 1
 if [ "$?" -ne "0" ]; then
     echo "Error in pcm-pcie"
     exit 1
 fi
 
+echo Testing pcm-latency
 ./pcm-latency -i=1
 if [ "$?" -ne "0" ]; then
     echo "Error in pcm-latency"
     exit 1
 fi
 
+echo Testing pcm-tsx
 ./pcm-tsx -- sleep 1
 if [ "$?" -ne "0" ]; then
     echo "Error in pcm-tsx"
@@ -137,15 +168,16 @@ fi
 # TODO add more tests
 # e.g for ./pcm-sensor-server, ./pcm-sensor, ...
 
+echo Testing urltest
 ./tests/urltest
 # We have 2 expected errors, anything else is a bug
 if [ "$?" != 2 ]; then
     echo "Error in urltest, 2 expected errors but found $?!"
     exit 1
 fi
 
-### Check pcm-raw with event files
-# Download necessary files
+echo Testing pcm-raw with event files
+echo   Download necessary files
 if [ ! -f "mapfile.csv" ]; then
     echo "Downloading https://download.01.org/perfmon/mapfile.csv"
     wget -q --timeout=10 https://download.01.org/perfmon/mapfile.csv
@@ -193,15 +225,15 @@ do
     fi
 done
 
-# Now check pcm-raw with JSON files from mapFile.csv
+echo   Now check pcm-raw with JSON files from mapFile.csv
 ./pcm-raw -r -e LD_BLOCKS.STORE_FORWARD -e CPU_CLK_UNHALTED.THREAD_ANY -e INST_RETIRED.ANY -e UNC_CHA_CLOCKTICKS -- sleep 1
 
 if [ "$?" -ne "0" ]; then
     echo "Error in pcm-raw"
     exit 1
 fi
 
-# Now get corresponding TSV files and replace JSON files in mapFile.csv with them
+echo   Now get corresponding TSV files and replace JSON files in mapFile.csv with them
 cp "mapfile.csv" "mapfile.csv_orig"
 for FILE in $FILES
 do
@@ -220,7 +252,7 @@ do
 done
 
 
-# Check pcm-raw with TSV files
+# echo Test pcm-raw with TSV files
 #./pcm-raw -r -e LD_BLOCKS.STORE_FORWARD -e CPU_CLK_UNHALTED.THREAD_ANY -e INST_RETIRED.ANY -e UNC_CHA_CLOCKTICKS -- sleep 1
 
 #if [ "$?" -ne "0" ]; then
@@ -274,18 +306,21 @@ EOF
 
 fi
 
+echo Testing pcm-raw with -el event_file_test.txt -tr -csv
 ./pcm-raw -el event_file_test.txt -tr -csv=raw_tr_wo_ext.csv -i=4 0.25
 if [ "$?" -ne "0" ]; then
     echo "Error in pcm-raw"
     exit 1
 fi
 
+echo Testing pcm-raw with -el event_file_test.txt -tr -ext -csv
 ./pcm-raw -el event_file_test.txt -tr -ext -csv=raw_tr_wi_ext.csv -i=4 0.25
 if [ "$?" -ne "0" ]; then
     echo "Error in pcm-raw"
     exit 1
 fi
 
+echo Testing pcm-raw with -el event_file_test.txt -tr -ext -single-header -csv
 ./pcm-raw -el event_file_test.txt -tr -ext -single-header -csv=raw_tr_wi_ext_single_header.csv -i=4 0.25
 if [ "$?" -ne "0" ]; then
     echo "Error in pcm-raw"