Merge pull request #256 from sy-c/master

readout-monitor-memory

Author: sy-c

CMakeLists.txt

@@ -75,6 +75,7 @@ find_package(BookkeepingApi)
 find_package(ZMQ)
 find_package(MySQL)
 #find_package(gperftools)
+find_package(SDL)
 
 # extract include directories from targets
 get_target_property(InfoLogger_INCLUDE_DIRS AliceO2::InfoLogger INTERFACE_INCLUDE_DIRECTORIES)
@@ -147,9 +148,18 @@ if(${gperftools_FOUND})
   message(STATUS "gperftools enabled")
 endif()
 
+# check SDL
+if(SDL_FOUND)
+  message(
+    STATUS
+    "SDL found: ${SDL_INCLUDE_DIRS} ${SDL_LIBRARIES}")
+  else ()
+    message(STATUS "SDL not found, corresponding features will be disabled.")
+endif()
+
 
 # add flags to enable optional features in Readout, based on available dependencies
-add_compile_definitions($<$<BOOL:${Numa_FOUND}>:WITH_NUMA> $<$<BOOL:${RDMA_FOUND}>:WITH_RDMA> $<$<BOOL:${Configuration_FOUND}>:WITH_CONFIG> $<$<BOOL:${FairMQ_FOUND}>:WITH_FAIRMQ> $<$<BOOL:${Occ_FOUND}>:WITH_OCC> $<$<BOOL:${BookkeepingApi_FOUND}>:WITH_LOGBOOK> $<$<BOOL:${ZMQ_FOUND}>:WITH_ZMQ> $<$<BOOL:${MYSQL_FOUND}>:WITH_DB> $<$<BOOL:${ReadoutCard_FOUND}>:WITH_READOUTCARD> $<$<BOOL:${gperftools_FOUND}>:WITH_GPERFTOOLS>)
+add_compile_definitions($<$<BOOL:${Numa_FOUND}>:WITH_NUMA> $<$<BOOL:${RDMA_FOUND}>:WITH_RDMA> $<$<BOOL:${Configuration_FOUND}>:WITH_CONFIG> $<$<BOOL:${FairMQ_FOUND}>:WITH_FAIRMQ> $<$<BOOL:${Occ_FOUND}>:WITH_OCC> $<$<BOOL:${BookkeepingApi_FOUND}>:WITH_LOGBOOK> $<$<BOOL:${ZMQ_FOUND}>:WITH_ZMQ> $<$<BOOL:${MYSQL_FOUND}>:WITH_DB> $<$<BOOL:${ReadoutCard_FOUND}>:WITH_READOUTCARD> $<$<BOOL:${gperftools_FOUND}>:WITH_GPERFTOOLS> $<$<BOOL:${SDL_FOUND}>:WITH_SDL>)
 
 # define include directories
 set(READOUT_INCLUDE_DIRS
@@ -332,6 +342,9 @@ endif()
 # executables
 ###################################################
 
+# list of executables build (to be completed depending on dependencies found)
+set(executables o2-readout-exe o2-readout-receiver o2-readout-test-fmq-tx o2-readout-test-fmq-rx o2-readout-test-fmq-perf-tx o2-readout-test-fmq-perf-rx o2-readout-test-memorybanks o2-readout-rawreader o2-readout-test-lib-monitoring)
+
 # o2-readout-exe : main executable
 add_executable(
 	o2-readout-exe
@@ -428,7 +441,6 @@ add_executable(
 	o2-readout-eventdump
         ${SOURCE_DIR}/eventDump.cxx
 	$<TARGET_OBJECTS:objReadoutUtils>
-	
 )
 
 # a process to monitor stats of remote readout processes
@@ -441,6 +453,17 @@ add_executable(
 target_include_directories(o2-readout-monitor PRIVATE ${READOUT_INCLUDE_DIRS})
 target_link_libraries(o2-readout-monitor PRIVATE AliceO2::InfoLogger AliceO2::Common ${ZMQ_LIBRARIES})
 
+if(SDL_FOUND AND ZMQ_FOUND)
+# a process to monitor memory of remote readout processes
+add_executable(
+	o2-readout-monitor-memory
+        ${SOURCE_DIR}/readoutMemoryMonitor.cxx
+)
+target_include_directories(o2-readout-monitor-memory PRIVATE ${READOUT_INCLUDE_DIRS})
+target_link_libraries(o2-readout-monitor-memory PRIVATE AliceO2::InfoLogger AliceO2::Common ${ZMQ_LIBRARIES} $<$<BOOL:${SDL_FOUND}>:${SDL_LIBRARIES}> SDL2)
+list (APPEND executables o2-readout-monitor-memory)
+endif()
+
 # a process to admin readout database
 if (MYSQL_FOUND)
 add_executable(
@@ -449,6 +472,7 @@ add_executable(
 	$<TARGET_OBJECTS:objReadoutDatabase>
 )
 target_include_directories(o2-readout-admin-db PRIVATE ${READOUT_INCLUDE_DIRS})
+list (APPEND executables o2-readout-admin-db)
 endif()
 
 endif()
@@ -481,7 +505,6 @@ if (NOT FairMQ_FOUND)
 endif ()
 
 # set include and libraries for all
-set(executables o2-readout-exe o2-readout-receiver o2-readout-test-fmq-tx o2-readout-test-fmq-rx o2-readout-test-fmq-perf-tx o2-readout-test-fmq-perf-rx o2-readout-test-memorybanks o2-readout-rawreader o2-readout-test-lib-monitoring)
 if (ReadoutCard_FOUND)
   list (APPEND executables o2-readout-test-roc o2-readout-test-roc-find)
 endif()
@@ -491,9 +514,6 @@ endif()
 if (BookkeepingApi_FOUND)
   list (APPEND executables o2-readout-test-lib-bookkeeping)
 endif()
-if (MYSQL_FOUND)
-  list (APPEND executables o2-readout-admin-db)
-endif()
 foreach (exe ${executables})
 	target_include_directories(${exe} PRIVATE ${READOUT_INCLUDE_DIRS})
 	target_link_libraries(${exe} PRIVATE ${READOUT_LINK_LIBRARIES})

doc/configurationParameters.md

@@ -1,5 +1,36 @@
 # Readout configuration parameters reference
 
+This document describes all the configuration parameters that can be set in readout.
+
+Some parameters control the behavior of readout as soon as it starts, and valid for the lifetime of the process. (1)
+
+Other parameters (2) are set whenever the control state-machine receives the CONFIGURE command (or shortly after startup when running o2-readout-exe outside from the ECS workflow, typically from the command line) and until the RESET command.
+The corresponding parameters usually come from a local file or from a central reposity, as provided by the O2 system.
+They can be reloaded several time during the lifetime of the process.
+
+
+# Readout defaults (1)
+
+These are defined in local file `/etc/o2.d/readout-default.cfg`, which is loaded on `o2-readout-exe` startup (if file exists). They are valid for the lifetime of the o2-readout-exe process. There is usually no need for users to edit these settings. The file format is '.ini' style, with `[sections]` and `key=value` pairs.
+
+Following table describe the parameters of the `[readout]` section.
+
+| Parameter name  | Type | Default value | Description |
+|--|--|--|--|
+
+| memLock | int | 0 | Defines the memory swapping policy. If set, all readout memory is locked in RAM with mlockall() |
+| readoutExe | string | | If set, overwrites the current running executable with the one at given path. Used for tests. |
+| readoutConfig | string | | If set with readoutExe, launch the new exe with this parameter as argument. Used for tests. |
+| statsPublishAddress | string | | If set, address where to publish readout stats, eg "tcp://127.0.0.1:6008". Connect to this service with o2-readout-monitor. |
+| statsPublishInterval | string | | If set, interval for readout stats publish, in seconds. |
+| db | string | | If set, defines connection parameters to a MySQL database in the form 'user:password@host/dbname'. Runtime statistics are stored in a table. Structure can be created with o2-readout-admin-db. |
+| customCommandsEnabled | int | 0 | If set, custom commands are enabled, i.e. launching external scripts at some specific state transitions. See customCommands keyword below. |
+| membanksMonitorPath | string | | Path to memory banks monitor output (a local file FIFO, to be connected with o2-readout-monitor). |
+| membanksMonitorRate | double | 0 | Rate (in Hertz) at which to publish memory status. |
+
+
+# Readout configuration file (2)
+
 ## Sections
 The readout configuration is split in different sections. Some sections have a unique instance (single name), some may have multiple instances (prefix-[instance name], shown as prefix-\* in this document).
 

doc/memory.md

@@ -61,3 +61,10 @@ The grand-total for the shared memory block is therefore typically: `((128 * (nu
 Which makes, for a FLP with 3 CRUs, 12 links per end-point, 5GB/s data throughput, 0.5s aggregator STF timeout, .25M page size and 30% safety margin: 
 ( (128 * 3 * 2 * 12 * 0.25 + 0.5 * 5000 ) * 1.2 + 42000 ) * 1.3 = 62 GB
 
+
+## Checking runtime memory status
+
+The current usage of the data pages can be checked at runtime with a command such as:
+`o2-readout-monitor-memory tcp://127.0.0.1:50002`
+
+ provided that the parameters `membanksMonitorRate` is defined in `readout-defaults.cfg`.

doc/releaseNotes.md

@@ -567,3 +567,4 @@ This file describes the main feature changes for each readout.exe released versi
 ## next version
 - Added a separate thread to call bookkeeping API functions, so that data flow is not affected in case they are blocking.
 - o2-readout-test-lib-bookkeeping: more options.
+- Added o2-readout-monitor-memory: to view in real time state of data pages.

src/MemoryBankManager.cxx

@@ -18,6 +18,9 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 
+#ifdef WITH_ZMQ
+#include <zmq.h>
+#endif
 
 #include "readoutInfoLogger.h"
 
@@ -258,6 +261,55 @@ void MemoryBankManager::reset()
 void MemoryBankManager::monitorThLoop() {
   AliceO2::Common::Timer t;
   t.reset(1000000.0 / monitorUpdateRate);
+  MemoryPagesPool::Stats mps;
+
+#ifdef WITH_ZMQ
+  void* zmqContext = nullptr;
+  void* zmqHandle = nullptr;
+  bool zmqEnabled = 1;
+  void zmqCleanup();
+  int zmqError = 0;
+  std::string zmqPort = "tcp://127.0.0.1:50002";
+  try {
+    zmqContext = zmq_ctx_new();
+    if (zmqContext == nullptr) {
+      zmqError = zmq_errno();
+      throw __LINE__;
+    }
+
+    zmqHandle = zmq_socket(zmqContext, ZMQ_PUB);
+    if (zmqHandle == nullptr) {
+      zmqError = zmq_errno();
+      throw __LINE__;
+    }
+
+    const int cfgZmqLinger = 1000;
+    zmqError = zmq_setsockopt(zmqHandle, ZMQ_LINGER, (void*)&cfgZmqLinger, sizeof(cfgZmqLinger)); // close timeout
+    if (zmqError) {
+      throw __LINE__;
+    }
+
+    zmqError = zmq_bind(zmqHandle, zmqPort.c_str());
+    if (zmqError) {
+      throw __LINE__;
+    }
+
+    zmqEnabled = 1;
+
+  } catch (int lineErr) {
+    if (zmqError) {
+      theLog.log(LogErrorDevel, "ZeroMQ error @%d : (%d) %s", lineErr, zmqError, zmq_strerror(zmqError));
+    } else {
+      theLog.log(LogErrorDevel, "Error @%d", lineErr);
+    }
+    // ZMQ unavailable does not cause consumer to fail starting
+    theLog.log(LogErrorDevel, "Memory banks manager: ZMQ stats publishing disabled");
+  }
+  if (zmqEnabled) {
+    theLog.log(LogInfoDevel, "Memory banks manager: ZMQ stats publishing enabled on %s", zmqPort.c_str());
+  }
+#endif
+
   for(;!monitorThShutdown.load();) {
     if (t.isTimeout()) {
       std::unique_lock<std::mutex> lock(bankMutex);
@@ -271,6 +323,34 @@ void MemoryBankManager::monitorThLoop() {
           fclose(fp);
         }
       }
+#ifdef WITH_ZMQ
+      if (zmqEnabled) {
+        int msgSize =  0;
+        uint32_t numberOfPools = pools.size();
+        zmq_send(zmqHandle, &numberOfPools, sizeof(numberOfPools), ZMQ_SNDMORE);
+        for (auto& it : pools) {
+          it->getDetailedStats(mps);
+          zmq_send(zmqHandle, &mps, sizeof(mps), ZMQ_SNDMORE);
+          zmq_send(zmqHandle, &mps.states[0], sizeof(mps.states[0]) * mps.states.size(), ZMQ_SNDMORE);
+          msgSize += (int)(sizeof(mps) + sizeof(mps.states[0]) * mps.states.size());
+        }
+        uint32_t trailer = 0xF00F;
+        zmq_send(zmqHandle, &trailer, sizeof(trailer), ZMQ_DONTWAIT);
+        msgSize += (int)(sizeof(numberOfPools) +  sizeof(trailer));
+        //theLog.log(LogDebugDevel, "mem monitor: published %d bytes", msgSize);
+      }
+#endif
+      /*
+      for (auto& it : pools) {
+        it->getDetailedStats(mps);
+        printf("pool %d : %f - %f = %f\n", mps.id, mps.t1, mps.t0, mps.t1-mps.t0);
+        for (unsigned int i=0; i<10; i++) {
+          if (i>=mps.states.size()) break;
+          printf("%d %s %.6f\n",i,MemoryPage::getPageStateString(mps.states[i].state), mps.states[i].timeInCurrentState);
+        }
+      }
+      */
+
       t.increment();
     } else {
       std::this_thread::sleep_for(std::chrono::microseconds(10000));

src/ReadoutStats.cxx

@@ -44,7 +44,9 @@ ReadoutStats::~ReadoutStats() {
     publishThread->join();
     publishThread = nullptr;
   }
+  #ifdef WITH_ZMQ
   zmqCleanup();
+  #endif
 }
 
 void ReadoutStats::reset(bool lightReset)
@@ -93,9 +95,8 @@ uint64_t stringToUint64(const char* in)
 
 int ReadoutStats::startPublish(const std::string &cfgZmqPublishAddress, double cfgZmqPublishInterval) {
 
-  if (zmqEnabled) return __LINE__;
-
   #ifdef WITH_ZMQ
+  if (zmqEnabled) return __LINE__;
 
   // default ZMQ settings for data monitoring
   int cfg_ZMQ_CONFLATE = 0; // buffer last message only
@@ -157,6 +158,9 @@ int ReadoutStats::startPublish(const std::string &cfgZmqPublishAddress, double c
   } else {
     return -1; //disabled
   }
+  #else
+  (void)cfgZmqPublishAddress;
+  (void)cfgZmqPublishInterval;
   #endif
 
   publishNow();
@@ -182,7 +186,9 @@ void ReadoutStats::zmqCleanup() {
 
 int ReadoutStats::stopPublish() {
   publishNow();
+  #ifdef WITH_ZMQ
   zmqCleanup();
+  #endif
   return 0;
 }