Buffer status counters

Author: sy-c

doc/releaseNotes.md

@@ -451,3 +451,6 @@ This file describes the main feature changes for each readout.exe released versi
 ## v2.10.1 - 20/04/2022
 - Updated configuration parameters:
   - equipment-file-*.updateOrbits: when set to zero, RDH orbits are not updated in file loop replay. This is needed for some reconstruction tests. This however creates a stream of data with inconsistent orbit ids and mismatching timeframe information.
+
+## v2.10.2 - 04/05/2022
+- Added some buffer monitoring counters. They are logged in console stats and in monitoring (readout.bufferUsage, one per buffer, from 0% to 100%).

src/MemoryBankManager.cxx

@@ -142,5 +142,6 @@ void MemoryBankManager::reset()
     theLog.log(LogInfoDevel_(3008), "Releasing bank %s%s", it.name.c_str(), (useCount == 1) ? "" : "warning - still in use elsewhere !");
   }
   banks.clear();
+  poolIndex = -1;
 }
 

src/ReadoutStats.cxx

@@ -46,7 +46,7 @@ ReadoutStats::~ReadoutStats() {
   zmqCleanup();
 }
 
-void ReadoutStats::reset()
+void ReadoutStats::reset(bool lightReset)
 {
   counters.notify = 0;
 
@@ -66,8 +66,10 @@ void ReadoutStats::reset()
 
   counters.firstOrbit = undefinedOrbit;
 
-  for (unsigned int i = 0; i < ReadoutStatsMaxItems; i++) {
-    counters.bufferUsage[i] = -1.0;
+  if (!lightReset) {
+    for (unsigned int i = 0; i < ReadoutStatsMaxItems; i++) {
+      counters.bufferUsage[i] = -1.0;
+    }
   }
 }
 

src/ReadoutStats.h

@@ -24,7 +24,7 @@
 #include <zmq.h>
 #endif
 
-const int ReadoutStatsMaxItems = 32;
+const int ReadoutStatsMaxItems = 25;
 
 struct ReadoutStatsCounters {
   uint32_t version; // version number of this header
@@ -63,7 +63,7 @@ class ReadoutStats
  public:
   ReadoutStats();
   ~ReadoutStats();
-  void reset();
+  void reset(bool lightReset=0); // light reset: to preserve the buffer counters
   void print();
 
   ReadoutStatsCounters counters;

src/mainReadout.cxx

@@ -1039,7 +1039,7 @@ int Readout::start()
 {
   theLog.resetMessageCount();
   theLog.log(LogInfoSupport_(3005), "Readout executing START");
-  gReadoutStats.reset();
+  gReadoutStats.reset(1);
   gReadoutStats.counters.state = stringToUint64("> start");
   gReadoutStats.counters.notify++;
   gReadoutStats.publishNow();

src/readoutMonitor.cxx

@@ -288,9 +288,14 @@ int main(int argc, const char** argv)
       if (nRfmq) {
 	avgTfmq = (counters->pagesPendingFairMQtime.load() / nRfmq) / (deltaT * 1000000.0);
       }
+      std::string vBufferUsage;
+      for (int i = 0; i < ReadoutStatsMaxItems; i++) {
+	double r = counters->bufferUsage[i].load();
+	vBufferUsage += ( (i==0) ? "" : ",") + ( (r<0) ? "" : std::to_string((int)(r*100)) );
+      }
       char buf[1024];
       if (cfgRawBytes) {
-        snprintf(buf, sizeof(buf), "%f\t%s\t%s\t%llu\t%llu\t%llu\t%llu\t%llu\t%.2lf\t%.6lf\t%d\n",
+        snprintf(buf, sizeof(buf), "%f\t%s\t%s\t%llu\t%llu\t%llu\t%llu\t%llu\t%.2lf\t%.6lf\t%d\t%s\n",
            (double)t,
 	   counters->source,
            (char*)&state,
@@ -301,8 +306,9 @@ int main(int argc, const char** argv)
            (unsigned long long)counters->pagesPendingFairMQ.load(),
            nRfmq,
            avgTfmq,
-           (int)counters->timeframeIdFairMQ.load());
-	   fflush(stdout);
+           (int)counters->timeframeIdFairMQ.load(),
+	   vBufferUsage.c_str()
+	 );
       } else {
         snprintf(buf, sizeof(buf), "%s  %s %s     %8llu     %s   %s   %s   %6llu    %7.2lf    %6.4lf %8d\n",
            t ? getStringTime(t).c_str() : "-",
@@ -322,6 +328,7 @@ int main(int argc, const char** argv)
         broadcastSocket->broadcast(buf);
       } else {
         printf("%s",buf);
+	fflush(stdout);
       }
 
       if (metricsLogEnabled) {