Merge pull request #253 from sy-c/master

v2.19.0

Author: sy-c

doc/releaseNotes.md

@@ -558,3 +558,8 @@ This file describes the main feature changes for each readout.exe released versi
 
 ## v2.18.5 - 17/05/2023
 - Fixed bug causing crash in some conditions of buffer empty. Related to same feature (dropIncomplete) as issue fixed in v2.18.3. Different corner case, involving threads, and not always happening.
+
+## v2.19.0 - 01/06/2023
+- o2-readout-test-fmq-memory: more options.
+- Added protection against unhandled exceptions (e.g. from ReadoutCard).
+- Added protection against unexpected state machine transitions (e.g. ECS sending RESET while STARTING).

src/ReadoutEquipmentCruEmulator.cxx

@@ -138,7 +138,9 @@ ReadoutEquipmentCruEmulator::ReadoutEquipmentCruEmulator(ConfigFile& cfg, std::s
   payloadDistrib = std::normal_distribution<> (cfgPayloadSize, cfgPayloadSizeStdev * cfgPayloadSize);
 }
 
-ReadoutEquipmentCruEmulator::~ReadoutEquipmentCruEmulator() {}
+ReadoutEquipmentCruEmulator::~ReadoutEquipmentCruEmulator() {
+  abortThread();
+}
 
 Thread::CallbackResult ReadoutEquipmentCruEmulator::prepareBlocks()
 {

src/ReadoutEquipmentDummy.cxx

@@ -58,7 +58,9 @@ ReadoutEquipmentDummy::ReadoutEquipmentDummy(ConfigFile& cfg, std::string cfgEnt
   }
 }
 
-ReadoutEquipmentDummy::~ReadoutEquipmentDummy() {}
+ReadoutEquipmentDummy::~ReadoutEquipmentDummy() {
+  abortThread();
+}
 
 DataBlockContainerReference ReadoutEquipmentDummy::getNextBlock()
 {

src/ReadoutEquipmentPlayer.cxx

@@ -187,6 +187,7 @@ ReadoutEquipmentPlayer::ReadoutEquipmentPlayer(ConfigFile& cfg, std::string cfgE
 
 ReadoutEquipmentPlayer::~ReadoutEquipmentPlayer()
 {
+  abortThread();
   fpLock.lock();
   if (fp != nullptr) {
     fclose(fp);

src/ReadoutEquipmentRORC.cxx

@@ -195,7 +195,9 @@ ReadoutEquipmentRORC::ReadoutEquipmentRORC(ConfigFile& cfg, std::string name) :
   isInitialized = true;
 }
 
-ReadoutEquipmentRORC::~ReadoutEquipmentRORC() {}
+ReadoutEquipmentRORC::~ReadoutEquipmentRORC() {
+    abortThread();
+}
 
 Thread::CallbackResult ReadoutEquipmentRORC::prepareBlocks()
 {

src/ReadoutEquipmentZmq.cxx

@@ -193,6 +193,7 @@ ReadoutEquipmentZmq::ReadoutEquipmentZmq(ConfigFile& cfg, std::string cfgEntryPo
 
 ReadoutEquipmentZmq::~ReadoutEquipmentZmq()
 {
+  abortThread();
 
   // stopping snapshot thread
   if (snapshotThread != nullptr) {

src/ReadoutVersion.h

@@ -9,5 +9,5 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
-#define READOUT_VERSION "2.18.5"
+#define READOUT_VERSION "2.19.0"
 

src/mainReadout.cxx

@@ -198,13 +198,71 @@ class Readout
 
  public:
   ~Readout();
-  int init(int argc, char* argv[]);
-  int configure(const boost::property_tree::ptree& properties);
-  int reset(); // as opposed to configure()
-  int start();
-  int stop(); // as opposed to start()
-  int iterateRunning();
-  int iterateCheck();
+
+  // protected calls catching all exceptions
+  int init(int argc, char* argv[]) {
+    try {
+      return _init(argc, argv);
+    }
+    catch (const std::exception& e) {
+      theLog.log(LogErrorSupport_(3245), "Exception : %s", e.what());
+    }
+    return -1;
+  }
+  int configure(const boost::property_tree::ptree& properties) {
+    try {
+      return _configure(properties);
+    }
+    catch (const std::exception& e) {
+      theLog.log(LogErrorSupport_(3245), "Exception : %s", e.what());
+    }
+    return -1;
+  }
+  int reset() { // as opposed to configure()
+    try {
+      return _reset();
+    }
+    catch (const std::exception& e) {
+      theLog.log(LogErrorSupport_(3245), "Exception : %s", e.what());
+    }
+    return -1;
+  }
+  int start() {
+    try {
+      return _start();
+    }
+    catch (const std::exception& e) {
+      theLog.log(LogErrorSupport_(3245), "Exception : %s", e.what());
+    }
+    return -1;
+  }
+  int stop() { // as opposed to start()
+    try {
+      return _stop();
+    }
+    catch (const std::exception& e) {
+      theLog.log(LogErrorSupport_(3245), "Exception : %s", e.what());
+    }
+    return -1;
+  }
+  int iterateRunning() {
+    try {
+      return _iterateRunning();
+    }
+    catch (const std::exception& e) {
+      theLog.log(LogErrorSupport_(3245), "Exception : %s", e.what());
+    }
+    return -1;
+  }
+  int iterateCheck() {
+    try {
+      return _iterateCheck();
+    }
+    catch (const std::exception& e) {
+      theLog.log(LogErrorSupport_(3245), "Exception : %s", e.what());
+    }
+    return -1;
+  }
 
   void loopRunning(); // called in state "running"
 
@@ -213,6 +271,14 @@ class Readout
   int cfgTimeStop = 0;         // time at which STOP should be executed in standalone mode
 
  private:
+  int _init(int argc, char* argv[]);
+  int _configure(const boost::property_tree::ptree& properties);
+  int _reset(); // as opposed to configure()
+  int _start();
+  int _stop(); // as opposed to start()
+  int _iterateRunning();
+  int _iterateCheck();
+
   ConfigFile cfg;
   const char* cfgFileURI = "";
   const char* cfgFileEntryPoint = ""; // where in the config tree to look for
@@ -330,7 +396,7 @@ void Readout::publishLogbookStats()
 #endif
 }
 
-int Readout::init(int argc, char* argv[])
+int Readout::_init(int argc, char* argv[])
 {
   setThreadName(nullptr);
 
@@ -577,7 +643,7 @@ int Readout::init(int argc, char* argv[])
 
 //#include <boost/property_tree/json_parser.hpp>
 
-int Readout::configure(const boost::property_tree::ptree& properties)
+int Readout::_configure(const boost::property_tree::ptree& properties)
 {
   theLog.log(LogInfoSupport_(3005), "Readout executing CONFIGURE");
   gReadoutStats.counters.state = stringToUint64("> conf");
@@ -1249,7 +1315,7 @@ int Readout::configure(const boost::property_tree::ptree& properties)
   return 0;
 }
 
-int Readout::start()
+int Readout::_start()
 {
   theLog.resetMessageCount();
   theLog.log(LogInfoSupport_(3005), "Readout executing START");
@@ -1359,6 +1425,7 @@ void Readout::loopRunning()
   //ProfilerStart(gperfOutputFile.c_str());
 #endif
 
+  try {
   for (;;) {
     if ((!isRunning) && ((cfgFlushEquipmentTimeout <= 0) || (stopTimer.isTimeout()))) {
       break;
@@ -1423,6 +1490,10 @@ void Readout::loopRunning()
       usleep(1000);
     }
   }
+  }
+  catch (const std::exception& e) {
+    theLog.log(LogErrorSupport_(3245), "Exception : %s", e.what());
+  }
 
 #ifdef CALLGRIND
   CALLGRIND_STOP_INSTRUMENTATION;
@@ -1437,7 +1508,7 @@ void Readout::loopRunning()
   theLog.log(LogInfoDevel, "Exiting main loop");
 }
 
-int Readout::iterateCheck()
+int Readout::_iterateCheck()
 {
   usleep(100000);
   for (auto&& readoutDevice : readoutDevices) {
@@ -1465,7 +1536,7 @@ int Readout::iterateCheck()
   return 0;
 }
 
-int Readout::iterateRunning()
+int Readout::_iterateRunning()
 {
   usleep(100000);
   // printf("running time: %.2f\n",startTimer.getTime());
@@ -1492,7 +1563,7 @@ int Readout::iterateRunning()
   return 0;
 }
 
-int Readout::stop()
+int Readout::_stop()
 {
 
   theLog.log(LogInfoSupport_(3005), "Readout executing STOP");
@@ -1593,7 +1664,7 @@ int Readout::stop()
   return 0;
 }
 
-int Readout::reset()
+int Readout::_reset()
 {
 
   theLog.log(LogInfoSupport_(3005), "Readout executing RESET");
@@ -1640,6 +1711,10 @@ int Readout::reset()
 
   theLog.log(LogInfoDevel, "Releasing readout devices");
   for (size_t i = 0, size = readoutDevices.size(); i != size; ++i) {
+    if (readoutDevices[i] != nullptr) {
+      // ensure readout equipment threads stopped before releasing resources
+      readoutDevices[i]->abortThread();
+    }
     readoutDevices[i] = nullptr; // effectively deletes the device
   }
   readoutDevices.clear();

src/readoutErrorCodes.h

@@ -52,5 +52,6 @@
   { 3242, "Database error", nullptr},
   { 3243, "Control problem", nullptr},
   { 3244, "Should not happen problem", nullptr},
+  { 3245, "Unhandled exception", nullptr},
 */