Merge pull request #233 from sy-c/master

check first orbit: also links + fatal error

Author: sy-c

doc/releaseNotes.md

@@ -493,3 +493,6 @@ This file describes the main feature changes for each readout.exe released versi
 
 ## v2.14.3 - 12/10/2022
 - Fix for DataDistribution multithreaded message formatting: TF ordering could get wrong out of the thread pool after having memory buffer empty.
+
+## next release
+- equipment-*.rdhCheckFirstOrbit: when set, readout also check consistency of orbits between all the links of an equipment. If not all links/equipments first orbit are the same, readout will interrupt the run.

src/ReadoutEquipment.cxx

@@ -139,7 +139,7 @@ ReadoutEquipment::ReadoutEquipment(ConfigFile& cfg, std::string cfgEntryPoint, b
   cfg.getOptionalValue<int>(cfgEntryPoint + ".rdhUseFirstInPageEnabled", cfgRdhUseFirstInPageEnabled);
   // configuration parameter: | equipment-* | rdhDumpFirstInPageEnabled | int | 0 | If set, the first RDH in each data page is logged. Setting a negative number will printit only for the first N pages. |
   cfg.getOptionalValue<int>(cfgEntryPoint + ".rdhDumpFirstInPageEnabled", cfgRdhDumpFirstInPageEnabled);
-  // configuration parameter: | equipment-* | rdhCheckFirstOrbit | int | 1 | If set, it is checked that the first orbit of all equipments is the same. |
+  // configuration parameter: | equipment-* | rdhCheckFirstOrbit | int | 1 | If set, it is checked that the first orbit of all equipments and links is the same. |
   cfg.getOptionalValue<int>(cfgEntryPoint + ".rdhCheckFirstOrbit", cfgRdhCheckFirstOrbit);
   // configuration parameter: | equipment-* | rdhCheckDetectorField | int | 0 | If set, the detector field is checked and changes reported. |
   cfg.getOptionalValue<int>(cfgEntryPoint + ".rdhCheckDetectorField", cfgRdhCheckDetectorField);  
@@ -255,8 +255,11 @@ void ReadoutEquipment::start()
   }
   equipmentLinksUsed.reset();
   equipmentLinksData.resize(RdhMaxLinkId + 1);
-  equipmentLinksData.clear();
+  equipmentLinksStats statsInit;
+  std::fill(equipmentLinksData.begin(), equipmentLinksData.end(), statsInit);
+  firstLinkId = undefinedLinkId;
   isError = 0;
+  isFatalError = 0;
   currentBlockId = 0;
   isDataOn = false;
   saveErrorPagesCount = 0;
@@ -314,7 +317,7 @@ void ReadoutEquipment::stop()
   std::string perLinkStats;
   for (unsigned int i = 0; i<= RdhMaxLinkId; i++) {
     if (equipmentLinksUsed[i]) {
-      perLinkStats += "[" + std::to_string(i) + "]=" + NumberOfBytesToString(equipmentLinksData[i], "B", 1024) + " ";
+      perLinkStats += "[" + std::to_string(i) + "]=" + NumberOfBytesToString(equipmentLinksData[i].bytesRx, "B", 1024) + " ";
     }
   }
   theLog.log(LogInfoDevel_(3003), "Links data received: %s", perLinkStats.c_str());
@@ -635,7 +638,8 @@ uint64_t ReadoutEquipment::getTimeframeFromOrbit(uint32_t hbOrbit)
     theLog.log(LogInfoDevel_(3011), "Equipment %s : first HB orbit = %X", name.c_str(), (unsigned int)firstTimeframeHbOrbitBegin);
     if (!isOk) {
       if (cfgRdhCheckFirstOrbit) {
-        theLog.log(LogErrorDevel_(3241), "Equipment %s : first HB orbit is different from other equipments", name.c_str());
+        theLog.log(LogErrorSupport_(3241), "Equipment %s : first HB orbit is different from other equipments", name.c_str());
+        isFatalError++;
       }
     }
   }
@@ -810,9 +814,29 @@ int ReadoutEquipment::processRdh(DataBlockContainerReference& block)
     // update links statistics
     if (h.getLinkId() <= RdhMaxLinkId) {
       equipmentLinksUsed[h.getLinkId()] = 1;
-      equipmentLinksData[h.getLinkId()] += blockHeader.dataSize;
+      equipmentLinksData[h.getLinkId()].bytesRx += blockHeader.dataSize;
+
+      // check link first orbit
+      // at this stage gReadoutStats.counters.firstOrbit is defined (done in getTimeframeFromOrbit())
+      uint32_t orbitId = h.getHbOrbit();
+      uint8_t linkId = h.getLinkId();
+      if (firstLinkId == undefinedLinkId) {
+        firstLinkId = linkId;
+        firstLinkOrbit = orbitId;
+      }
+      if (!equipmentLinksData[linkId].firstOrbitIsDefined) {
+        if (orbitId != firstLinkOrbit) {
+          if (cfgRdhCheckFirstOrbit) {
+            theLog.log(LogErrorSupport_(3241), "Equipment %s : first HB orbit of link %d is different from first link(%d): 0x%X != 0x%X", name.c_str(), linkId, firstLinkId, orbitId, firstLinkOrbit);
+            isFatalError++;
+          }
+        } else {
+          //theLog.log(LogInfoDevel_(3241), "Equipment %s : first HB orbit of link %d is same as first link(%d): 0x%X = 0x%X", name.c_str(), linkId, firstLinkId, orbitId, firstLinkOrbit);
+        }
+        equipmentLinksData[linkId].firstOrbit = orbitId;
+        equipmentLinksData[linkId].firstOrbitIsDefined = 1;
+      }
     }
-
   }
 
   // Dump RDH if configured to do so

src/ReadoutEquipment.h

@@ -51,6 +51,7 @@ class ReadoutEquipment
 
   bool stopOnError = false; // if set, readout will stop when this equipment reports an error (isError flag)
   int isError = 0;          // flag which might be used to count number of errors occuring in the equipment
+  int isFatalError = 0;     // flag which might be used to count number of fatal errors occuring in the equipment
 
   // protected:
   // todo: give direct access to output FIFO?
@@ -114,7 +115,16 @@ class ReadoutEquipment
   std::vector<CounterStats> equipmentStats;
   std::vector<CounterValue> equipmentStatsLast;
   std::bitset<RdhMaxLinkId + 1> equipmentLinksUsed;
-  std::vector<uint64_t> equipmentLinksData;
+
+  struct equipmentLinksStats {
+    uint64_t bytesRx = 0; // number of bytes received
+    uint32_t firstOrbit = undefinedOrbit; // first orbit received from this link
+    bool firstOrbitIsDefined = 0; // when 0, no value has been set for first orbit yet. need this flag because undefinedOrbit=0.
+  };
+  uint8_t firstLinkId = undefinedLinkId; // id of first link which sent data
+  uint32_t firstLinkOrbit = undefinedOrbit; // 1st orbit received from this equipment
+
+  std::vector<equipmentLinksStats> equipmentLinksData;
 
   double cfgConsoleStatsUpdateTime = 0;     // number of seconds between regular printing of statistics on console (if zero, only on stop)
   AliceO2::Common::Timer consoleStatsTimer; // timer to keep track of elapsed time between console statistics updates

src/mainReadout.cxx

@@ -1302,6 +1302,9 @@ int Readout::iterateCheck()
     if ((readoutDevice->isError) && (readoutDevice->stopOnError)) {
       isError = 1;
     }
+    if (readoutDevice->isFatalError) {
+      isError = 1;
+    }
   }
   if (isError) {
     return -1;