added checkIncomplete option

Author: sy-c

doc/configurationParameters.md

@@ -54,6 +54,7 @@ The parameters related to 3rd-party libraries are described here for convenience
 | consumer-* | numaNode | int | -1 | If set (>=0), memory / thread will try to use given NUMA node. |
 | consumer-* | stopOnError | int | 0 | If 1, readout will stop automatically on consumer error. |
 | consumer-data-sampling-* | address | string | ipc:///tmp/readout-pipe-1 | Address of the data sampling. |
+| consumer-FairMQChannel-* | checkIncomplete | int | 0 | If set, readout checks for the completeness of HBF and issues warnings. Set automatically when dropIncomplete=1. |
 | consumer-FairMQChannel-* | checkResources | string | | Check beforehand if unmanaged region would fit in given list of resources. Comma-separated list of items to be checked: eg /dev/shm, MemFree, MemAvailable. (any filesystem path, and any /proc/meminfo entry).|
 | consumer-FairMQChannel-* | disableSending | int | 0 | If set, no data is output to FMQ channel. Used for performance test to create FMQ shared memory segment without pushing the data. |
 | consumer-FairMQChannel-* | dropIncomplete | int | 0 | If set, TF with incomplete HBF (i.e. HBF having missing packets) are discarded. |

doc/releaseNotes.md

@@ -544,3 +544,7 @@ This file describes the main feature changes for each readout.exe released versi
 ## v2.18.1 - 23/03/2023
 - Optimized EOR sequence in case FMQ stuck, as per timeout readout.flushConsumerTimeout definition. Pending data are discarded.
 - Moved FMQ repack stats logs at STOP instead of RESET.
+
+## v2.18.2 - 29/03/2023
+- Updated configuration parameters:
+  - added consumer-FairMQchannel.checkIncomplete: when set, TF with incomplete HBF are detected and warnings issued. Off by default (in particular, check is currently not valid for DDL detector). In previous v2.18 versions, this check was enabled by default.

src/ConsumerFMQchannel.cxx

@@ -157,6 +157,7 @@ class ConsumerFMQchannel : public Consumer
   bool enableStfSuperpage = false; // optimized stf transport: minimize STF packets
   bool enableRawFormatDatablock = false;
   int enablePackedCopy = 1; // default mode for repacking of page overlapping HBF. 0 = one page per copy, 1 = change page on TF only
+  int checkIncomplete = 0; // TF are checked to detect missing packets
   int dropIncomplete = 0; // TF with missing packets are discarded
 
   std::shared_ptr<MemoryBank> memBank; // a dedicated memory bank allocated by FMQ mechanism
@@ -248,10 +249,16 @@ class ConsumerFMQchannel : public Consumer
       gReadoutStats.isFairMQ = 1; // enable FMQ stats
     }
 
+    // configuration parameter: | consumer-FairMQChannel-* | checkIncomplete | int | 0 | If set, readout checks for the completeness of HBF and issues warnings. Set automatically when dropIncomplete=1. |
+    cfg.getOptionalValue<int>(cfgEntryPoint + ".checkIncomplete", checkIncomplete, checkIncomplete);
+
     // configuration parameter: | consumer-FairMQChannel-* | dropIncomplete | int | 0 | If set, TF with incomplete HBF (i.e. HBF having missing packets) are discarded. |
     cfg.getOptionalValue<int>(cfgEntryPoint + ".dropIncomplete", dropIncomplete, dropIncomplete);
     if (dropIncomplete) {
+      checkIncomplete = 1;
       theLog.log(LogInfoDevel_(3002), "TF with incomplete HBF will be discarded");
+    } else if (checkIncomplete) {
+      theLog.log(LogInfoDevel_(3002), "TF with incomplete HBF will be checked");
     }
 
     // configuration parameter: | consumer-FairMQChannel-* | enableRawFormat | int | 0 | If 0, data is pushed 1 STF header + 1 part per HBF. If 1, data is pushed in raw format without STF headers, 1 FMQ message per data page. If 2, format is 1 STF header + 1 part per data page.|
@@ -847,6 +854,7 @@ int ConsumerFMQchannel::DDformatMessage(DataSetReference &bc, DDMessage &ddm) {
     HBFerr += " (" + std::to_string(++HBFerrid) + ") ";
   };
   auto checkLastHB = [&] () {
+    if (!checkIncomplete) return;
     if (HBFisFirst) {
       return; // no HBF seen so far
     }
@@ -935,25 +943,27 @@ int ConsumerFMQchannel::DDformatMessage(DataSetReference &bc, DDMessage &ddm) {
         // printf("block %p : offset %d = %p\n",b,offset,rdh);
       }
 
-      uint16_t HBFpagescounterNew = (uint16_t)rdh->pagesCounter;
-      if (HBFisFirst) {
-        HBFpagescounterFirst = HBFpagescounterNew;
-        HBFisFirst = 0;
-        if (HBFpagescounterFirst != 0) {
-          HBFincrerr();
-          HBFerr += "first pagesCounter not zero: " + std::to_string((int)HBFpagescounterFirst);
-        }
-      } else {
-        if (HBFpagescounterNew != HBFpagescounterLast + 1) {
-          HBFincrerr();
-          HBFerr += "pagesCounter jump from " + std::to_string((int)HBFpagescounterLast)+ " to " + std::to_string( (int)HBFpagescounterNew);
-          HBFisOk = 0;
+      if (checkIncomplete) {
+        uint16_t HBFpagescounterNew = (uint16_t)rdh->pagesCounter;
+        if (HBFisFirst) {
+          HBFpagescounterFirst = HBFpagescounterNew;
+          HBFisFirst = 0;
+          if (HBFpagescounterFirst != 0) {
+            HBFincrerr();
+            HBFerr += "first pagesCounter not zero: " + std::to_string((int)HBFpagescounterFirst);
+          }
+        } else {
+          if (HBFpagescounterNew != HBFpagescounterLast + 1) {
+            HBFincrerr();
+            HBFerr += "pagesCounter jump from " + std::to_string((int)HBFpagescounterLast)+ " to " + std::to_string( (int)HBFpagescounterNew);
+            HBFisOk = 0;
+          }
         }
+        HBFpagescounter++;
+        HBFpagescounterLast = HBFpagescounterNew;
+        HBFstop += rdh->stopBit;
+        HBFstopLast = rdh->stopBit;
       }
-      HBFpagescounter++;
-      HBFpagescounterLast = HBFpagescounterNew;
-      HBFstop += rdh->stopBit;
-      HBFstopLast = rdh->stopBit;
 
       uint16_t offsetNextPacket = rdh->offsetNextPacket;
       if (offsetNextPacket == 0) {

src/readoutConfigEditor.tcl

@@ -30,6 +30,7 @@ set configurationParametersDescriptor {
 | consumer-* | numaNode | int | -1 | If set (>=0), memory / thread will try to use given NUMA node. |
 | consumer-* | stopOnError | int | 0 | If 1, readout will stop automatically on consumer error. |
 | consumer-data-sampling-* | address | string | ipc:///tmp/readout-pipe-1 | Address of the data sampling. |
+| consumer-FairMQChannel-* | checkIncomplete | int | 0 | If set, readout checks for the completeness of HBF and issues warnings. Set automatically when dropIncomplete=1. |
 | consumer-FairMQChannel-* | checkResources | string | | Check beforehand if unmanaged region would fit in given list of resources. Comma-separated list of items to be checked: eg /dev/shm, MemFree, MemAvailable. (any filesystem path, and any /proc/meminfo entry).|
 | consumer-FairMQChannel-* | disableSending | int | 0 | If set, no data is output to FMQ channel. Used for performance test to create FMQ shared memory segment without pushing the data. |
 | consumer-FairMQChannel-* | dropIncomplete | int | 0 | If set, TF with incomplete HBF (i.e. HBF having missing packets) are discarded. |