Merge pull request #222 from sy-c/master

v2.10.4

Author: sy-c

doc/memory.md

@@ -44,3 +44,20 @@ Considerations to make good use of the FLP memory and tune the corresponding Rea
 	* the good way to evaluate this parameter is too look at readout logs and see what is the maximum actual size used for this buffer after some reference runs, and add some 50% margin. The current parameters have been set based on initial detector tests, and can be reevaluated any time with the same method.
 
 The readout memory parameters are not critical in most conditions. Tuning them may help to handle high peak loads (e.g. in case of bursts in rates) or handle startup setup time for the processing system to reach a stable state. It will however not solve issues where the output system can not cope with the input rate or when it degrades over time, it will just delay the appearance of the symptoms.
+
+
+## How much memory is needed by Readout ?
+
+The following is the minimum amount of buffer needed to run Readout, in number of pages: `128 * (number of enabled links) + readout.aggregatorStfTimeout * (page rate)`
+
+* Each end-point needs at least 128 pages per enabled link. This is the size of the FIFO providing empty data pages to ROC device, and it should be filled to guarantee uninterrupted DMA. Readout logs can be checked for runtime statistics about the effective page rate observed (see 'blockRate' metric).
+* The readout.aggregatorStfTimeout configuration parameter defines the amount of time the ready pages are waiting in the readout aggregator before the corresponding TF to be processed. This allows some delay to receive data from all the different equipments / links (the links contributing to a TF are not known, so readout must wait before assuming all data are there). A corresponding amount of buffer must be reserved, according to the superpage rate observed at runtime.
+* 20% extra is needed for the 'overlapping HBF data' copy.
+* DD can buffer up to 32GB before stfSender goes to "discard mode" and release memory, when data input exceeds FLP output.
+* Additionnal processing pipelines in the FMQ chain must be accounted for: up to 10GB can be pending in DPL before reaching stfSender, and are not counted in the DD 32GB.
+
+The grand-total for the shared memory block is therefore typically: `((128 * (number of enabled links) * pageSize + readout.aggregatorStfTimeout * throughput) * 1.2 + 42 GB)`
+
+Which makes, for a FLP with 3 CRUs, 12 links per end-point, 5GB/s data throughput, 0.5s aggregator STF timeout, .25M page size and 30% safety margin: 
+( (128 * 3 * 2 * 12 * 0.25 + 0.5 * 5000 ) * 1.2 + 42000 ) * 1.3 = 62 GB
+

doc/releaseNotes.md

@@ -457,3 +457,6 @@ This file describes the main feature changes for each readout.exe released versi
 
 ## v2.10.3 - 05/05/2022
 - Bug fixed in the readout TF rate regulator (was ineffective after 35 minutes = 2^31 microseconds - wrong variable width in a conversion).
+
+## v2.10.4 - 18/05/2022
+- Changed level of "Bypassing RORC firmware compatibility check", retrograded from support warning to developper info.

src/MemoryPagesPool.cxx

@@ -321,5 +321,5 @@ int MemoryPagesPool::getId() {
 void MemoryPagesPool::setBufferStateVariable(std::atomic<double> *bufferStateVar) {
   pBufferState=bufferStateVar;
   updateBufferState();
-  printf("buffer usage = %lf @ 0x%p\n", pBufferState->load(),pBufferState);
+  //printf("buffer usage = %lf @ 0x%p\n", pBufferState->load(),pBufferState);
 }

src/ReadoutEquipmentRORC.cxx

@@ -102,7 +102,7 @@ ReadoutEquipmentRORC::ReadoutEquipmentRORC(ConfigFile& cfg, std::string name) :
     // configuration parameter: | equipment-rorc-* | firmwareCheckEnabled | int | 1 | If set, RORC driver checks compatibility with detected firmware. Use 0 to bypass this check (eg new fw version not yet recognized by ReadoutCard version). |
     cfg.getOptionalValue<int>(name + ".firmwareCheckEnabled", cfgFirmwareCheckEnabled);
     if (!cfgFirmwareCheckEnabled) {
-      theLog.log(LogWarningSupport_(3002), "Bypassing RORC firmware compatibility check");
+      theLog.log(LogInfoDevel_(3002), "Bypassing RORC firmware compatibility check");
     }
 
     // configuration parameter: | equipment-rorc-* | debugStatsEnabled | int | 0 | If set, enable extra statistics about internal buffers status. (printed to stdout when stopping) |

src/ReadoutVersion.h

@@ -9,5 +9,5 @@
 // granted to it by virtue of its status as an Intergovernmental Organization
 // or submit itself to any jurisdiction.
 
-#define READOUT_VERSION "2.10.3"
+#define READOUT_VERSION "2.10.4"
 

src/mainReadout.cxx

@@ -1475,6 +1475,7 @@ void Readout::executeCustomCommand(const char *stateChange) {
       theLog.log(LogInfoDevel_(3013), "Executing custom command for %s : %s", it->first.c_str(), it->second.c_str());
       std::string cmd = it->second + "\n";
       write(customCommandsShellFdIn, cmd.c_str(), cmd.length());
+      fsync(customCommandsShellFdIn);
       LineBuffer b;
       const int cmdTimeout = 10000; // 10s timeout
       b.appendFromFileDescriptor(customCommandsShellFdOut, cmdTimeout);

src/readoutStatus.tcl

@@ -434,21 +434,28 @@ if {1} {
 proc NumberOfBytesToString {value suffix} {
   set prefixes { " " "k" "M" "G" "T" "P" }
   set maxPrefixIndex [llength $prefixes]
-  set prefixIndex [expr int(floor(log($value) / log(1024)))]
-  if {$prefixIndex > $maxPrefixIndex} {
-    set prefixIndex $maxPrefixIndex
-  }
-  if {$prefixIndex < 0} {
-    set prefixIndex 0
-  }
-  set scaledValue [expr $value / pow(1024, $prefixIndex)]
-  set l [expr int(floor(log10(abs($scaledValue))))]
-  if {$l < 0} {
-    set l 3
-  } elseif {$l <= 3} {
-    set l [expr 3 - $l]
-  } else {
-    set l 0
+  set prefixIndex 0
+  set l 0
+  set scaledValue $value
+  catch {
+    if {$value > 0} {
+      set prefixIndex [expr int(floor(log($value) / log(1024)))]
+      if {$prefixIndex > $maxPrefixIndex} {
+	set prefixIndex $maxPrefixIndex
+      }
+      if {$prefixIndex < 0} {
+	set prefixIndex 0
+      }
+      set scaledValue [expr $value / pow(1024, $prefixIndex)]
+      set l [expr int(floor(log10(abs($scaledValue))))]
+      if {$l < 0} {
+	set l 3
+      } elseif {$l <= 3} {
+	set l [expr 3 - $l]
+      } else {
+	set l 0
+      }
+    }
   }
   return [format "%.*f %s%s" $l $scaledValue [lindex $prefixes $prefixIndex] $suffix];
 }