FMQ memory checks added

Author: sy-c

doc/releaseNotes.md

@@ -352,3 +352,5 @@ This file describes the main feature changes for each readout.exe released versi
 - FMQ stats not printed when consoleUpdate=1 unless there is a running consumerFMQchannel with disableSending=0.
 - tfRateLimit is handled in the equipment directly and avoid potential issues with timeframe slicing at very slow rates.
 - equipment-cruemulator: TF id extracted from trigger counters (single timer source for improved coherency).
+- Memory allocation policy updated: all readout memory is locked (RAM only, can not be swapped). A warning is reported if not.
+- consumer-FMQchannel: checks are done before FMQ shared memory region is created, to avoid going in a state with over-committed memory (no checks done in FMQ library about the validity of the region created, which can cause severe crash when trying to access it). Both /proc/meminfo (MemFree) and /dev/shm (if using shmem transport type) should report enough available memory before proceeding. Memory is also immediately locked and zeroed to avoid later crashes.

src/ConsumerFMQchannel.cxx

@@ -185,14 +185,47 @@ class ConsumerFMQchannel : public Consumer
     cfg.getOptionalValue<std::string>(cfgEntryPoint + ".unmanagedMemorySize", cfgUnmanagedMemorySize);
     long long mMemorySize = ReadoutUtils::getNumberOfBytesFromString(cfgUnmanagedMemorySize.c_str());
     if (mMemorySize > 0) {
+    
+      // check system resources first, as region creation does not check available memory, so bad crash could occur later      
+      theLog.log(LogInfoDevel_(3002), "Configuring memory buffer %lld MB", (long long)(mMemorySize/1048576LL));
+	    
+      // free system memory
+      unsigned long long freeBytes;
+      if (getStatsFreeMemory(freeBytes)) {
+        theLog.log(LogWarningSupport_(3230), "Can not get stats about system free memory available");
+      } else {
+          theLog.log(LogInfoSupport_(3230), "Stats free memory available: %lld MB", (long long)(freeBytes/1048576LL));
+	  if ((long long)freeBytes < mMemorySize) {
+          theLog.log(LogErrorSupport_(3230), "Not enough system memory available - check /proc/meminfo");
+          throw "ConsumerFMQ: can not allocate shared memory region";
+	} else {
+
+	}
+      }
+            
+      // free SHM memory
+      // check only if transport is of type shmem
+      if (cfgTransportType == "shmem") {
+	if (getStatsFreeSHM(freeBytes)) {
+          theLog.log(LogWarningSupport_(3230), "Can not get stats about shared memory available");
+	} else {
+          theLog.log(LogInfoSupport_(3230), "Stats shared memory available: %lld MB", (long long)(freeBytes/1048576LL));
+          if ((long long)freeBytes < mMemorySize) {
+            theLog.log(LogErrorSupport_(3230), "Not enough shared memory available - check /dev/shm");
+            throw "ConsumerFMQ: can not allocate shared memory region";
+	  }
+	}
+      }
+            
+      theLog.log(LogInfoDevel_(3008), "Creating FMQ unmanaged memory region");
       memoryBuffer = sendingChannel->Transport()->CreateUnmanagedRegion(mMemorySize, [](void* /*data*/, size_t /*size*/, void* hint) { // cleanup callback
         if (hint != nullptr) {
           DataBlockContainerReference* blockRef = (DataBlockContainerReference*)hint;
           //printf("ack hint=%p page %p\n",hint,(*blockRef)->getData());
           decDataBlockStats((*blockRef)->getData());
           delete blockRef;
         }
-      });
+      },"",0,fair::mq::RegionConfig{true, true});  // lock / zero
 
       theLog.log(LogInfoDevel_(3008), "Got FMQ unmanaged memory buffer size %lu @ %p", memoryBuffer->GetSize(), memoryBuffer->GetData());
     }