[pda] Improve logging & add SGL stats

Author: kostorr

src/DmaBufferProvider/PdaDmaBufferProvider.h

@@ -34,8 +34,8 @@ class PdaDmaBufferProvider : public DmaBufferProviderInterface
 {
  public:
   PdaDmaBufferProvider(PciDevice* pciDevice, void* userBufferAddress, size_t userBufferSize,
-                       int dmaBufferId, bool requireHugepage)
-    : mAddress(userBufferAddress), mSize(userBufferSize), mPdaBuffer(pciDevice, userBufferAddress, userBufferSize, dmaBufferId, requireHugepage)
+                       int dmaBufferId, SerialId serialId, bool requireHugepage)
+    : mAddress(userBufferAddress), mSize(userBufferSize), mPdaBuffer(pciDevice, userBufferAddress, userBufferSize, dmaBufferId, serialId, requireHugepage)
   {
   }
 

src/DmaChannelPdaBase.cxx

@@ -24,6 +24,7 @@
 #include "DmaBufferProvider/PdaDmaBufferProvider.h"
 #include "DmaBufferProvider/FilePdaDmaBufferProvider.h"
 #include "DmaBufferProvider/NullDmaBufferProvider.h"
+#include "ReadoutCard/ParameterTypes/SerialId.h"
 #include "Visitor.h"
 
 namespace o2
@@ -56,7 +57,7 @@ DmaChannelPdaBase::DmaChannelPdaBase(const Parameters& parameters,
       [&](buffer_parameters::Memory parameters) {
         log("Initializing with DMA buffer from memory region", LogDebugDevel);
         return std::make_unique<PdaDmaBufferProvider>(mRocPciDevice->getPciDevice(), parameters.address,
-                                                      parameters.size, bufferId, true);
+                                                      parameters.size, bufferId, mRocPciDevice->getSerialId(), true);
       },
       [&](buffer_parameters::File parameters) {
         log("Initializing with DMA buffer from memory-mapped file", LogDebugDevel);
@@ -78,12 +79,11 @@ DmaChannelPdaBase::DmaChannelPdaBase(const Parameters& parameters,
     auto listSize = mBufferProvider->getScatterGatherListSize();
     auto hugePageMinSize = 1024 * 1024 * 2; // 2 MiB, the smallest hugepage size
     auto bufferSize = getBufferProvider().getSize();
-    //log(std::string("Scatter-gather list size: ") + std::to_string(listSize));
     if (listSize > (bufferSize / hugePageMinSize)) {
       std::string message =
         "Scatter-gather list size greater than buffer size divided by 2MiB (minimum hugepage size)."
         " This means the IOMMU is off and the buffer is not backed by hugepages - an unsupported buffer configuration.";
-      log(message, LogErrorDevel); //TODO: Why log + throw?
+      log(message, LogErrorDevel);
       BOOST_THROW_EXCEPTION(Exception() << ErrorInfo::Message(message));
     }
   }
@@ -92,20 +92,22 @@ DmaChannelPdaBase::DmaChannelPdaBase(const Parameters& parameters,
   if (getBufferProvider().getSize() > 0) {
     // Non-null buffer
     bool checked = false;
+    // Get the memory mappings from linux
     const auto maps = Utilities::getMemoryMaps();
+    const auto bufferAddress = reinterpret_cast<uintptr_t>(getBufferProvider().getAddress());
     for (const auto& map : maps) {
-      const auto bufferAddress = reinterpret_cast<uintptr_t>(getBufferProvider().getAddress());
+      // Match the map address with the provided buffer address
       if (map.addressStart == bufferAddress) {
         if (map.pageSizeKiB > 4) {
-          log("Buffer is hugepage-backed", LogDebugDevel);
+          log("Buffer is hugepage-backed", LogDebugTrace);
         } else {
           if (AliceO2::Common::Iommu::isEnabled()) {
-            log("Buffer is NOT hugepage-backed, but IOMMU is enabled", LogWarningDevel);
+            log("Buffer is NOT hugepage-backed, but IOMMU is enabled", LogWarningTrace);
           } else {
             std::string message =
               "Buffer is NOT hugepage-backed and IOMMU is disabled - unsupported buffer "
               "configuration";
-            log(message, LogErrorDevel); //TODO: Why log + throw?
+            log(message, LogWarningDevel);
             BOOST_THROW_EXCEPTION(Exception() << ErrorInfo::Message(message)
                                               << ErrorInfo::PossibleCauses({ "roc-setup-hugetlbfs was not run" }));
           }
@@ -115,7 +117,7 @@ DmaChannelPdaBase::DmaChannelPdaBase(const Parameters& parameters,
       }
     }
     if (!checked) {
-      log("Failed to check if buffer is hugepage-backed", LogWarningDevel);
+      log("Failed to check if buffer is hugepage-backed", LogDebugTrace);
     }
   }
 }

src/Pda/PdaDmaBuffer.cxx

@@ -15,6 +15,7 @@
 /// \author Pascal Boeschoten ([email protected])
 /// \author Kostas Alexopoulos ([email protected])
 
+#include <numeric>
 #include "PdaDmaBuffer.h"
 #include <pda.h>
 #include "ExceptionInternal.h"
@@ -30,7 +31,7 @@ namespace Pda
 {
 
 PdaDmaBuffer::PdaDmaBuffer(PciDevice* pciDevice, void* userBufferAddress, size_t userBufferSize,
-                           int dmaBufferId, bool requireHugepage) : mPciDevice(pciDevice)
+                           int dmaBufferId, SerialId serialId, bool requireHugepage) : mPciDevice(pciDevice)
 {
   // Safeguard against PDA kernel module deadlocks, since it does not like parallel buffer registration
   try {
@@ -83,15 +84,13 @@ PdaDmaBuffer::PdaDmaBuffer(PciDevice* pciDevice, void* userBufferAddress, size_t
     }
 
     auto node = sgList;
+    std::vector<size_t> nodeSizes;
     while (node != nullptr) {
-      if (requireHugepage) {
-        size_t hugePageMinSize = 1024 * 1024 * 2; // 2 MiB, the smallest hugepage size
-        if (node->length < hugePageMinSize) {
-          BOOST_THROW_EXCEPTION(
-            PdaException() << ErrorInfo::Message("Scatter-gather node smaller than 2 MiB (minimum hugepage"
-                                                 " size. This means the IOMMU is off and the buffer is not backed by hugepages - an unsupported buffer "
-                                                 "configuration."));
-        }
+      nodeSizes.emplace_back(node->length);
+      size_t hugePageMinSize = 1024 * 1024 * 2; // 2 MiB, the smallest hugepage size
+      if (requireHugepage && node->length < hugePageMinSize) {
+        BOOST_THROW_EXCEPTION(
+          PdaException() << ErrorInfo::Message("SGL node smaller than 2 MiB. IOMMU off and buffer not backed by hugapages - unsupported buffer configuration"));
       }
 
       ScatterGatherEntry e;
@@ -107,6 +106,20 @@ PdaDmaBuffer::PdaDmaBuffer(PciDevice* pciDevice, void* userBufferAddress, size_t
       BOOST_THROW_EXCEPTION(PdaException() << ErrorInfo::Message(
                               "Failed to initialize scatter-gather list, was empty"));
     }
+
+    // Print some stats regarding the Scatter-Gather list
+    std::sort(nodeSizes.begin(), nodeSizes.end());
+    int n = nodeSizes.size();
+    int minSize = nodeSizes[0];
+    int maxSize = nodeSizes[n - 1];
+    double median = nodeSizes[n / 2];
+    if (n % 2 == 0) {
+      median = (median + nodeSizes[n / 2 - 1]) / 2;
+    }
+    int totalSize = std::accumulate(nodeSizes.begin(), nodeSizes.end(), 0);
+
+    Logger::get() << "[" << serialId << " |"
+                  << " PDA buffer SGL stats] #nodes: " << n << " | total: " << totalSize << " | min: " << minSize << " | max: " << maxSize << " | median: " << median << LogInfoDevel << endm;
   } catch (const PdaException&) {
     PciDevice_deleteDMABuffer(mPciDevice, mDmaBuffer);
     throw;

src/Pda/PdaDmaBuffer.h

@@ -20,6 +20,7 @@
 #include <vector>
 #include <pda.h>
 #include "Pda/PdaDevice.h"
+#include "ReadoutCard/ParameterTypes/SerialId.h"
 
 namespace o2
 {
@@ -40,7 +41,7 @@ class PdaDmaBuffer
   /// \param dmaBufferId Unique ID to use for registering the buffer (uniqueness must be card-wide)
   /// \param requireHugepage Require the buffer to have hugepage-sized scatter-gather list nodes
   PdaDmaBuffer(PciDevice* pciDevice, void* userBufferAddress, size_t userBufferSize,
-               int dmaBufferId, bool requireHugepage = true);
+               int dmaBufferId, SerialId serialId, bool requireHugepage = true);
 
   ~PdaDmaBuffer();