RoC: Add checks for hugepage backing of DMA buffer

Author: PascalBoeschoten

CMakeLists.txt

@@ -84,6 +84,7 @@ set(SRCS
   src/ParameterTypes/ResetLevel.cxx
   src/ParameterTypes/ReadoutMode.cxx
   src/RorcStatusCode.cxx
+  src/Utilities/MemoryMaps.cxx
   src/CommandLineUtilities/AliceLowlevelFrontend/Sca.cxx
   src/CommandLineUtilities/AliceLowlevelFrontend/ServiceNames.cxx
   src/CommandLineUtilities/Common.cxx

src/DmaChannelPdaBase.cxx

@@ -5,6 +5,8 @@
 
 #include "DmaChannelPdaBase.h"
 #include <boost/filesystem/path.hpp>
+#include "Common/Iommu.h"
+#include "Utilities/MemoryMaps.h"
 #include "Utilities/SmartPointer.h"
 #include "Utilities/Util.h"
 #include "Visitor.h"
@@ -47,6 +49,34 @@ DmaChannelPdaBase::DmaChannelPdaBase(const Parameters& parameters,
       BOOST_THROW_EXCEPTION(Exception() << ErrorInfo::Message(message));
     }
   }
+
+  // Check memory mappings if it's hugepage
+  {
+    bool checked = false;
+    const auto maps = Utilities::getMemoryMaps();
+    for (const auto& map : maps) {
+      const auto bufferAddress = reinterpret_cast<uintptr_t>(getBufferProvider().getAddress());
+      if (map.addressStart == bufferAddress) {
+        if (map.pageSizeKiB > 4) {
+          log("Buffer is hugepage-backed", InfoLogger::InfoLogger::Info);
+        } else {
+          if (Common::Iommu::isEnabled()) {
+            log("Buffer is NOT hugepage-backed, but IOMMU is enabled", InfoLogger::InfoLogger::Warning);
+          } else {
+            std::string message = "Buffer is NOT hugepage-backed and IOMMU is disabled - unsupported buffer "
+              "configuration";
+            log(message, InfoLogger::InfoLogger::Error);
+            BOOST_THROW_EXCEPTION(Exception() << ErrorInfo::Message(message));
+          }
+        }
+        checked = true;
+        break;
+      }
+    }
+    if (!checked) {
+      log("Failed to check if buffer is hugepage-backed", InfoLogger::InfoLogger::Warning);
+    }
+  }
 }
 
 DmaChannelPdaBase::~DmaChannelPdaBase()

src/Utilities/MemoryMaps.cxx

@@ -3,42 +3,137 @@
 ///
 /// \author Pascal Boeschoten ([email protected])
 
-#pragma once
+#include "MemoryMaps.h"
 
+#include <fstream>
 #include <map>
+#include <sstream>
+#include <string>
+#include <iostream>
 #include <boost/throw_exception.hpp>
-#include <boost/algorithm/string/split.hpp>
+#include <boost/algorithm/string.hpp>
+#include <boost/lexical_cast.hpp>
+#include "Common/System.h"
 
 namespace AliceO2 {
 namespace roc {
 namespace Utilities {
 
-std::vector<MemoryMap> getMemoryMaps()
+namespace {
+std::string slurp(std::string file)
 {
-  std::vector<MemoryMap> memoryMaps;
+  std::ifstream ifstream(file);
+  std::stringstream buffer;
+  buffer << ifstream.rdbuf();
+  return buffer.str();
+}
+
+/// Splits string into rows and "columns"
+std::vector<std::vector<std::string>> tablify(std::string string) {
+  std::vector<std::string> lines;
+  boost::split(lines, string, boost::is_any_of("\n"), boost::token_compress_on);
+  std::vector<std::vector<std::string>> splitLines(lines.size());
+  for (size_t i = 0; i < lines.size(); ++i) {
+    boost::split(splitLines.at(i), lines.at(i), boost::is_any_of(" "), boost::token_compress_on);
+  }
+  return splitLines;
+}
+
+struct Mapping
+{
+    uintptr_t addressStart;
+    uintptr_t addressEnd;
+    std::string permissions;
+    uintptr_t offset;
+    std::string dev;
+    size_t inode;
+    std::string path;
+};
+
+struct NumaMapping
+{
+    std::string path;
+    size_t pageSizeKiB;
+};
+
+std::vector<Mapping> getMaps()
+{
+  std::vector<Mapping> maps;
+  auto procMaps = slurp("/proc/self/maps");
+  auto table = tablify(procMaps);
+
+  for (const auto& row : table) {
+    if (row.size() != 6) {
+      continue;
+    }
+
+    Mapping mapping;
+    std::vector<std::string> addressSplit;
+    boost::split(addressSplit, row.at(0), boost::is_any_of("-"), boost::token_compress_on);
+    mapping.addressStart = std::strtoul(addressSplit.at(0).c_str(), nullptr, 16);
+    mapping.addressEnd = std::strtoul(addressSplit.at(1).c_str(), nullptr, 16);
+    mapping.permissions = boost::trim_copy(row.at(1));
+    mapping.offset = std::strtoul(row.at(2).c_str(), nullptr, 16);
+    mapping.dev = boost::trim_copy(row.at(3));
+    mapping.inode = boost::lexical_cast<size_t>(row.at(4));
+    mapping.path = boost::trim_copy(row.at(5));
+    maps.push_back(mapping);
+  }
 
-  auto procMaps = executeCommand("cat /proc/self/maps");
-  auto procNumaMaps = executeCommand("cat /proc/self/numa_maps");
+  return maps;
+}
+
+std::map<uintptr_t, NumaMapping> getNumaMaps()
+{
+  std::map<uintptr_t, NumaMapping> maps;
+  auto numaMaps = slurp("/proc/self/numa_maps");
+  auto table = tablify(numaMaps);
+
+  for (const auto& row : table) {
+    auto address = std::strtoul(row.at(0).c_str(), nullptr, 16);
 
-  auto splitLines = [](std::string string) {
-    std::vector<std::string> lines;
-    boost::split(lines, string, boost::is_any_of("\n"));
+    NumaMapping mapping;
 
-    std::vector<std::vector<std::string>> splitLines(lines.size());
-    for (int i = 0; i < lines.size(); ++i) {
-      boost::split(splitLines.at(i), lines, boost::is_any_of(" "));
+    for (const auto& item : row) {
+      auto setKeyValue = [&](std::string key, auto setFunction) {
+        if (item.find(key) == 0) {
+          setFunction(item.substr(key.size() + 1)); // Add 1 for the =
+        }
+      };
+
+      setKeyValue("file", [&](auto value){mapping.path = value;});
+      setKeyValue("kernelpagesize_kB", [&](auto value){mapping.pageSizeKiB = boost::lexical_cast<size_t>(value);});
+
+      if (item.find("huge") == 0) {
+        mapping.pageSizeKiB = 2*1024;
+      }
     }
 
-    return splitLines;
-  };
+    maps[address] = mapping;
+  }
+
+  return maps;
+}
+
+
+} // Anonymous namespace
+
+std::vector<MemoryMap> getMemoryMaps()
+{
+  std::vector<MemoryMap> memoryMaps;
 
-  auto mapsTable = splitLines(procMaps);
-  auto numaMapsTable = splitLines(procNumaMaps);
+  const auto maps = getMaps();
+  const auto numaMaps = getNumaMaps();
 
-  for (int i = 0; i < mapsTable.size(); ++i) {
-    std::vector<std::string> memoryRange;
-    const auto& memoryRangeString = mapsTable.at(i).at(0);
-    boost::split(memoryRange, memoryRangeString, boost::is_any_of(" "));
+  for (const auto& map : maps) {
+    MemoryMap memMap;
+    memMap.addressStart = map.addressStart;
+    memMap.addressEnd = map.addressEnd;
+    memMap.path = map.path;
+    if (numaMaps.count(map.addressStart)) {
+      memMap.pageSizeKiB = numaMaps.at(map.addressStart).pageSizeKiB;
+    }
+    memoryMaps.push_back(memMap);
   }
 
   return memoryMaps;

src/Utilities/MemoryMaps.h

@@ -5,17 +5,21 @@
 
 #pragma once
 
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+#include <string>
+
 namespace AliceO2 {
 namespace roc {
 namespace Utilities {
 
 struct MemoryMap
 {
-    uintptr_t startAddress;
-    uintptr_t endAddress;
-    size_t size;
-    bool isHuge; ///< Is hugepage memory
-    size_t kernelPageSize; ///< Size of the
+    uintptr_t addressStart; ///< Starting address of the mapping
+    uintptr_t addressEnd; ///< End address of the mapping
+    size_t pageSizeKiB; ///< Size of the pages. 0 if unknown.
+    std::string path; ///< Pathname of the mapping.
 };
 
 /// TODO Work in progress