[roc-cleanup] Update tool to clean stale hugepage / uio_pci_dma resources

Author: kostorr

CMakeLists.txt

@@ -214,7 +214,7 @@ if(PDA_FOUND)
   list(APPEND EXE_NAMES
     roc-bar-stress
     roc-config
-    roc-channel-cleanup
+    roc-cleanup
     roc-example
     roc-flash
     roc-flash-read

README.md

@@ -300,9 +300,10 @@ It may use files in these directories for DMA buffers:
 The program will report the exact file used. 
 They can be inspected manually if needed, e.g. with hexdump: `hexdump -e '"%07_ax" " | " 4/8 "%08x " "\n"' [filename]`
 
-### roc-channel-cleanup
-In the event of a serious crash, such as a segfault, it may be necessary to clean up and reset a channel.
-See section "Channel ownership lock" for more details.
+### roc-cleanup
+In the event of a serious crash, such as a segfault, it may be necessary to clean up and reset.
+This tool serves this purpose and is intended to be run as root. Be aware that this will make every
+running instance of readout.exe or roc-bench-dma fail.
 
 ### roc-config
 Configures the CRU. Uses the [Card Configurator](#card-configurator).

src/CommandLineUtilities/ProgramCleanup.cxx

@@ -9,10 +9,12 @@
 // or submit itself to any jurisdiction.
 
 /// \file ProgramCleanup.cxx
-/// \brief Utility that cleans up channel state
+/// \brief Utility that cleans up stale readout/readoutcard resources
 ///
 /// \author Pascal Boeschoten ([email protected])
+/// \author Kostas Alexopoulos ([email protected])
 
+#include <boost/algorithm/string/predicate.hpp>
 #include <iostream>
 #include "ReadoutCard/ChannelFactory.h"
 #include "CommandLineUtilities/Common.h"
@@ -21,48 +23,59 @@
 
 using namespace AliceO2::roc::CommandLineUtilities;
 using namespace AliceO2::roc;
-using std::cout;
-using std::endl;
+namespace algo = boost::algorithm;
 namespace po = boost::program_options;
 
-namespace {
-
-class ProgramCleanup: public Program
+class ProgramCleanup : public Program
 {
-  public:
+ public:
+  virtual Description getDescription()
+  {
+    return { "Cleanup", "Cleans up ReadoutCard state", "roc-cleanup" };
+  }
 
-    virtual Description getDescription()
-    {
-      return {"Cleanup", "Cleans up ReadoutCard state", "roc-cleanup --id=12345 --channel=0"};
-    }
+  virtual void addOptions(po::options_description&)
+  {
+  }
 
-    virtual void addOptions(po::options_description& options)
-    {
-      Options::addOptionCardId(options);
-      Options::addOptionChannel(options);
-      //options.add_options()("force",po::bool_switch(&mForceCleanup),
-       //   "Force cleanup of shared state files if normal cleanup fails");
+  virtual void run(const po::variables_map&)
+  {
+    std::cout << "\033[1;31m"
+              << "!!! WARNING !!!"
+              << "\033[0m" << std::endl;
+    std::cout << std::endl;
+    std::cout << "Execution of this tool will:" << std::endl;
+    std::cout << "1. Clean all hugepage resources under /var/lib/hugetlbfs/global/pagesize-{2MB, 1GB}/ which match readout* and roc-bench-dma*" << std::endl;
+    std::cout << "2. Remove and reinsert the uio_pci_dma kernel module" << std::endl;
+    std::cout << std::endl;
+    std::cout << "In case instances of readout.exe or roc-bench-dma are running, they will fail." << std::endl;
+    std::cout << std::endl;
+    std::cout << "This tool is intended to be run with elevated privileges." << std::endl;
+    std::cout << "Are you sure you want to continue? (yes/no)" << std::endl;
+    std::string response;
+    std::cin >> response;
+    if (!algo::starts_with(response, "y")) {
+      std::cout << "Terminated" << std::endl;
+      return;
     }
 
-    virtual void run(const boost::program_options::variables_map& map)
-    {
-      auto cardId = Options::getOptionCardId(map);
-      auto channelNumber = Options::getOptionChannel(map);
+    std::cout << "Removing readout 2MB hugepage mappings" << std::endl;
+    system("rm /var/lib/hugetlbfs/global/pagesize-2MB/readout*");
+    std::cout << "Removing readout 1GB hugepage mappings" << std::endl;
+    system("rm /var/lib/hugetlbfs/global/pagesize-1GB/readout*");
+    std::cout << "Removing roc-bench-dma 2MB hugepage mappings" << std::endl;
+    system("rm /var/lib/hugetlbfs/global/pagesize-2MB/roc-bench-dma*");
+    std::cout << "Removing roc-bench-dma 1GB hugepage mappings" << std::endl;
+    system("rm /var/lib/hugetlbfs/global/pagesize-1GB/roc-bench-dma*");
 
-      // This non-forced cleanup asks the DmaChannel to clean up itself.
-      // It will not succeed if the channel was not initialized properly before the running of this program.
-      cout << "### Attempting cleanup...\n";
-      auto params = AliceO2::roc::Parameters::makeParameters(cardId, channelNumber);
-      //params.setForcedUnlockEnabled(mForceCleanup);
-      params.setBufferParameters(buffer_parameters::Null());
-      auto channel = ChannelFactory().getDmaChannel(params);
-      cout << "### Done!\n";
-    }
+    std::cout << "Removing uio_pci_dma" << std::endl;
+    system("modprobe -r uio_pci_dma");
+    std::cout << "Reinserting uio_pci_dma" << std::endl;
+    system("modprobe uio_pci_dma");
+  }
 
-  private:
-    //bool mForceCleanup;
+ private:
 };
-} // Anonymous namespace
 
 int main(int argc, char** argv)
 {

src/MemoryMappedFile.cxx

@@ -21,19 +21,20 @@
 #include "ExceptionInternal.h"
 #include "Utilities/SmartPointer.h"
 
-namespace AliceO2 {
-namespace roc {
+namespace AliceO2
+{
+namespace roc
+{
 
 namespace b = boost;
 namespace bip = boost::interprocess;
 namespace bfs = boost::filesystem;
 
-struct MemoryMappedFileInternal
-{
-    std::string fileName;
-    boost::interprocess::file_mapping fileMapping;
-    boost::interprocess::mapped_region mappedRegion;
-    bool deleteFileOnDestruction;
+struct MemoryMappedFileInternal {
+  std::string fileName;
+  boost::interprocess::file_mapping fileMapping;
+  boost::interprocess::mapped_region mappedRegion;
+  bool deleteFileOnDestruction;
 };
 
 MemoryMappedFile::MemoryMappedFile()
@@ -42,26 +43,26 @@ MemoryMappedFile::MemoryMappedFile()
 }
 
 MemoryMappedFile::MemoryMappedFile(const std::string& fileName, size_t fileSize, bool deleteFileOnDestruction,
-    bool lockMap)
-    : MemoryMappedFile()
+                                   bool lockMap)
+  : MemoryMappedFile()
 {
   mInternal->fileName = fileName;
   mInternal->deleteFileOnDestruction = deleteFileOnDestruction;
 
   if (lockMap) {
     // Try to acquire the lock on the file
-    try{
+    try {
       Utilities::resetSmartPtr(mInterprocessLock, "Alice_O2_RoC_MMF_" + fileName + "_lock");
     } catch (const boost::exception& e) {
       BOOST_THROW_EXCEPTION(LockException()
-          << ErrorInfo::Message("Couldn't lock Memory Mapped File; " + boost::diagnostic_information(e)));
+                            << ErrorInfo::Message("Couldn't lock Memory Mapped File; " + boost::diagnostic_information(e)));
     }
 
     try {
       mMapAcquired = map(fileName, fileSize);
     } catch (const boost::exception& e) {
       BOOST_THROW_EXCEPTION(MemoryMapException()
-          << ErrorInfo::Message(boost::diagnostic_information(e)));
+                            << ErrorInfo::Message(boost::diagnostic_information(e)));
     }
   }
 }
@@ -96,11 +97,11 @@ bool MemoryMappedFile::map(const std::string& fileName, size_t fileSize)
       auto dir = bfs::path(fileName.c_str()).parent_path();
       if (!(bfs::is_directory(dir) && bfs::exists(dir))) {
         BOOST_THROW_EXCEPTION(MemoryMapException()
-            << ErrorInfo::Message("Failed to open memory map file, parent directory does not exist"));
+                              << ErrorInfo::Message("Failed to open memory map file, parent directory does not exist"));
       }
     }
 
-    // We don't care if the file exists. 
+    // We don't care if the file exists.
     // Locks are in place that make sure we don't get here unless we're allowed to
     // Check the file exists
     /*{
@@ -113,38 +114,35 @@ bool MemoryMappedFile::map(const std::string& fileName, size_t fileSize)
     // Similar operation to calling "touch" command, making sure the file exists
     try {
       std::ofstream ofs(fileName.c_str(), std::ios::app);
-    }
-    catch (const std::exception& e) {
+    } catch (const std::exception& e) {
       BOOST_THROW_EXCEPTION(MemoryMapException()
-          << ErrorInfo::Message(std::string("Failed to open memory map file: ") + e.what()));
+                            << ErrorInfo::Message(std::string("Failed to open memory map file: ") + e.what()));
     }
 
     // Resize and map file to memory
     try {
       bfs::resize_file(fileName.c_str(), fileSize);
-    }
-    catch (const std::exception& e) {
+    } catch (const std::exception& e) {
       BOOST_THROW_EXCEPTION(MemoryMapException()
-          << ErrorInfo::Message(std::string("Failed to resize memory map file: ") + e.what())
-          << ErrorInfo::PossibleCauses({
-            "Size not a multiple of page size",
-            "Not enough memory available",
-            "Not enough memory available (check 'hugeadm --pool-list')",
-            "Insufficient permissions"}));
+                            << ErrorInfo::Message(std::string("Failed to resize memory map file: ") + e.what())
+                            << ErrorInfo::PossibleCauses({ "Size not a multiple of page size",
+                                                           "Not enough memory available",
+                                                           "Not enough hugepages allocated (check 'hugeadm --pool-list')",
+                                                           "Insufficient permissions",
+                                                           "Stale hugepage / uio_pci_dma resources (run 'roc-cleanup')" }));
     }
 
     try {
       mInternal->fileMapping = bip::file_mapping(fileName.c_str(), bip::read_write);
       mInternal->mappedRegion = bip::mapped_region(mInternal->fileMapping, bip::read_write, 0, fileSize);
     } catch (const std::exception& e) {
       BOOST_THROW_EXCEPTION(MemoryMapException()
-          << ErrorInfo::Message(std::string("Failed to memory map file: ") + e.what())
-          << ErrorInfo::PossibleCauses({
-            "Not enough memory available",
-            "Not enough hugepages allocated (check 'hugeadm --pool-list')"}));
+                            << ErrorInfo::Message(std::string("Failed to memory map file: ") + e.what())
+                            << ErrorInfo::PossibleCauses({ "Not enough memory available",
+                                                           "Not enough hugepages allocated (check 'hugeadm --pool-list')",
+                                                           "Stale hugepage / uio_pci_dma resources (run 'roc-cleanup')" }));
     }
-  }
-  catch (MemoryMapException& e) {
+  } catch (MemoryMapException& e) {
     e << ErrorInfo::FileName(fileName) << ErrorInfo::FileSize(fileSize);
     throw;
   }