refactor: Improved FPE mon, extend to macOS

.github/workflows/builds.yml

@@ -20,6 +20,7 @@ env:
   CCACHE_DIR: ${{ github.workspace }}/ccache
   CCACHE_MAXSIZE: 500M
   CCACHE_KEY_SUFFIX: r2
+  ACTS_SEQUENCER_FAIL_ON_UNMASKED_FPE: 0
 
 jobs:
   linux_ubuntu:
@@ -120,7 +121,7 @@ jobs:
     container: ghcr.io/acts-project/ubuntu2404:83
     needs: [linux_ubuntu]
     env:
-      ACTS_SEQUENCER_DISABLE_FPEMON: true
+      ACTS_SEQUENCER_FAIL_ON_UNMASKED_FPE: 0
 
     steps:
       - uses: actions/checkout@v6
@@ -158,8 +159,6 @@ jobs:
     runs-on: ubuntu-latest
     container: ghcr.io/acts-project/ubuntu2404:83
     needs: [linux_ubuntu]
-    env:
-      ACTS_SEQUENCER_DISABLE_FPEMON: true
 
     steps:
       - uses: actions/checkout@v6

.gitlab-ci.yml

@@ -395,8 +395,7 @@ linux_ubuntu_2404_clang19:
 
   variables:
     ACTS_LOG_FAILURE_THRESHOLD: WARNING
-    # disable fpe monitoring as we can't mask without backtrace support
-    ACTS_SEQUENCER_DISABLE_FPEMON: true
+    ACTS_SEQUENCER_FAIL_ON_UNMASKED_FPE: 0
     INSTALL_DIR: ${CI_PROJECT_DIR}/install
     # force off even if it is CI mode
     ROOT_HASH_CHECKS: off

CMakePresets.json

@@ -59,6 +59,7 @@
                 "ACTS_BUILD_PLUGIN_ACTSVG": "ON",
                 "ACTS_BUILD_PLUGIN_DD4HEP": "ON",
                 "ACTS_BUILD_PLUGIN_EDM4HEP": "ON",
+                "ACTS_BUILD_PLUGIN_FPEMON": "ON",
                 "ACTS_BUILD_PLUGIN_GEOMODEL": "ON",
                 "ACTS_BUILD_PLUGIN_TRACCC": "ON",
                 "ACTS_BUILD_PLUGIN_GEANT4": "ON",
@@ -110,8 +111,7 @@
             "displayName": "GitLab-CI",
             "inherits": "ci-common",
             "cacheVariables": {
-                "CMAKE_BUILD_TYPE": "RelWithDebInfo",
-                "ACTS_BUILD_PLUGIN_FPEMON": "ON"
+                "CMAKE_BUILD_TYPE": "RelWithDebInfo"
             }
         },
         {
@@ -148,6 +148,7 @@
                 "ACTS_FORCE_ASSERTIONS": "ON",
                 "ACTS_ENABLE_LOG_FAILURE_THRESHOLD": "ON",
                 "ACTS_BUILD_PYTHON_BINDINGS": "ON",
+                "ACTS_BUILD_PLUGIN_FPEMON": "ON",
                 "ACTS_BUILD_PLUGIN_GNN": "ON",
                 "ACTS_BUILD_EXAMPLES": "ON",
                 "ACTS_BUILD_EXAMPLES_GNN": "ON",
@@ -168,6 +169,7 @@
                 "CMAKE_CXX_COMPILER_LAUNCHER": "ccache",
                 "CMAKE_CUDA_ARCHITECTURES": "86",
                 "TORCH_CUDA_ARCH_LIST": "8.6",
+                "ACTS_BUILD_PLUGIN_FPEMON": "ON",
                 "ACTS_BUILD_PLUGIN_GNN": "ON",
                 "ACTS_GNN_ENABLE_TORCH": "OFF",
                 "ACTS_GNN_ENABLE_CUDA": "ON",
@@ -192,7 +194,7 @@
                 "ACTS_BUILD_PYTHON_BINDINGS": "ON",
                 "ACTS_BUILD_EXAMPLES": "ON",
                 "ACTS_BUILD_PLUGIN_ROOT": "OFF",
-                "ACTS_BUILD_PLUGIN_FPEMON": "OFF",
+                "ACTS_BUILD_PLUGIN_FPEMON": "ON",
                 "ACTS_BUILD_EXAMPLES_ROOT": "OFF",
                 "ACTS_USE_SYSTEM_NLOHMANN_JSON": "ON",
                 "ACTS_USE_SYSTEM_PYBIND11": "ON"

Examples/Framework/include/ActsExamples/Framework/Sequencer.hpp

@@ -77,9 +77,16 @@ class Sequencer {
     /// @warning This function can be called from multiple threads and should therefore be thread-safe
     IterationCallback iterationCallback = []() {};
 
+    /// If true, FPEs are tracked.
     bool trackFpes = true;
+    /// If true, FPEs are masked and reported.
     std::vector<FpeMask> fpeMasks{};
+    /// If true, the first FPE encountered makes Sequencer::run() fail.
     bool failOnFirstFpe = false;
+    /// If false, unmasked FPEs are reported but do not make Sequencer::run()
+    /// fail.
+    bool failOnUnmaskedFpe = true;
+    /// The number of stack frames to include in the FPE report.
     std::size_t fpeStackTraceLength = 8;
   };
 

Examples/Framework/src/Framework/Sequencer.cpp

@@ -58,6 +58,26 @@
   return res;
 }
 
+std::optional<bool> parseBoolEnv(const char* envName) {
+  const char* rawValue = std::getenv(envName);
+  if (rawValue == nullptr) {
+    return std::nullopt;
+  }
+
+  std::string value = rawValue;
+  boost::algorithm::trim(value);
+  boost::algorithm::to_lower(value);
+  if (value == "1" || value == "true" || value == "yes" || value == "on") {
+    return true;
+  }
+  if (value == "0" || value == "false" || value == "no" || value == "off") {
+    return false;
+  }
+  throw SequenceConfigurationException(
+      std::string{"Unable to parse "} + envName + " value '" + rawValue +
+      "'. Supported values are: 0/1/false/true/no/yes/off/on");
+}
+
 }  // namespace
 
 Sequencer::Sequencer(const Sequencer::Config& cfg)
@@ -80,12 +100,23 @@
     ACTS_INFO("Create Sequencer with " << m_cfg.numThreads << " threads");
   }
 
-  const char* envvar = std::getenv("ACTS_SEQUENCER_DISABLE_FPEMON");
-  if (envvar != nullptr) {
+  if (auto disableFpeEnv = parseBoolEnv("ACTS_SEQUENCER_DISABLE_FPEMON");
+      disableFpeEnv.has_value()) {
+    m_cfg.trackFpes = !disableFpeEnv.value();
     ACTS_INFO(
-        "Overriding FPE tracking Sequencer based on environment variable "
-        "ACTS_SEQUENCER_DISABLE_FPEMON");
-    m_cfg.trackFpes = false;
+        "FPE tracking is "
+        << (m_cfg.trackFpes ? "enabled" : "disabled")
+        << " based on environment variable ACTS_SEQUENCER_DISABLE_FPEMON");
+  }
+
+  if (auto failUnmaskedEnv =
+          parseBoolEnv("ACTS_SEQUENCER_FAIL_ON_UNMASKED_FPE");
+      failUnmaskedEnv.has_value()) {
+    m_cfg.failOnUnmaskedFpe = failUnmaskedEnv.value();
+    ACTS_INFO("Sequencer failOnUnmaskedFpe is "
+              << (m_cfg.failOnUnmaskedFpe ? "enabled" : "disabled")
+              << " based on environment variable "
+                 "ACTS_SEQUENCER_FAIL_ON_UNMASKED_FPE");
   }
 
   if (m_cfg.trackFpes && !m_cfg.fpeMasks.empty() &&
@@ -512,26 +543,29 @@
               if (mon) {
                 auto& local = fpe->local();
 
-                for (const auto& [count, type, st] :
-                     mon->result().stackTraces()) {
-                  auto [maskLoc, nMasked] = fpeMaskCount(*st, type);
+                for (const auto& info : mon->result().stackTraces()) {
+                  const auto count = info.count;
+                  const auto type = info.type;
+                  const auto& st = *info.st;
+                  auto [maskLoc, nMasked] = fpeMaskCount(st, type);
                   if (nMasked < count) {
                     std::stringstream ss;
                     ss << "FPE of type " << type
                        << " exceeded configured per-event threshold of "
                        << nMasked << " (mask: " << maskLoc
                        << ") (seen: " << count << " FPEs)\n"
                        << ActsPlugins::FpeMonitor::stackTraceToString(
-                              *st, m_cfg.fpeStackTraceLength);
+                              st, m_cfg.fpeStackTraceLength);
 
                     m_nUnmaskedFpe += (count - nMasked);
 
-                    if (m_cfg.failOnFirstFpe) {
+                    if (m_cfg.failOnFirstFpe && m_cfg.failOnUnmaskedFpe) {
                       ACTS_ERROR(ss.str());
                       local.merge(mon->result());  // merge so we get correct
                                                    // results after throwing
                       throw FpeFailure{ss.str()};
-                    } else if (!local.contains(type, *st)) {
+                    } else if (m_cfg.failOnUnmaskedFpe &&
+                               !local.contains(info)) {
                       ACTS_INFO(ss.str());
                     }
                   }
@@ -607,7 +641,7 @@
                 joinPaths(m_cfg.outputDir, m_cfg.outputTimingFile));
   }
 
-  if (m_nUnmaskedFpe > 0) {
+  if (m_cfg.failOnUnmaskedFpe && m_nUnmaskedFpe > 0) {
     return EXIT_FAILURE;
   }
 
@@ -644,20 +678,25 @@
         remaining;
 
     for (const auto& el : sorted) {
-      const auto& [count, type, st] = el.get();
-      auto [maskLoc, nMasked] = fpeMaskCount(*st, type);
+      const auto& info = el.get();
+      const auto count = info.count;
+      const auto type = info.type;
+      const auto& st = *info.st;
+      auto [maskLoc, nMasked] = fpeMaskCount(st, type);
       ACTS_INFO("- " << type << ": (" << count << " times) "
                      << (nMasked > 0 ? "[MASKED: " + std::to_string(nMasked) +
                                            " per event by " + maskLoc + "]"
                                      : "")
                      << "\n"
                      << ActsPlugins::FpeMonitor::stackTraceToString(
-                            *st, m_cfg.fpeStackTraceLength));
+                            st, m_cfg.fpeStackTraceLength));
     }
   }
 
   if (m_nUnmaskedFpe > 0) {
-    ACTS_ERROR("Encountered " << m_nUnmaskedFpe << " unmasked FPEs");
+    Acts::Logging::Level level =
+        m_cfg.failOnUnmaskedFpe ? Acts::Logging::ERROR : Acts::Logging::INFO;
+    ACTS_LOG(level, "Encountered " << m_nUnmaskedFpe << " unmasked FPEs");
   } else {
     ACTS_INFO("No unmasked FPEs encountered");
   }

Plugins/FpeMonitoring/CMakeLists.txt

@@ -1,6 +1,22 @@
+set(_fpe_platform_source src/FpeMonitorPlatformUnsupported.cpp)
+string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _acts_fpe_arch)
+
+if(APPLE)
+    if(_acts_fpe_arch MATCHES "^(x86_64|amd64)$")
+        set(_fpe_platform_source src/FpeMonitorPlatformDarwinX86_64.cpp)
+    elseif(_acts_fpe_arch MATCHES "^(arm64|aarch64)$")
+        set(_fpe_platform_source src/FpeMonitorPlatformDarwinArm64.cpp)
+    endif()
+elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    if(_acts_fpe_arch MATCHES "^(x86_64|amd64)$")
+        set(_fpe_platform_source src/FpeMonitorPlatformLinuxX86_64.cpp)
+    endif()
+endif()
+
 acts_add_library(
     PluginFpeMonitoring
     src/FpeMonitor.cpp
+    ${_fpe_platform_source}
     ACTS_INCLUDE_FOLDER include/ActsPlugins
 )
 

Plugins/FpeMonitoring/include/ActsPlugins/FpeMonitoring/FpeMonitor.hpp

@@ -12,6 +12,7 @@
 #include <atomic>
 #include <csignal>
 #include <cstddef>
+#include <cstdint>
 #include <limits>
 #include <memory>
 #include <mutex>
@@ -75,7 +76,7 @@ class FpeMonitor {
     /// @param offset Number of bytes to advance
     void pushOffset(std::size_t offset) {
       assert(m_offset + offset < m_size);
-      m_offset = offset;
+      m_offset += offset;
     }
 
     /// Reset buffer offset to beginning
@@ -108,13 +109,17 @@ class FpeMonitor {
       FpeType type;
       /// Stack trace where the exception occurred
       std::shared_ptr<const boost::stacktrace::stacktrace> st;
+      /// Faulting instruction address if available from signal context
+      std::uintptr_t location;
 
       /// Constructor
       /// @param countIn Number of occurrences
       /// @param typeIn Exception type
       /// @param stIn Stack trace
+      /// @param locationIn Faulting instruction address if available
       FpeInfo(std::size_t countIn, FpeType typeIn,
-              std::shared_ptr<const boost::stacktrace::stacktrace> stIn);
+              std::shared_ptr<const boost::stacktrace::stacktrace> stIn,
+              std::uintptr_t locationIn = 0);
       ~FpeInfo();
     };
 
@@ -145,11 +150,10 @@ class FpeMonitor {
     /// Remove duplicate stack traces
     void deduplicate();
 
-    /// Check if result contains a specific exception type and stack trace
-    /// @param type Exception type
-    /// @param st Stack trace to check
+    /// Check if result contains an exception info entry under merge semantics
+    /// @param info Exception info to check
     /// @return True if contained
-    bool contains(FpeType type, const boost::stacktrace::stacktrace &st) const;
+    bool contains(const FpeInfo &info) const;
 
     /// Print summary of exceptions
     /// @param os Output stream
@@ -168,7 +172,9 @@ class FpeMonitor {
     /// @param type Exception type
     /// @param stackPtr Pointer to stack data
     /// @param bufferSize Size of stack buffer
-    void add(FpeType type, void *stackPtr, std::size_t bufferSize);
+    /// @param location Faulting instruction address if available
+    void add(FpeType type, void *stackPtr, std::size_t bufferSize,
+             std::uintptr_t location = 0);
 
    private:
     std::vector<FpeInfo> m_stackTraces;
@@ -210,6 +216,9 @@ class FpeMonitor {
   /// Check if stack trace symbolization is available
   /// @return True if symbolization is available
   static bool canSymbolize();
+  /// Check if trapping-based FPE monitoring is supported on this platform
+  /// @return True if runtime support is available
+  static bool isSupported();
 
  private:
   void enable();
@@ -232,7 +241,8 @@ class FpeMonitor {
 
   Buffer m_buffer{65536};
 
-  boost::container::static_vector<std::tuple<FpeType, void *, std::size_t>, 128>
+  boost::container::static_vector<
+      std::tuple<FpeType, void *, std::size_t, std::uintptr_t>, 128>
       m_recorded;
 };