fix assertion when isolated cpus are used together with manual thread affinities

Author: Pavel Harbanau

apps/gnb/gnb_appconfig_cli11_schema.cpp

@@ -1842,6 +1842,17 @@ static void configure_cli11_hal_args(CLI::App& app, optional<hal_appconfig>& con
   app.add_option("--eal_args", config->eal_args, "EAL configuration parameters used to initialize DPDK");
 }
 
+static void validate_cpu_range(const os_sched_affinity_bitmask& allowed_cpus_mask,
+                               const os_sched_affinity_bitmask& mask,
+                               const std::string&               name)
+{
+  for (unsigned i = 0; i != mask.size(); ++i) {
+    if (mask.test(i) && !allowed_cpus_mask.test(i)) {
+      report_error("CPU core {} selected in '{}' option doesn't belong to isolated cpuset.", i, name);
+    }
+  }
+}
+
 static error_type<std::string> is_valid_cpu_index(unsigned cpu_idx)
 {
   unsigned nof_cpus = compute_host_nof_hardware_threads();
@@ -2036,6 +2047,26 @@ static void configure_cli11_affinity_args(CLI::App& app, cpu_affinities_appconfi
         parsing_isolated_cpus_fcn(config, value, "isolated_cpus");
       },
       "CPU cores isolated for gNB application");
+
+  // Callback function for validating that thread affinities do use allowed set of CPUs in case the 'isolated_cpus'
+  // option was specified.
+  auto validate_isolation = [parsing_fcn](CLI::App& cli_app, cpu_affinities_appconfig& cfg) {
+    unsigned isol_cpus_option_count = cli_app.count("--isolated_cpus");
+    if (!isol_cpus_option_count) {
+      return;
+    }
+    os_sched_affinity_bitmask isol_cpus_bitmask;
+    parsing_fcn(isol_cpus_bitmask, cfg.isol_cpus.value().isolated_cpus, "isolated_cpus");
+
+    validate_cpu_range(isol_cpus_bitmask, cfg.l1_dl_cpu_cfg.mask, "l1_dl_cpus");
+    validate_cpu_range(isol_cpus_bitmask, cfg.l1_ul_cpu_cfg.mask, "l1_ul_cpus");
+    validate_cpu_range(isol_cpus_bitmask, cfg.l2_cell_cpu_cfg.mask, "l2_cell_cpus");
+    validate_cpu_range(isol_cpus_bitmask, cfg.ru_cpu_cfg.mask, "ru_cpus");
+    validate_cpu_range(isol_cpus_bitmask, cfg.low_priority_cpu_cfg.mask, "low_priority_cpus");
+  };
+
+  // Post-parsing callback for the case when both manual pinning and isolated_cpus were used.
+  app.callback([&]() { validate_isolation(app, config); });
 }
 
 static void configure_cli11_upper_phy_threads_args(CLI::App& app, upper_phy_threads_appconfig& config)

apps/gnb/gnb_os_sched_affinity_manager.h

@@ -56,7 +56,7 @@ struct gnb_os_sched_affinity_config {
   /// Affinity mask.
   os_sched_affinity_bitmask mask;
   /// Thread pinning policy.
-  gnb_sched_affinity_mask_policy pinning_policy = gnb_sched_affinity_mask_policy::round_robin;
+  gnb_sched_affinity_mask_policy pinning_policy = gnb_sched_affinity_mask_policy::mask;
 };
 
 /// \brief Scheduler affinity mask manager.
@@ -101,7 +101,7 @@ class gnb_os_sched_affinity_manager
         last_pos_used = (pos < 0) ? 0 : pos + 1;
       }
 
-      return os_sched_affinity_bitmask(pos);
+      return os_sched_affinity_bitmask(mask.size(), pos);
     }
 
   private:

include/srsran/support/unique_thread.h

@@ -99,6 +99,8 @@ struct os_sched_affinity_bitmask {
 
   explicit os_sched_affinity_bitmask(size_t cpu_idx) : cpu_bitset(compute_host_nof_hardware_threads()) { set(cpu_idx); }
 
+  os_sched_affinity_bitmask(size_t bitset_size, size_t cpu_idx) : cpu_bitset(bitset_size) { set(cpu_idx); }
+
   size_t size() const { return cpu_bitset.size(); }
 
   void set(size_t cpu_idx) { cpu_bitset.set(cpu_idx); }

lib/support/sysinfo.cpp

@@ -134,6 +134,11 @@ void cleanup_cgroups()
   std::string housekeeping_cgroup_path = "/sys/fs/cgroup/housekeeping";
   std::string root_cgroup_path         = "/sys/fs/cgroup/cgroup.procs";
 
+  struct stat sysfs_info;
+  if (::stat("/sys/fs/cgroup", &sysfs_info) < 0) {
+    return;
+  }
+
   struct stat info;
   if (::stat(housekeeping_cgroup_path.c_str(), &info) == 0) {
     move_procs_between_cgroups(root_cgroup_path, housekeeping_cgroup_path + "/cgroup.procs");
@@ -155,4 +160,5 @@ void cleanup_cgroups()
   if (cgroup_changed) {
     std::this_thread::sleep_for(100ms);
   }
+  move_to_cgroup("/sys/fs/cgroup");
 }

lib/support/unique_thread.cpp

@@ -40,19 +40,24 @@ static bool thread_set_param(pthread_t t, os_thread_realtime_priority prio)
   return true;
 }
 
-static bool thread_set_affinity(pthread_t t, const os_sched_affinity_bitmask& bitmap)
+static bool thread_set_affinity(pthread_t t, const os_sched_affinity_bitmask& bitmap, const std::string& name)
 {
-  cpu_set_t cpuset;
-  CPU_ZERO(&cpuset);
+  cpu_set_t* cpusetp     = CPU_ALLOC(bitmap.size());
+  size_t     cpuset_size = CPU_ALLOC_SIZE(bitmap.size());
+  CPU_ZERO_S(cpuset_size, cpusetp);
+
   for (size_t i = 0; i < bitmap.size(); ++i) {
     if (bitmap.test(i)) {
-      CPU_SET(i, &cpuset);
+      CPU_SET_S(i, cpuset_size, cpusetp);
     }
   }
-  if (pthread_setaffinity_np(t, sizeof(cpu_set_t), &cpuset) != 0) {
-    perror("pthread_setaffinity_np");
+  int ret;
+  if ((ret = pthread_setaffinity_np(t, cpuset_size, cpusetp)) != 0) {
+    fmt::print("Couldn't set affinity for {} thread. Cause: '{}'\n", name, strerror(ret));
+    CPU_FREE(cpusetp);
     return false;
   }
+  CPU_FREE(cpusetp);
   return true;
 }
 
@@ -153,7 +158,7 @@ std::thread unique_thread::make_thread(const std::string&               name,
       thread_set_param(tself, prio);
     }
     if (cpu_mask.any()) {
-      thread_set_affinity(tself, cpu_mask);
+      thread_set_affinity(tself, cpu_mask, name);
     }
 #endif