crimson: introduce crimson_seastar_num_threads

    This new option is there to allow *basic* cluster deployments.
    crimson_seastar_num_threads can be used globally for all the OSDs in the cluster:
    ```
    osd:
      crimson_seastar_num_threads: <n>
      crimson_alien_op_num_threads: <m>
    ```

    As a result, all the available CPUs will be allocated to *both* seastar reactor threads
    and to Alienstore threads - without any exclusion.

    Notes:

    * The core allocation will most likely overlap between OSDS and/or Seastar and Alienstore.
    An optiomal deployment requires `crimson_alien_thread_cpu_cores`
    and `crimson_seastar_cpu_cores` to be set for each OSD based on the host its located at.

    * Seastar reactor number (smp::count) will be deduced directly from crimson_seastar_num_threads

    ---

    Documentation followup: ceph#57352

---
    ### Option 1 (Optimal)
    * CPUs are pinned to the provided sets (seastar and alien cpu sets)
    * thread-affinity = 1 (seastar's default).
    * smp (smp::count / reactor count): `seastar_cpu_set.size()`
    * Seastar reactor threads: `seastar_cpu_set.size()`
    * Alienstore threads num: `crimson_alien_op_num_threads`
    ---
    ### Option 2 (Basic):
    * No CPU pinning.
    * thread-affinity = 0
    * smp (smp::count / reactor count) = `crimson_seastar_num_threads`
    * Seastar reactor threads num: `crimson_seastar_num_threads`
    * Alienstore threads num: `crimson_alien_op_num_threads`
---
    Fixes: https://tracker.ceph.com/issues/65752

    Signed-off-by: Matan Breizman <[email protected]>

Author: Matan-B

src/common/options/crimson.yaml.in

@@ -31,6 +31,15 @@ options:
   desc: CPU cores on which alienstore threads will run in cpuset(7) format
   flags:
   - startup
+- name: crimson_seastar_num_threads
+  type: uint
+  level: advanced
+  default: 0
+  desc: The number of threads for serving seastar reactors without CPU pinning, overridden if crimson_seastar_cpu_cores is set
+  flags:
+  - startup
+  min: 0
+  max: 32
 - name: crimson_osd_stat_interval
   type: int
   level: advanced

src/crimson/os/alienstore/alien_store.cc

@@ -101,19 +101,23 @@ seastar::future<> AlienStore::start()
   if (!store) {
     ceph_abort_msgf("unsupported objectstore type: %s", type.c_str());
   }
-  auto cpu_cores = seastar::resource::parse_cpuset(
-    get_conf<std::string>("crimson_alien_thread_cpu_cores"));
-  //  crimson_alien_thread_cpu_cores are assigned to alien threads.
-  if (!cpu_cores.has_value()) {
-    // no core isolation by default, seastar_cpu_cores will be
-    // shared between both alien and seastar reactor threads.
-    cpu_cores = seastar::resource::parse_cpuset(
-      get_conf<std::string>("crimson_seastar_cpu_cores"));
-    ceph_assert(cpu_cores.has_value());
+  /*
+   * crimson_alien_thread_cpu_cores must be set for optimal performance.
+   * Otherwise, no CPU pinning will take place.
+  */
+  std::optional<seastar::resource::cpuset> alien_thread_cpu_cores;
+
+  if (std::string conf_cpu_cores =
+        get_conf<std::string>("crimson_alien_thread_cpu_cores");
+      !conf_cpu_cores.empty()) {
+    logger().debug("{} using crimson_alien_thread_cpu_cores", __func__);
+    alien_thread_cpu_cores =
+      seastar::resource::parse_cpuset(conf_cpu_cores);
   }
+
   const auto num_threads =
     get_conf<uint64_t>("crimson_alien_op_num_threads");
-  tp = std::make_unique<crimson::os::ThreadPool>(num_threads, 128, cpu_cores);
+  tp = std::make_unique<crimson::os::ThreadPool>(num_threads, 128, alien_thread_cpu_cores);
   return tp->start();
 }
 

src/crimson/osd/main_config_bootstrap_helpers.cc

@@ -150,16 +150,32 @@ _get_early_config(int argc, const char *argv[])
 	      std::end(early_args),
 	      [](auto* arg) { return "--cpuset"sv == arg; });
 	    found == std::end(early_args)) {
-	  auto smp_config = crimson::common::get_conf<std::string>("crimson_seastar_cpu_cores");
-	  if (!smp_config.empty()) {
+	  auto cpu_cores = crimson::common::get_conf<std::string>("crimson_seastar_cpu_cores");
+	  if (!cpu_cores.empty()) {
 	    // Set --cpuset based on crimson_seastar_cpu_cores config option
 	    // --smp default is one per CPU
 	    ret.early_args.emplace_back("--cpuset");
-	    ret.early_args.emplace_back(smp_config);
-	    logger().info("get_early_config: set --cpuset {}", smp_config);
+	    ret.early_args.emplace_back(cpu_cores);
+	    ret.early_args.emplace_back("--thread-affinity");
+	    ret.early_args.emplace_back("1");
+	    logger().info("get_early_config: set --thread-affinity 1 --cpuset {}",
+	                  cpu_cores);
 	  } else {
-	    logger().warn("get_early_config: no cpuset specified, falling back"
-	                  " to seastar's default of: all");
+	    auto reactor_num = crimson::common::get_conf<uint64_t>("crimson_seastar_num_threads");
+	    if (!reactor_num) {
+	      logger().error("get_early_config: crimson_seastar_cpu_cores"
+                             " or crimson_seastar_num_threads"
+                             " must be set");
+	      ceph_abort();
+	    }
+	    std::string smp = fmt::format("{}", reactor_num);
+	    ret.early_args.emplace_back("--smp");
+	    ret.early_args.emplace_back(smp);
+	    ret.early_args.emplace_back("--thread-affinity");
+	    ret.early_args.emplace_back("0");
+	    logger().info("get_early_config: set --thread-affinity 0 --smp {}",
+	                  smp);
+
 	  }
 	} else {
 	  logger().error("get_early_config: --cpuset can be "