Merge pull request ceph#57593 from Matan-B/wip-matanb-crimson-default-cpu-cores

Crimson: Support basic deployments

Reviewed-by: Yingxin Cheng <[email protected]>

Author: Matan-B

src/common/options/crimson.yaml.in

@@ -31,6 +31,15 @@ options:
   desc: CPU cores on which alienstore threads will run in cpuset(7) format
   flags:
   - startup
+- name: crimson_seastar_num_threads
+  type: uint
+  level: advanced
+  default: 0
+  desc: The number of threads for serving seastar reactors without CPU pinning, overridden if crimson_seastar_cpu_cores is set
+  flags:
+  - startup
+  min: 0
+  max: 32
 - name: crimson_osd_stat_interval
   type: int
   level: advanced

src/crimson/os/alienstore/alien_store.cc

@@ -101,19 +101,23 @@ seastar::future<> AlienStore::start()
   if (!store) {
     ceph_abort_msgf("unsupported objectstore type: %s", type.c_str());
   }
-  auto cpu_cores = seastar::resource::parse_cpuset(
-    get_conf<std::string>("crimson_alien_thread_cpu_cores"));
-  //  crimson_alien_thread_cpu_cores are assigned to alien threads.
-  if (!cpu_cores.has_value()) {
-    // no core isolation by default, seastar_cpu_cores will be
-    // shared between both alien and seastar reactor threads.
-    cpu_cores = seastar::resource::parse_cpuset(
-      get_conf<std::string>("crimson_seastar_cpu_cores"));
-    ceph_assert(cpu_cores.has_value());
+  /*
+   * crimson_alien_thread_cpu_cores must be set for optimal performance.
+   * Otherwise, no CPU pinning will take place.
+  */
+  std::optional<seastar::resource::cpuset> alien_thread_cpu_cores;
+
+  if (std::string conf_cpu_cores =
+        get_conf<std::string>("crimson_alien_thread_cpu_cores");
+      !conf_cpu_cores.empty()) {
+    logger().debug("{} using crimson_alien_thread_cpu_cores", __func__);
+    alien_thread_cpu_cores =
+      seastar::resource::parse_cpuset(conf_cpu_cores);
   }
+
   const auto num_threads =
     get_conf<uint64_t>("crimson_alien_op_num_threads");
-  tp = std::make_unique<crimson::os::ThreadPool>(num_threads, 128, cpu_cores);
+  tp = std::make_unique<crimson::os::ThreadPool>(num_threads, 128, alien_thread_cpu_cores);
   return tp->start();
 }
 

src/crimson/osd/main_config_bootstrap_helpers.cc

@@ -150,16 +150,32 @@ _get_early_config(int argc, const char *argv[])
 	      std::end(early_args),
 	      [](auto* arg) { return "--cpuset"sv == arg; });
 	    found == std::end(early_args)) {
-	  auto smp_config = crimson::common::get_conf<std::string>("crimson_seastar_cpu_cores");
-	  if (!smp_config.empty()) {
+	  auto cpu_cores = crimson::common::get_conf<std::string>("crimson_seastar_cpu_cores");
+	  if (!cpu_cores.empty()) {
 	    // Set --cpuset based on crimson_seastar_cpu_cores config option
 	    // --smp default is one per CPU
 	    ret.early_args.emplace_back("--cpuset");
-	    ret.early_args.emplace_back(smp_config);
-	    logger().info("get_early_config: set --cpuset {}", smp_config);
+	    ret.early_args.emplace_back(cpu_cores);
+	    ret.early_args.emplace_back("--thread-affinity");
+	    ret.early_args.emplace_back("1");
+	    logger().info("get_early_config: set --thread-affinity 1 --cpuset {}",
+	                  cpu_cores);
 	  } else {
-	    logger().warn("get_early_config: no cpuset specified, falling back"
-	                  " to seastar's default of: all");
+	    auto reactor_num = crimson::common::get_conf<uint64_t>("crimson_seastar_num_threads");
+	    if (!reactor_num) {
+	      logger().error("get_early_config: crimson_seastar_cpu_cores"
+                             " or crimson_seastar_num_threads"
+                             " must be set");
+	      ceph_abort();
+	    }
+	    std::string smp = fmt::format("{}", reactor_num);
+	    ret.early_args.emplace_back("--smp");
+	    ret.early_args.emplace_back(smp);
+	    ret.early_args.emplace_back("--thread-affinity");
+	    ret.early_args.emplace_back("0");
+	    logger().info("get_early_config: set --thread-affinity 0 --smp {}",
+	                  smp);
+
 	  }
 	} else {
 	  logger().error("get_early_config: --cpuset can be "

src/crimson/osd/osd.cc

@@ -361,7 +361,12 @@ seastar::future<> OSD::start()
 {
   LOG_PREFIX(OSD::start);
   INFO("seastar::smp::count {}", seastar::smp::count);
-
+  if (auto cpu_cores =
+        local_conf().get_val<std::string>("crimson_seastar_cpu_cores");
+      cpu_cores.empty()) {
+    clog->warn() << "for optimal performance please set "
+                    "crimson_seastar_cpu_cores";
+  }
   startup_time = ceph::mono_clock::now();
   ceph_assert(seastar::this_shard_id() == PRIMARY_CORE);
   return store.start().then([this] {