Merge pull request ceph#56775 from cyx1231st/wip-crimson-osd-report-stats

crimson/osd: implement basic reactor-utilization stats report to log


Reviewed-by: Samuel Just <[email protected]>
Reviewed-by: chunmei-liu <[email protected]>
Reviewed-by: Matan Breizman <[email protected]>

Author: Matan-B

src/common/options/crimson.yaml.in

@@ -31,6 +31,11 @@ options:
   desc: CPU cores on which alienstore threads will run in cpuset(7) format
   flags:
   - startup
+- name: crimson_osd_stat_interval
+  type: int
+  level: advanced
+  default: 0
+  desc: Report OSD status periodically in seconds, 0 to disable
 - name: seastore_segment_size
   type: size
   desc: Segment size to use for SegmentManager

src/crimson/common/utility.h

@@ -5,6 +5,8 @@
 
 #include <type_traits>
 
+#include <seastar/core/metrics_api.hh>
+
 namespace _impl {
   template <class T> struct always_false : std::false_type {};
 };
@@ -36,3 +38,16 @@ auto apply_method_to_tuple(Obj &obj, Method method, ArgTuple &&tuple) {
     obj, method, std::forward<ArgTuple>(tuple),
     std::make_index_sequence<tuple_size>());
 }
+
+inline double get_reactor_utilization() {
+  auto &value_map = seastar::metrics::impl::get_value_map();
+  auto found = value_map.find("reactor_utilization");
+  assert(found != value_map.end());
+  auto &[full_name, metric_family] = *found;
+  std::ignore = full_name;
+  assert(metric_family.size() == 1);
+  const auto& [labels, metric] = *metric_family.begin();
+  std::ignore = labels;
+  auto value = (*metric)();
+  return value.d();
+}

src/crimson/osd/osd.cc

@@ -387,6 +387,26 @@ seastar::future<> OSD::start()
         std::ref(osd_states));
     });
   }).then([this, FNAME] {
+    auto stats_seconds = local_conf().get_val<int64_t>("crimson_osd_stat_interval");
+    if (stats_seconds > 0) {
+      shard_stats.resize(seastar::smp::count);
+      stats_timer.set_callback([this, FNAME] {
+        std::ignore = shard_services.invoke_on_all(
+          [this](auto &local_service) {
+          auto stats = local_service.report_stats();
+          shard_stats[seastar::this_shard_id()] = stats;
+        }).then([this, FNAME] {
+          std::ostringstream oss;
+          for (const auto &stats : shard_stats) {
+            oss << int(stats.reactor_utilization);
+            oss << ",";
+          }
+          INFO("reactor_utilizations: {}", oss.str());
+        });
+      });
+      stats_timer.arm_periodic(std::chrono::seconds(stats_seconds));
+    }
+
     heartbeat.reset(new Heartbeat{
 	whoami, get_shard_services(),
 	*monc, *hb_front_msgr, *hb_back_msgr});
@@ -1320,6 +1340,7 @@ seastar::future<> OSD::restart()
 {
   beacon_timer.cancel();
   tick_timer.cancel();
+  stats_timer.cancel();
   return pg_shard_manager.set_up_epoch(
     0
   ).then([this] {

src/crimson/osd/osd.h

@@ -128,6 +128,9 @@ class OSD final : public crimson::net::Dispatcher,
   std::unique_ptr<Heartbeat> heartbeat;
   seastar::timer<seastar::lowres_clock> tick_timer;
 
+  seastar::timer<seastar::lowres_clock> stats_timer;
+  std::vector<ShardServices::shard_stats_t> shard_stats;
+
   const char** get_tracked_conf_keys() const final;
   void handle_conf_change(const ConfigProxy& conf,
                           const std::set<std::string> &changed) final;

src/crimson/osd/shard_services.h

@@ -402,6 +402,13 @@ class ShardServices : public OSDMapService {
     return local_state.store;
   }
 
+  struct shard_stats_t {
+    double reactor_utilization;
+  };
+  shard_stats_t report_stats() {
+    return {get_reactor_utilization()};
+  }
+
   auto remove_pg(spg_t pgid) {
     local_state.pg_map.remove_pg(pgid);
     return pg_to_shard_mapping.remove_pg_mapping(pgid);