crimson/os/seastore/segment_cleaner: add statistics for segment utilization

Signed-off-by: Xuehan Xu <xxhdx1985126@gmail.com>

Author: xxhdx1985126

src/crimson/os/seastore/segment_cleaner.cc

@@ -49,14 +49,14 @@ bool SpaceTrackerSimple::equals(const SpaceTrackerI &_other) const
   bool all_match = true;
   for (auto i = live_bytes_by_segment.begin(), j = other.live_bytes_by_segment.begin();
        i != live_bytes_by_segment.end(); ++i, ++j) {
-    if (i->second != j->second) {
+    if (i->second.live_bytes != j->second.live_bytes) {
       all_match = false;
       logger().debug(
 	"{}: segment_id {} live bytes mismatch *this: {}, other: {}",
 	__func__,
 	i->first,
-	i->second,
-	j->second);
+	i->second.live_bytes,
+	j->second.live_bytes);
     }
   }
   return all_match;
@@ -176,13 +176,20 @@ SegmentCleaner::SegmentCleaner(
     config(config),
     scanner(std::move(scr)),
     gc_process(*this)
-{
-  register_metrics();
-}
+{}
 
 void SegmentCleaner::register_metrics()
 {
   namespace sm = seastar::metrics;
+  stats.segment_util.buckets.resize(11);
+  for (int i = 0; i < 11; i++) {
+    stats.segment_util.buckets[i].upper_bound = ((double)(i + 1)) / 10;
+    if (!i) {
+      stats.segment_util.buckets[i].count = segments.num_segments();
+    } else {
+      stats.segment_util.buckets[i].count = 0;
+    }
+  }
   metrics.add_group("segment_cleaner", {
     sm::make_counter("segments_released", stats.segments_released,
 		     sm::description("total number of extents released by SegmentCleaner")),
@@ -205,7 +212,12 @@ void SegmentCleaner::register_metrics()
 		    [this] {
 		      return segments.get_opened_segments();
 		    },
-		    sm::description("the number of segments whose state is open"))
+		    sm::description("the number of segments whose state is open")),
+    sm::make_histogram("segment_utilization_distribution",
+		       [this]() -> seastar::metrics::histogram& {
+		         return stats.segment_util;
+		       },
+		       sm::description("utilization distribution of all segments"))
   });
 }
 

src/crimson/os/seastore/segment_cleaner.h

@@ -4,6 +4,7 @@
 #pragma once
 
 #include <boost/intrusive/set.hpp>
+#include <seastar/core/metrics_types.hh>
 
 #include "common/ceph_time.h"
 
@@ -255,6 +256,16 @@ class segment_info_set_t {
   device_segment_id_t get_journal_segments() const {
     return journal_segments;
   }
+  device_segment_id_t num_segments() const {
+    device_segment_id_t num = 0;
+    for (auto& sm_info : sm_infos) {
+      if (!sm_info) {
+	continue;
+      }
+      num += sm_info->num_segments;
+    }
+    return num;
+  }
 private:
   std::vector<std::optional<segment_manager_info_t>> sm_infos;
   segment_map_t<segment_info_t> segments;
@@ -309,20 +320,26 @@ class SpaceTrackerI {
 
   virtual void dump_usage(segment_id_t) const = 0;
 
+  virtual double calc_utilization(segment_id_t segment) const = 0;
+
   virtual void reset() = 0;
 
   virtual ~SpaceTrackerI() = default;
 };
 using SpaceTrackerIRef = std::unique_ptr<SpaceTrackerI>;
 
 class SpaceTrackerSimple : public SpaceTrackerI {
+  struct segment_bytes_t {
+    int64_t live_bytes = 0;
+    seastore_off_t total_bytes = 0;
+  };
   // Tracks live space for each segment
-  segment_map_t<int64_t> live_bytes_by_segment;
+  segment_map_t<segment_bytes_t> live_bytes_by_segment;
 
   int64_t update_usage(segment_id_t segment, int64_t delta) {
-    live_bytes_by_segment[segment] += delta;
-    assert(live_bytes_by_segment[segment] >= 0);
-    return live_bytes_by_segment[segment];
+    live_bytes_by_segment[segment].live_bytes += delta;
+    assert(live_bytes_by_segment[segment].live_bytes >= 0);
+    return live_bytes_by_segment[segment].live_bytes;
   }
 public:
   SpaceTrackerSimple(const SpaceTrackerSimple &) = default;
@@ -337,7 +354,7 @@ class SpaceTrackerSimple : public SpaceTrackerI {
       live_bytes_by_segment.add_device(
 	sm->get_device_id(),
 	sm->get_num_segments(),
-	0);
+	{0, sm->get_segment_size()});
     }
   }
 
@@ -356,14 +373,19 @@ class SpaceTrackerSimple : public SpaceTrackerI {
   }
 
   int64_t get_usage(segment_id_t segment) const final {
-    return live_bytes_by_segment[segment];
+    return live_bytes_by_segment[segment].live_bytes;
+  }
+
+  double calc_utilization(segment_id_t segment) const final {
+    auto& seg_bytes = live_bytes_by_segment[segment];
+    return (double)seg_bytes.live_bytes / (double)seg_bytes.total_bytes;
   }
 
   void dump_usage(segment_id_t) const final {}
 
   void reset() final {
     for (auto &i : live_bytes_by_segment) {
-      i.second = 0;
+      i.second = {0, 0};
     }
   }
 
@@ -379,10 +401,15 @@ class SpaceTrackerSimple : public SpaceTrackerI {
 class SpaceTrackerDetailed : public SpaceTrackerI {
   class SegmentMap {
     int64_t used = 0;
+    seastore_off_t total_bytes = 0;
     std::vector<bool> bitmap;
 
   public:
-    SegmentMap(size_t blocks) : bitmap(blocks, false) {}
+    SegmentMap(
+      size_t blocks,
+      seastore_off_t total_bytes)
+    : total_bytes(total_bytes),
+      bitmap(blocks, false) {}
 
     int64_t update_usage(int64_t delta) {
       used += delta;
@@ -407,6 +434,10 @@ class SpaceTrackerDetailed : public SpaceTrackerI {
 
     void dump_usage(extent_len_t block_size) const;
 
+    double calc_utilization() const {
+      return (double)used / (double)total_bytes;
+    }
+
     void reset() {
       used = 0;
       for (auto &&i: bitmap) {
@@ -435,7 +466,8 @@ class SpaceTrackerDetailed : public SpaceTrackerI {
 	sm->get_device_id(),
 	sm->get_num_segments(),
 	SegmentMap(
-	  sm->get_segment_size() / sm->get_block_size()));
+	  sm->get_segment_size() / sm->get_block_size(),
+	  sm->get_segment_size()));
       block_size_by_segment_manager[sm->get_device_id()] = sm->get_block_size();
     }
   }
@@ -466,6 +498,10 @@ class SpaceTrackerDetailed : public SpaceTrackerI {
     return segment_usage[segment].get_usage();
   }
 
+  double calc_utilization(segment_id_t segment) const final {
+    return segment_usage[segment].calc_utilization();
+  }
+
   void dump_usage(segment_id_t seg) const final;
 
   void reset() final {
@@ -647,6 +683,7 @@ class SegmentCleaner : public SegmentProvider {
     uint64_t accumulated_blocked_ios = 0;
     uint64_t empty_segments = 0;
     int64_t ios_blocking = 0;
+    seastar::metrics::histogram segment_util;
   } stats;
   seastar::metrics::metric_group metrics;
   void register_metrics();
@@ -717,6 +754,12 @@ class SegmentCleaner : public SegmentProvider {
     return mark_empty(segment);
   }
 
+  void adjust_segment_util(double old_usage, double new_usage) {
+    assert(stats.segment_util.buckets[std::floor(old_usage * 10)].count > 0);
+    stats.segment_util.buckets[std::floor(old_usage * 10)].count--;
+    stats.segment_util.buckets[std::floor(new_usage * 10)].count++;
+  }
+
   void mark_space_used(
     paddr_t addr,
     extent_len_t len,
@@ -731,10 +774,14 @@ class SegmentCleaner : public SegmentProvider {
       return;
 
     stats.used_bytes += len;
+    auto old_usage = space_tracker->calc_utilization(seg_addr.get_segment_id());
     [[maybe_unused]] auto ret = space_tracker->allocate(
       seg_addr.get_segment_id(),
       seg_addr.get_segment_off(),
       len);
+    auto new_usage = space_tracker->calc_utilization(seg_addr.get_segment_id());
+    adjust_segment_util(old_usage, new_usage);
+
     gc_process.maybe_wake_on_space_used();
     assert(ret > 0);
   }
@@ -748,15 +795,18 @@ class SegmentCleaner : public SegmentProvider {
     ceph_assert(stats.used_bytes >= len);
     stats.used_bytes -= len;
     auto& seg_addr = addr.as_seg_paddr();
-    assert(addr.get_device_id() ==
+    assert(seg_addr.get_segment_id().device_id() ==
       segments[seg_addr.get_segment_id().device_id()]->device_id);
     assert(seg_addr.get_segment_id().device_segment_id() <
       segments[seg_addr.get_segment_id().device_id()]->num_segments);
 
+    auto old_usage = space_tracker->calc_utilization(seg_addr.get_segment_id());
     [[maybe_unused]] auto ret = space_tracker->release(
       seg_addr.get_segment_id(),
       seg_addr.get_segment_off(),
       len);
+    auto new_usage = space_tracker->calc_utilization(seg_addr.get_segment_id());
+    adjust_segment_util(old_usage, new_usage);
     maybe_wake_gc_blocked_io();
     assert(ret >= 0);
   }