crimson/os/cache: report dirty usage/in/out by trans and extent type

Signed-off-by: Yingxin Cheng <[email protected]>

Author: cyx1231st

src/crimson/os/seastore/cache.cc

@@ -146,6 +146,8 @@ void Cache::register_metrics()
   DEBUG("");
 
   stats = {};
+  last_dirty_io = {};
+  last_dirty_io_by_src_ext = {};
 
   namespace sm = seastar::metrics;
   using src_t = Transaction::src_t;
@@ -767,6 +769,7 @@ void Cache::add_to_dirty(
   ).account_in(extent_length);
   if (p_src != nullptr) {
     assert(!is_root_type(ref->get_type()));
+    stats.dirty_io.in_sizes.account_in(extent_length);
     get_by_ext(
       get_by_src(stats.dirty_io_by_src_ext, *p_src),
       ref->get_type()
@@ -790,6 +793,8 @@ void Cache::remove_from_dirty(
   ).account_out(extent_length);
   if (p_src != nullptr) {
     assert(!is_root_type(ref->get_type()));
+    stats.dirty_io.out_sizes.account_in(extent_length);
+    stats.dirty_io.out_versions += ref->get_version();
     auto& dirty_stats = get_by_ext(
       get_by_src(stats.dirty_io_by_src_ext, *p_src),
       ref->get_type());
@@ -822,6 +827,7 @@ void Cache::replace_dirty(
   assert(!is_root_type(next->get_type()));
   assert(prev->get_type() == next->get_type());
 
+  stats.dirty_io.num_replace += 1;
   get_by_ext(
     get_by_src(stats.dirty_io_by_src_ext, src),
     next->get_type()).num_replace += 1;
@@ -2242,8 +2248,98 @@ Cache::do_get_caching_extent_by_type(
 cache_stats_t Cache::get_stats(
   bool report_detail, double seconds) const
 {
+  LOG_PREFIX(Cache::get_stats);
+
   cache_stats_t ret;
   lru.get_stats(ret, report_detail, seconds);
+
+  /*
+   * get dirty stats
+   */
+
+  ret.dirty_sizes = cache_size_stats_t{stats.dirty_bytes, dirty.size()};
+  ret.dirty_io = stats.dirty_io;
+  ret.dirty_io.minus(last_dirty_io);
+
+  if (report_detail && seconds != 0) {
+    counter_by_src_t<counter_by_extent_t<dirty_io_stats_t> >
+      _trans_io_by_src_ext = stats.dirty_io_by_src_ext;
+    counter_by_src_t<dirty_io_stats_t> trans_io_by_src;
+    for (uint8_t _src=0; _src<TRANSACTION_TYPE_MAX; ++_src) {
+      auto src = static_cast<transaction_type_t>(_src);
+      auto& io_by_ext = get_by_src(_trans_io_by_src_ext, src);
+      const auto& last_io_by_ext = get_by_src(last_dirty_io_by_src_ext, src);
+      auto& trans_io_per_src = get_by_src(trans_io_by_src, src);
+      for (uint8_t _ext=0; _ext<EXTENT_TYPES_MAX; ++_ext) {
+        auto ext = static_cast<extent_types_t>(_ext);
+        auto& extent_io = get_by_ext(io_by_ext, ext);
+        const auto& last_extent_io = get_by_ext(last_io_by_ext, ext);
+        extent_io.minus(last_extent_io);
+        trans_io_per_src.add(extent_io);
+      }
+    }
+
+    std::ostringstream oss;
+    oss << "\ndirty total" << ret.dirty_sizes;
+    cache_size_stats_t data_sizes;
+    cache_size_stats_t mdat_sizes;
+    cache_size_stats_t phys_sizes;
+    for (uint8_t _ext=0; _ext<EXTENT_TYPES_MAX; ++_ext) {
+      auto ext = static_cast<extent_types_t>(_ext);
+      const auto& extent_sizes = get_by_ext(stats.dirty_sizes_by_ext, ext);
+
+      if (is_data_type(ext)) {
+        data_sizes.add(extent_sizes);
+      } else if (is_logical_metadata_type(ext)) {
+        mdat_sizes.add(extent_sizes);
+      } else if (is_physical_type(ext)) {
+        phys_sizes.add(extent_sizes);
+      }
+    }
+    oss << "\n  data" << data_sizes
+        << "\n  mdat" << mdat_sizes
+        << "\n  phys" << phys_sizes;
+
+    oss << "\ndirty io: "
+        << dirty_io_stats_printer_t{seconds, ret.dirty_io};
+    for (uint8_t _src=0; _src<TRANSACTION_TYPE_MAX; ++_src) {
+      auto src = static_cast<transaction_type_t>(_src);
+      const auto& trans_io_per_src = get_by_src(trans_io_by_src, src);
+      if (trans_io_per_src.is_empty()) {
+        continue;
+      }
+      dirty_io_stats_t data_io;
+      dirty_io_stats_t mdat_io;
+      dirty_io_stats_t phys_io;
+      const auto& io_by_ext = get_by_src(_trans_io_by_src_ext, src);
+      for (uint8_t _ext=0; _ext<EXTENT_TYPES_MAX; ++_ext) {
+        auto ext = static_cast<extent_types_t>(_ext);
+        const auto extent_io = get_by_ext(io_by_ext, ext);
+        if (is_data_type(ext)) {
+          data_io.add(extent_io);
+        } else if (is_logical_metadata_type(ext)) {
+          mdat_io.add(extent_io);
+        } else if (is_physical_type(ext)) {
+          phys_io.add(extent_io);
+        }
+      }
+      oss << "\n  " << src << ": "
+          << dirty_io_stats_printer_t{seconds, trans_io_per_src}
+          << "\n    data: "
+          << dirty_io_stats_printer_t{seconds, data_io}
+          << "\n    mdat: "
+          << dirty_io_stats_printer_t{seconds, mdat_io}
+          << "\n    phys: "
+          << dirty_io_stats_printer_t{seconds, phys_io};
+    }
+
+    INFO("{}", oss.str());
+
+    last_dirty_io_by_src_ext = stats.dirty_io_by_src_ext;
+  }
+
+  last_dirty_io = stats.dirty_io;
+
   return ret;
 }
 

src/crimson/os/seastore/cache.h

@@ -1605,6 +1605,7 @@ class Cache {
 
     uint64_t dirty_bytes = 0;
     counter_by_extent_t<cache_size_stats_t> dirty_sizes_by_ext;
+    dirty_io_stats_t dirty_io;
     counter_by_src_t<counter_by_extent_t<dirty_io_stats_t> >
       dirty_io_by_src_ext;
 
@@ -1635,6 +1636,10 @@ class Cache {
     version_stat_t committed_reclaim_version;
   } stats;
 
+  mutable dirty_io_stats_t last_dirty_io;
+  mutable counter_by_src_t<counter_by_extent_t<dirty_io_stats_t> >
+    last_dirty_io_by_src_ext;
+
   void account_conflict(Transaction::src_t src1, Transaction::src_t src2) {
     assert(src1 < Transaction::src_t::MAX);
     assert(src2 < Transaction::src_t::MAX);

src/crimson/os/seastore/seastore.cc

@@ -705,6 +705,7 @@ seastar::future<> SeaStore::report_stats()
     for (const auto& s : shard_cache_stats) {
       cache_total.add(s);
     }
+
     cache_size_stats_t lru_sizes_ps = cache_total.lru_sizes;
     lru_sizes_ps.size /= seastar::smp::count;
     lru_sizes_ps.num_extents /= seastar::smp::count;
@@ -718,6 +719,23 @@ seastar::future<> SeaStore::report_stats()
          cache_io_stats_printer_t{seconds, cache_total.lru_io},
          lru_sizes_ps,
          cache_io_stats_printer_t{seconds, lru_io_ps});
+
+    cache_size_stats_t dirty_sizes_ps = cache_total.dirty_sizes;
+    dirty_sizes_ps.size /= seastar::smp::count;
+    dirty_sizes_ps.num_extents /= seastar::smp::count;
+    dirty_io_stats_t dirty_io_ps = cache_total.dirty_io;
+    dirty_io_ps.in_sizes.size /= seastar::smp::count;
+    dirty_io_ps.in_sizes.num_extents /= seastar::smp::count;
+    dirty_io_ps.num_replace /= seastar::smp::count;
+    dirty_io_ps.out_sizes.size /= seastar::smp::count;
+    dirty_io_ps.out_sizes.num_extents /= seastar::smp::count;
+    dirty_io_ps.out_versions /= seastar::smp::count;
+    INFO("cache dirty: total{} {}; per-shard: total{} {}",
+         cache_total.dirty_sizes,
+         dirty_io_stats_printer_t{seconds, cache_total.dirty_io},
+         dirty_sizes_ps,
+         dirty_io_stats_printer_t{seconds, dirty_io_ps});
+
     return seastar::now();
   });
 }

src/crimson/os/seastore/seastore_types.cc

@@ -1012,4 +1012,31 @@ std::ostream& operator<<(std::ostream& out, const cache_size_stats_t& p)
   return out;
 }
 
+std::ostream& operator<<(std::ostream& out, const cache_size_stats_printer_t& p)
+{
+  constexpr const char* dfmt = "{:.2f}";
+  out << "("
+      << fmt::format(dfmt, p.stats.get_mb()/p.seconds)
+      << "MiB/s,"
+      << fmt::format(dfmt, p.stats.get_avg_kb())
+      << "KiB,"
+      << fmt::format(dfmt, p.stats.num_extents/p.seconds)
+      << "ps)";
+  return out;
+}
+
+std::ostream& operator<<(std::ostream& out, const dirty_io_stats_printer_t& p)
+{
+  constexpr const char* dfmt = "{:.2f}";
+  out << "in"
+      << cache_size_stats_printer_t{p.seconds, p.stats.in_sizes}
+      << " replaces="
+      << fmt::format(dfmt, p.stats.num_replace/p.seconds)
+      << "ps out"
+      << cache_size_stats_printer_t{p.seconds, p.stats.out_sizes}
+      << " outv="
+      << fmt::format(dfmt, p.stats.get_avg_out_version());
+  return out;
+}
+
 } // namespace crimson::os::seastore

src/crimson/os/seastore/seastore_types.h

@@ -2617,6 +2617,10 @@ struct cache_size_stats_t {
   uint64_t size = 0;
   uint64_t num_extents = 0;
 
+  bool is_empty() const {
+    return num_extents == 0;
+  }
+
   double get_mb() const {
     return (size>>12)/static_cast<double>(256);
   }
@@ -2641,26 +2645,69 @@ struct cache_size_stats_t {
     size += o.size;
     num_extents += o.num_extents;
   }
+
+  void minus(const cache_size_stats_t& o) {
+    size -= o.size;
+    num_extents -= o.num_extents;
+  }
 };
 std::ostream& operator<<(std::ostream&, const cache_size_stats_t&);
+struct cache_size_stats_printer_t {
+  double seconds;
+  const cache_size_stats_t& stats;
+};
+std::ostream& operator<<(std::ostream&, const cache_size_stats_printer_t&);
+
+struct dirty_io_stats_t {
+  cache_size_stats_t in_sizes;
+  uint64_t num_replace = 0;
+  cache_size_stats_t out_sizes;
+  uint64_t out_versions = 0;
+
+  double get_avg_out_version() const {
+    return out_versions/static_cast<double>(out_sizes.num_extents);
+  }
+
+  bool is_empty() const {
+    return in_sizes.is_empty() &&
+           num_replace == 0 &&
+           out_sizes.is_empty();
+  }
+
+  void add(const dirty_io_stats_t& o) {
+    in_sizes.add(o.in_sizes);
+    num_replace += o.num_replace;
+    out_sizes.add(o.out_sizes);
+    out_versions += o.out_versions;
+  }
+
+  void minus(const dirty_io_stats_t& o) {
+    in_sizes.minus(o.in_sizes);
+    num_replace -= o.num_replace;
+    out_sizes.minus(o.out_sizes);
+    out_versions -= o.out_versions;
+  }
+};
+struct dirty_io_stats_printer_t {
+  double seconds;
+  const dirty_io_stats_t& stats;
+};
+std::ostream& operator<<(std::ostream&, const dirty_io_stats_printer_t&);
 
 struct cache_stats_t {
   cache_size_stats_t lru_sizes;
   cache_io_stats_t lru_io;
+  cache_size_stats_t dirty_sizes;
+  dirty_io_stats_t dirty_io;
 
   void add(const cache_stats_t& o) {
     lru_sizes.add(o.lru_sizes);
     lru_io.add(o.lru_io);
+    dirty_sizes.add(o.dirty_sizes);
+    dirty_io.add(o.dirty_io);
   }
 };
 
-struct dirty_io_stats_t {
-  cache_size_stats_t in_sizes;
-  uint64_t num_replace = 0;
-  cache_size_stats_t out_sizes;
-  uint64_t out_versions = 0;
-};
-
 }
 
 WRITE_CLASS_DENC_BOUNDED(crimson::os::seastore::seastore_meta_t)
@@ -2680,9 +2727,11 @@ WRITE_CLASS_DENC_BOUNDED(crimson::os::seastore::segment_tail_t)
 #if FMT_VERSION >= 90000
 template <> struct fmt::formatter<crimson::os::seastore::cache_io_stats_printer_t> : fmt::ostream_formatter {};
 template <> struct fmt::formatter<crimson::os::seastore::cache_size_stats_t> : fmt::ostream_formatter {};
+template <> struct fmt::formatter<crimson::os::seastore::cache_size_stats_printer_t> : fmt::ostream_formatter {};
 template <> struct fmt::formatter<crimson::os::seastore::data_category_t> : fmt::ostream_formatter {};
 template <> struct fmt::formatter<crimson::os::seastore::delta_info_t> : fmt::ostream_formatter {};
 template <> struct fmt::formatter<crimson::os::seastore::device_id_printer_t> : fmt::ostream_formatter {};
+template <> struct fmt::formatter<crimson::os::seastore::dirty_io_stats_printer_t> : fmt::ostream_formatter {};
 template <> struct fmt::formatter<crimson::os::seastore::extent_types_t> : fmt::ostream_formatter {};
 template <> struct fmt::formatter<crimson::os::seastore::journal_seq_t> : fmt::ostream_formatter {};
 template <> struct fmt::formatter<crimson::os::seastore::journal_tail_delta_t> : fmt::ostream_formatter {};