crimson/os/seastore/cache: report lru usage/in/out with trans and extent type

Signed-off-by: Yingxin Cheng <[email protected]>

Author: cyx1231st

src/crimson/os/seastore/cache.cc

@@ -2159,4 +2159,110 @@ Cache::do_get_caching_extent_by_type(
   });
 }
 
+cache_stats_t Cache::get_stats(
+  bool report_detail, double seconds) const
+{
+  cache_stats_t ret;
+  lru.get_stats(ret, report_detail, seconds);
+  return ret;
+}
+
+void Cache::LRU::get_stats(
+  cache_stats_t &stats,
+  bool report_detail,
+  double seconds) const
+{
+  LOG_PREFIX(Cache::LRU::get_stats);
+
+  stats.lru_sizes = cache_size_stats_t{current_size, lru.size()};
+  stats.lru_io = overall_io;
+  stats.lru_io.minus(last_overall_io);
+
+  if (report_detail && seconds != 0) {
+    cache_io_stats_t _trans_io = trans_io;
+    _trans_io.minus(last_trans_io);
+
+    cache_io_stats_t other_io = stats.lru_io;
+    other_io.minus(_trans_io);
+
+    counter_by_src_t<counter_by_extent_t<cache_io_stats_t> >
+      _trans_io_by_src_ext = trans_io_by_src_ext;
+    counter_by_src_t<cache_io_stats_t> trans_io_by_src;
+    for (uint8_t _src=0; _src<TRANSACTION_TYPE_MAX; ++_src) {
+      auto src = static_cast<transaction_type_t>(_src);
+      auto& io_by_ext = get_by_src(_trans_io_by_src_ext, src);
+      const auto& last_io_by_ext = get_by_src(last_trans_io_by_src_ext, src);
+      auto& trans_io_per_src = get_by_src(trans_io_by_src, src);
+      for (uint8_t _ext=0; _ext<EXTENT_TYPES_MAX; ++_ext) {
+        auto ext = static_cast<extent_types_t>(_ext);
+        auto& extent_io = get_by_ext(io_by_ext, ext);
+        const auto& last_extent_io = get_by_ext(last_io_by_ext, ext);
+        extent_io.minus(last_extent_io);
+        trans_io_per_src.add(extent_io);
+      }
+    }
+
+    std::ostringstream oss;
+    oss << "\nlru total" << stats.lru_sizes;
+    cache_size_stats_t data_sizes;
+    cache_size_stats_t mdat_sizes;
+    cache_size_stats_t phys_sizes;
+    for (uint8_t _ext=0; _ext<EXTENT_TYPES_MAX; ++_ext) {
+      auto ext = static_cast<extent_types_t>(_ext);
+      const auto extent_sizes = get_by_ext(sizes_by_ext, ext);
+      if (is_data_type(ext)) {
+        data_sizes.add(extent_sizes);
+      } else if (is_logical_metadata_type(ext)) {
+        mdat_sizes.add(extent_sizes);
+      } else if (is_physical_type(ext)) {
+        phys_sizes.add(extent_sizes);
+      }
+    }
+    oss << "\n  data" << data_sizes
+        << "\n  mdat" << mdat_sizes
+        << "\n  phys" << phys_sizes;
+
+    oss << "\nlru io: trans-"
+        << cache_io_stats_printer_t{seconds, _trans_io}
+        << "; other-"
+        << cache_io_stats_printer_t{seconds, other_io};
+    for (uint8_t _src=0; _src<TRANSACTION_TYPE_MAX; ++_src) {
+      auto src = static_cast<transaction_type_t>(_src);
+      const auto& trans_io_per_src = get_by_src(trans_io_by_src, src);
+      if (trans_io_per_src.is_empty()) {
+        continue;
+      }
+      cache_io_stats_t data_io;
+      cache_io_stats_t mdat_io;
+      cache_io_stats_t phys_io;
+      const auto& io_by_ext = get_by_src(_trans_io_by_src_ext, src);
+      for (uint8_t _ext=0; _ext<EXTENT_TYPES_MAX; ++_ext) {
+        auto ext = static_cast<extent_types_t>(_ext);
+        const auto extent_io = get_by_ext(io_by_ext, ext);
+        if (is_data_type(ext)) {
+          data_io.add(extent_io);
+        } else if (is_logical_metadata_type(ext)) {
+          mdat_io.add(extent_io);
+        } else if (is_physical_type(ext)) {
+          phys_io.add(extent_io);
+        }
+      }
+      oss << "\n  " << src << ": "
+          << cache_io_stats_printer_t{seconds, trans_io_per_src}
+          << "\n    data: "
+          << cache_io_stats_printer_t{seconds, data_io}
+          << "\n    mdat: "
+          << cache_io_stats_printer_t{seconds, mdat_io}
+          << "\n    phys: "
+          << cache_io_stats_printer_t{seconds, phys_io};
+    }
+
+    INFO("{}", oss.str());
+  }
+
+  last_overall_io = overall_io;
+  last_trans_io = trans_io;
+  last_trans_io_by_src_ext = trans_io_by_src_ext;
+}
+
 }

src/crimson/os/seastore/cache.h

@@ -198,6 +198,8 @@ class Cache {
   Cache(ExtentPlacementManager &epm);
   ~Cache();
 
+  cache_stats_t get_stats(bool report_detail, double seconds) const;
+
   /// Creates empty transaction by source
   TransactionRef create_transaction(
       Transaction::src_t src,
@@ -1363,7 +1365,7 @@ class Cache {
       return;
     }
     if (ext.is_stable_clean() && !ext.is_placeholder()) {
-      lru.move_to_top(ext);
+      lru.move_to_top(ext, p_src);
     }
   }
 
@@ -1419,6 +1421,7 @@ class Cache {
 
   friend class crimson::os::seastore::backref::BtreeBackrefManager;
   friend class crimson::os::seastore::BackrefManager;
+
   /**
    * lru
    *
@@ -1431,16 +1434,38 @@ class Cache {
     // current size (bytes)
     size_t current_size = 0;
 
+    counter_by_extent_t<cache_size_stats_t> sizes_by_ext;
+    cache_io_stats_t overall_io;
+    cache_io_stats_t trans_io;
+    counter_by_src_t<counter_by_extent_t<cache_io_stats_t> >
+      trans_io_by_src_ext;
+
+    mutable cache_io_stats_t last_overall_io;
+    mutable cache_io_stats_t last_trans_io;
+    mutable counter_by_src_t<counter_by_extent_t<cache_io_stats_t> >
+      last_trans_io_by_src_ext;
+
     CachedExtent::primary_ref_list lru;
 
-    void do_remove_from_lru(CachedExtent &extent) {
+    void do_remove_from_lru(
+        CachedExtent &extent,
+        const Transaction::src_t* p_src) {
       assert(extent.is_stable_clean() && !extent.is_placeholder());
       assert(extent.primary_ref_list_hook.is_linked());
       assert(lru.size() > 0);
-      assert(current_size >= extent.get_length());
+      auto extent_length = extent.get_length();
+      assert(current_size >= extent_length);
 
       lru.erase(lru.s_iterator_to(extent));
-      current_size -= extent.get_length();
+      current_size -= extent_length;
+      get_by_ext(sizes_by_ext, extent.get_type()).account_out(extent_length);
+      overall_io.account_out(extent_length);
+      if (p_src) {
+        trans_io.account_out(extent_length);
+        get_by_ext(
+          get_by_src(trans_io_by_src_ext, *p_src),
+          extent.get_type()).account_out(extent_length);
+      }
       intrusive_ptr_release(&extent);
     }
 
@@ -1459,32 +1484,48 @@ class Cache {
       return lru.size();
     }
 
+    void get_stats(
+        cache_stats_t &stats,
+        bool report_detail,
+        double seconds) const;
+
     void remove_from_lru(CachedExtent &extent) {
       assert(extent.is_stable_clean() && !extent.is_placeholder());
 
       if (extent.primary_ref_list_hook.is_linked()) {
-        do_remove_from_lru(extent);
+        do_remove_from_lru(extent, nullptr);
       }
     }
 
-    void move_to_top(CachedExtent &extent) {
+    void move_to_top(
+        CachedExtent &extent,
+        const Transaction::src_t* p_src) {
       assert(extent.is_stable_clean() && !extent.is_placeholder());
 
+      auto extent_length = extent.get_length();
       if (extent.primary_ref_list_hook.is_linked()) {
         // present, move to top (back)
         assert(lru.size() > 0);
-        assert(current_size >= extent.get_length());
+        assert(current_size >= extent_length);
         lru.erase(lru.s_iterator_to(extent));
         lru.push_back(extent);
       } else {
         // absent, add to top (back)
-        current_size += extent.get_length();
+        current_size += extent_length;
+        get_by_ext(sizes_by_ext, extent.get_type()).account_in(extent_length);
+        overall_io.account_in(extent_length);
+        if (p_src) {
+          trans_io.account_in(extent_length);
+          get_by_ext(
+            get_by_src(trans_io_by_src_ext, *p_src),
+            extent.get_type()).account_in(extent_length);
+        }
         intrusive_ptr_add_ref(&extent);
         lru.push_back(extent);
 
         // trim to capacity
         while (current_size > capacity) {
-          do_remove_from_lru(lru.front());
+          do_remove_from_lru(lru.front(), p_src);
         }
       }
     }
@@ -1493,7 +1534,7 @@ class Cache {
       LOG_PREFIX(Cache::LRU::clear);
       for (auto iter = lru.begin(); iter != lru.end();) {
 	SUBDEBUG(seastore_cache, "clearing {}", *iter);
-	do_remove_from_lru(*(iter++));
+	do_remove_from_lru(*(iter++), nullptr);
       }
     }
 

src/crimson/os/seastore/seastore.cc

@@ -590,6 +590,7 @@ seastar::future<> SeaStore::report_stats()
   ceph_assert(seastar::this_shard_id() == primary_core);
   shard_device_stats.resize(seastar::smp::count);
   shard_io_stats.resize(seastar::smp::count);
+  shard_cache_stats.resize(seastar::smp::count);
   return shard_stores.invoke_on_all([this](const Shard &local_store) {
     bool report_detail = false;
     double seconds = 0;
@@ -602,6 +603,8 @@ seastar::future<> SeaStore::report_stats()
       local_store.get_device_stats(report_detail, seconds);
     shard_io_stats[seastar::this_shard_id()] =
       local_store.get_io_stats(report_detail, seconds);
+    shard_cache_stats[seastar::this_shard_id()] =
+      local_store.get_cache_stats(report_detail, seconds);
   }).then([this] {
     LOG_PREFIX(SeaStore);
     auto now = seastar::lowres_clock::now();
@@ -697,6 +700,24 @@ seastar::future<> SeaStore::report_stats()
                  << ") ";
     }
     INFO("details: {}", oss_pending.str());
+
+    cache_stats_t cache_total = {};
+    for (const auto& s : shard_cache_stats) {
+      cache_total.add(s);
+    }
+    cache_size_stats_t lru_sizes_ps = cache_total.lru_sizes;
+    lru_sizes_ps.size /= seastar::smp::count;
+    lru_sizes_ps.num_extents /= seastar::smp::count;
+    cache_io_stats_t lru_io_ps = cache_total.lru_io;
+    lru_io_ps.in_size /= seastar::smp::count;
+    lru_io_ps.in_num_extents /= seastar::smp::count;
+    lru_io_ps.out_size /= seastar::smp::count;
+    lru_io_ps.out_num_extents /= seastar::smp::count;
+    INFO("cache lru: total{} {}; per-shard: total{} {}",
+         cache_total.lru_sizes,
+         cache_io_stats_printer_t{seconds, cache_total.lru_io},
+         lru_sizes_ps,
+         cache_io_stats_printer_t{seconds, lru_io_ps});
     return seastar::now();
   });
 }
@@ -2599,6 +2620,13 @@ shard_stats_t SeaStore::Shard::get_io_stats(
   return ret;
 }
 
+cache_stats_t SeaStore::Shard::get_cache_stats(
+    bool report_detail, double seconds) const
+{
+  return transaction_manager->get_cache_stats(
+      report_detail, seconds);
+}
+
 std::unique_ptr<SeaStore> make_seastore(
   const std::string &device)
 {

src/crimson/os/seastore/seastore.h

@@ -210,6 +210,8 @@ class SeaStore final : public FuturizedStore {
 
     shard_stats_t get_io_stats(bool report_detail, double seconds) const;
 
+    cache_stats_t get_cache_stats(bool report_detail, double seconds) const;
+
   private:
     struct internal_context_t {
       CollectionRef ch;
@@ -585,6 +587,7 @@ class SeaStore final : public FuturizedStore {
     seastar::lowres_clock::time_point::min();
   mutable std::vector<device_stats_t> shard_device_stats;
   mutable std::vector<shard_stats_t> shard_io_stats;
+  mutable std::vector<cache_stats_t> shard_cache_stats;
 };
 
 std::unique_ptr<SeaStore> make_seastore(

src/crimson/os/seastore/seastore_types.cc

@@ -980,4 +980,36 @@ std::ostream& operator<<(std::ostream& out, const writer_stats_printer_t& p)
   return out;
 }
 
+std::ostream& operator<<(std::ostream& out, const cache_io_stats_printer_t& p)
+{
+  constexpr const char* dfmt = "{:.2f}";
+  out << "in("
+      << fmt::format(dfmt, p.stats.get_in_mbs(p.seconds))
+      << "MiB/s,"
+      << fmt::format(dfmt, p.stats.get_in_avg_kb())
+      << "KiB,"
+      << fmt::format(dfmt, p.stats.in_num_extents/p.seconds)
+      << "ps) out("
+      << fmt::format(dfmt, p.stats.get_out_mbs(p.seconds))
+      << "MiB/s,"
+      << fmt::format(dfmt, p.stats.get_out_avg_kb())
+      << "KiB,"
+      << fmt::format(dfmt, p.stats.out_num_extents/p.seconds)
+      << "ps)";
+  return out;
+}
+
+std::ostream& operator<<(std::ostream& out, const cache_size_stats_t& p)
+{
+  constexpr const char* dfmt = "{:.2f}";
+  out << "("
+      << fmt::format(dfmt, p.get_mb())
+      << "MiB,"
+      << fmt::format(dfmt, p.get_avg_kb())
+      << "KiB,"
+      << p.num_extents
+      << ")";
+  return out;
 }
+
+} // namespace crimson::os::seastore