Merge pull request ceph#62818 from ronen-fr/wip-rf-iocnt-plus

osd/scrub: performance counters: count I/Os, use unlabeled counters

Reviewed-by: Alex Ainscow <[email protected]>
Reviewed-by: Radoslaw Zarzynski <[email protected]>
Reviewed-by: Bill Scales <[email protected]>
Reviewed-by: Samuel Just <[email protected]>

Author: ronen-fr

src/osd/osd_perf_counters.cc

@@ -356,7 +356,122 @@ PerfCounters *build_osd_logger(CephContext *cct) {
   osd_plb.add_u64_counter(
   l_osd_watch_timeouts, "watch_timeouts",
   "Number of watches that timed out or were blocklisted",
-  NULL, PerfCountersBuilder::PRIO_USEFUL);
+  nullptr, PerfCountersBuilder::PRIO_USEFUL);
+
+  // scrub I/O (no EC vs. replicated differentiation)
+  osd_plb.add_u64_counter(l_osd_scrub_omapgetheader_cnt, "scrub_omapgetheader_cnt", "scrub omap get header calls count");
+  osd_plb.add_u64_counter(l_osd_scrub_omapgetheader_bytes, "scrub_omapgetheader_bytes", "scrub omap get header bytes read");
+  osd_plb.add_u64_counter(l_osd_scrub_omapget_cnt, "scrub_omapget_cnt", "scrub omap get calls count");
+  osd_plb.add_u64_counter(l_osd_scrub_omapget_bytes, "scrub_omapget_bytes", "scrub omap get bytes read");
+  // scrub I/O performed for replicated pools
+  osd_plb.add_u64_counter(l_osd_scrub_rppool_getattr_cnt, "scrub_replicated_getattr_cnt", "scrub replicated pool getattr calls count");
+  osd_plb.add_u64_counter(l_osd_scrub_rppool_stats_cnt, "scrub_replicated_stats_cnt", "scrub replicated pool stats calls count");
+  osd_plb.add_u64_counter(l_osd_scrub_rppool_read_cnt, "scrub_replicated_read_cnt", "scrub replicated pool read calls count");
+  osd_plb.add_u64_counter(l_osd_scrub_rppool_read_bytes, "scrub_replicated_read_bytes", "scrub replicated pool read bytes read");
+  // scrub I/O performed for EC pools
+  osd_plb.add_u64_counter(l_osd_scrub_ec_getattr_cnt, "scrub_ec_getattr_cnt", "scrub EC getattr calls count");
+  osd_plb.add_u64_counter(l_osd_scrub_ec_stats_cnt, "scrub_ec_stats_cnt", "scrub EC stats calls count");
+  osd_plb.add_u64_counter(l_osd_scrub_ec_read_cnt, "scrub_ec_read_cnt", "scrub EC read calls count");
+  osd_plb.add_u64_counter(l_osd_scrub_ec_read_bytes, "scrub_ec_read_bytes", "scrub EC read bytes read");
+
+  // scrub - replicated pools
+  osd_plb.add_u64_counter(
+      l_osd_scrub_rppool_started,
+      "num_scrubs_started_replicated",
+      "replicated scrubs attempted count");
+  osd_plb.add_u64_counter(
+      l_osd_scrub_rppool_active_started,
+      "num_scrubs_past_reservation_replicated",
+      "replicated scrubs count");
+  osd_plb.add_u64_counter(
+      l_osd_scrub_rppool_successful,
+      "successful_scrubs_replicated",
+      "successful replicated scrubs count");
+  osd_plb.add_time_avg(
+      l_osd_scrub_rppool_successful_elapsed,
+      "successful_scrubs_replicated_elapsed",
+      "time to complete a successful replicated scrub");
+  osd_plb.add_u64_counter(
+      l_osd_scrub_rppool_failed, "failed_scrubs_replicated",
+      "failed replicated scrubs count");
+  osd_plb.add_time_avg(
+      l_osd_scrub_rppool_failed_elapsed,
+      "failed_scrubs_replicated_elapsed",
+      "time to scrub failure replicated");
+
+  // the replica reservation process - replicated pool
+  osd_plb.add_u64_counter(
+      l_osd_scrub_rppool_reserv_success,
+      "scrub_replicated_scrub_reservations_completed",
+      "successfully completed reservation processes");
+  osd_plb.add_time_avg(
+      l_osd_scrub_rppool_reserv_successful_elapsed,
+      "scrub_replicated_successful_reservations_elapsed",
+      "time to scrub reservation completion");
+  osd_plb.add_u64_counter(
+      l_osd_scrub_rppool_reserv_aborted,
+      "scrub_replicated_reservation_process_aborted",
+      "scrub replicated pool reservation was aborted");
+  osd_plb.add_u64_counter(
+      l_osd_scrub_rppool_reserv_rejected,
+      "scrub_replicated_reservation_process_failure",
+      "scrub replicated pool reservation failed due to replica denial");
+  osd_plb.add_u64_counter(
+      l_osd_scrub_rppool_reserv_skipped,
+      "scrub_replicated_reservation_process_skipped",
+      "scrub replicated pool reservation skipped for high priority scrub");
+  osd_plb.add_time_avg(
+      l_osd_scrub_rppool_reserv_failed_elapsed,
+      "scrub_replicated_failed_reservations_elapsed",
+      "scrub replicated pool time for scrub reservation to fail");
+  osd_plb.add_u64(
+      l_osd_scrub_rppool_reserv_secondaries_num,
+      "scrub_replicated_replicas_in_reservation",
+      "scrub replicated pool number of replicas to reserve");
+
+  // scrub - EC
+  osd_plb.add_u64_counter(
+      l_osd_scrub_ec_started, "num_scrubs_started_ec",
+      "EC scrubs attempted count");
+  osd_plb.add_u64_counter(
+      l_osd_scrub_ec_active_started, "num_scrubs_past_reservation_ec",
+      "EC scrubs count");
+  osd_plb.add_u64_counter(
+      l_osd_scrub_ec_successful, "successful_scrubs_ec",
+      "successful EC scrubs count");
+  osd_plb.add_time_avg(
+      l_osd_scrub_ec_successful_elapsed, "successful_scrubs_ec_elapsed",
+      "time to complete a successful EC scrub");
+  osd_plb.add_u64_counter(
+      l_osd_scrub_ec_failed, "failed_scrubs_ec", "failed scrubs count EC");
+  osd_plb.add_time_avg(
+      l_osd_scrub_ec_failed_elapsed, "failed_scrubs_ec_elapsed",
+      "time to scrub failure ec");
+
+  // the replica reservation process - EC
+  osd_plb.add_u64_counter(
+      l_osd_scrub_ec_reserv_success, "scrub_ec_reservations_completed",
+      "successfully completed reservation processes EC");
+  osd_plb.add_time_avg(
+      l_osd_scrub_ec_reserv_successful_elapsed,
+      "scrub_ec_successful_reservations_elapsed",
+      "time to EC scrub reservation completion");
+  osd_plb.add_u64_counter(
+      l_osd_scrub_ec_reserv_aborted, "scrub_ec_reservation_process_aborted",
+      "scrub reservation was aborted EC");
+  osd_plb.add_u64_counter(
+      l_osd_scrub_ec_reserv_rejected, "scrub_ec_reservation_process_failure",
+      "scrub reservation failed due to replica denial EC");
+  osd_plb.add_u64_counter(
+      l_osd_scrub_ec_reserv_skipped, "scrub_ec_reservation_process_skipped",
+      "scrub reservation skipped for high priority scrub EC");
+  osd_plb.add_time_avg(
+      l_osd_scrub_ec_reserv_failed_elapsed,
+      "scrub_ec_failed_reservations_elapsed",
+      "time for scrub reservation to fail EC");
+  osd_plb.add_u64(
+      l_osd_scrub_ec_reserv_secondaries_num, "scrub_ec_replicas_in_reservation",
+      "number of replicas to reserve EC");
 
   return osd_plb.create_perf_counters();
 }
@@ -409,27 +524,12 @@ PerfCounters *build_scrub_labeled_perf(CephContext *cct, std::string label)
 
   scrub_perf.set_prio_default(PerfCountersBuilder::PRIO_INTERESTING);
 
-  scrub_perf.add_u64_counter(scrbcnt_started, "num_scrubs_started", "scrubs attempted count");
-  scrub_perf.add_u64_counter(scrbcnt_active_started, "num_scrubs_past_reservation", "scrubs count");
-  scrub_perf.add_u64_counter(scrbcnt_failed, "failed_scrubs", "failed scrubs count");
-  scrub_perf.add_u64_counter(scrbcnt_successful, "successful_scrubs", "successful scrubs count");
-  scrub_perf.add_time_avg(scrbcnt_failed_elapsed, "failed_scrubs_elapsed", "time to scrub failure");
-  scrub_perf.add_time_avg(scrbcnt_successful_elapsed, "successful_scrubs_elapsed", "time to scrub completion");
-
   scrub_perf.add_u64_counter(scrbcnt_preempted, "preemptions", "preemptions on scrubs");
   scrub_perf.add_u64_counter(scrbcnt_chunks_selected, "chunk_selected", "chunk selection during scrubs");
   scrub_perf.add_u64_counter(scrbcnt_chunks_busy, "chunk_busy", "chunk busy during scrubs");
   scrub_perf.add_u64_counter(scrbcnt_blocked, "locked_object", "waiting on locked object events");
   scrub_perf.add_u64_counter(scrbcnt_write_blocked, "write_blocked_by_scrub", "write blocked by scrub");
 
-  // the replica reservation process
-  scrub_perf.add_u64_counter(scrbcnt_resrv_success, "scrub_reservations_completed", "successfully completed reservation processes");
-  scrub_perf.add_time_avg(scrbcnt_resrv_successful_elapsed, "successful_reservations_elapsed", "time to scrub reservation completion");
-  scrub_perf.add_u64_counter(scrbcnt_resrv_aborted, "reservation_process_aborted", "scrub reservation was aborted");
-  scrub_perf.add_u64_counter(scrbcnt_resrv_rejected, "reservation_process_failure", "scrub reservation failed due to replica denial");
-  scrub_perf.add_u64_counter(scrbcnt_resrv_skipped, "reservation_process_skipped", "scrub reservation skipped for high priority scrub");
-  scrub_perf.add_time_avg(scrbcnt_resrv_failed_elapsed, "failed_reservations_elapsed", "time for scrub reservation to fail");
-  scrub_perf.add_u64(scrbcnt_resrv_replicas_num, "replicas_in_reservation", "number of replicas in reservation");
 
   return scrub_perf.create_perf_counters();
 }

src/osd/osd_perf_counters.h

@@ -7,7 +7,7 @@
 #include "common/perf_counters.h"
 #include "common/perf_counters_key.h"
 
-enum {
+enum osd_counter_idx_t {
   l_osd_first = 10000,
   l_osd_op_wip,
   l_osd_op,
@@ -143,6 +143,75 @@ enum {
 
   l_osd_watch_timeouts,
 
+  // scrub I/O (no EC vs. replicated differentiation)
+  l_osd_scrub_omapgetheader_cnt,  ///< omap get header calls count
+  l_osd_scrub_omapgetheader_bytes,  ///< bytes read by omap get header
+  l_osd_scrub_omapget_cnt,      ///< omap get calls count
+  l_osd_scrub_omapget_bytes,    ///< total bytes read by omap get
+
+  // ----   scrub I/O - replicated pools
+  l_osd_scrub_rppool_getattr_cnt, ///< get_attr calls count
+  l_osd_scrub_rppool_stats_cnt, ///< stats calls count
+  l_osd_scrub_rppool_read_cnt, ///< read calls count
+  l_osd_scrub_rppool_read_bytes, ///< total bytes read
+
+  // ----   scrub I/O - EC
+  l_osd_scrub_ec_getattr_cnt, ///< get_attr calls count
+  l_osd_scrub_ec_stats_cnt, ///< stats calls count
+  l_osd_scrub_ec_read_cnt, ///< read calls count
+  l_osd_scrub_ec_read_bytes, ///< total bytes read
+
+  // ----   scrub - replicated pools
+  l_osd_scrub_rppool_started, ///< scrubs that got started
+  l_osd_scrub_rppool_active_started, ///< scrubs that got past replicas reservation
+  l_osd_scrub_rppool_successful, ///< successful scrubs count
+  l_osd_scrub_rppool_successful_elapsed, ///< time to complete a successful scrub
+  l_osd_scrub_rppool_failed, ///< failed scrubs count
+  l_osd_scrub_rppool_failed_elapsed, ///< time from start to failure
+
+  // ----   scrub reservation process - replicated pools
+
+  /// successful replicas reservation count
+  l_osd_scrub_rppool_reserv_success,
+  /// time to complete a successful replicas reservation
+  l_osd_scrub_rppool_reserv_successful_elapsed,
+  /// failed attempt to reserve replicas due to an abort
+  l_osd_scrub_rppool_reserv_aborted,
+  /// reservation failed due to a 'rejected' response
+  l_osd_scrub_rppool_reserv_rejected,
+  /// reservation skipped for high-priority scrubs
+  l_osd_scrub_rppool_reserv_skipped,
+  /// time for a replicas reservation process to fail
+  l_osd_scrub_rppool_reserv_failed_elapsed,
+  /// number of replicas
+  l_osd_scrub_rppool_reserv_secondaries_num,
+
+
+  // ----   scrub - EC
+  l_osd_scrub_ec_started, ///< scrubs that got started
+  l_osd_scrub_ec_active_started, /// scrubs that got past secondaries reservation
+  l_osd_scrub_ec_successful, ///< successful scrubs count
+  l_osd_scrub_ec_successful_elapsed, ///< time to complete a successful scrub
+  l_osd_scrub_ec_failed, ///< failed scrubs count
+  l_osd_scrub_ec_failed_elapsed, ///< time from start to failure
+
+  // ----   scrub reservation process - EC
+
+  /// successful replicas reservation count
+  l_osd_scrub_ec_reserv_success,
+  /// time to complete a successful replicas reservation
+  l_osd_scrub_ec_reserv_successful_elapsed,
+  /// failed attempt to reserve replicas due to an abort
+  l_osd_scrub_ec_reserv_aborted,
+  /// reservation failed due to a 'rejected' response
+  l_osd_scrub_ec_reserv_rejected,
+  /// reservation skipped for high-priority scrubs
+  l_osd_scrub_ec_reserv_skipped,
+  /// time for a replicas reservation process to fail
+  l_osd_scrub_ec_reserv_failed_elapsed,
+  /// number of replicas
+  l_osd_scrub_ec_reserv_secondaries_num,
+
   l_osd_last,
 };
 
@@ -192,20 +261,6 @@ PerfCounters *build_recoverystate_perf(CephContext *cct);
 enum {
   scrbcnt_first = 20500,
 
-  // -- basic statistics --
-  /// The number of times we started a scrub
-  scrbcnt_started,
-  /// # scrubs that got past replicas reservation
-  scrbcnt_active_started,
-  /// # successful scrubs
-  scrbcnt_successful,
-  /// time to complete a successful scrub
-  scrbcnt_successful_elapsed,
-  /// # failed scrubs
-  scrbcnt_failed,
-  /// time for a scrub to fail
-  scrbcnt_failed_elapsed,
-
   // -- interruptions of various types
   /// # preemptions
   scrbcnt_preempted,
@@ -218,22 +273,6 @@ enum {
   /// # write blocked by the scrub
   scrbcnt_write_blocked,
 
-  // -- replicas reservation
-  /// # successfully completed reservation steps
-  scrbcnt_resrv_success,
-  /// time to complete a successful replicas reservation
-  scrbcnt_resrv_successful_elapsed,
-  /// # failed attempt to reserve replicas due to an abort
-  scrbcnt_resrv_aborted,
-  /// # reservation failed due to a 'rejected' response
-  scrbcnt_resrv_rejected,
-  /// # reservation skipped for high-priority scrubs
-  scrbcnt_resrv_skipped,
-  /// time for a replicas reservation process to fail
-  scrbcnt_resrv_failed_elapsed,
-  /// # number of replicas
-  scrbcnt_resrv_replicas_num,
-
   scrbcnt_last,
 };
 

src/osd/scrubber/pg_scrubber.cc

@@ -999,7 +999,7 @@ std::optional<uint64_t> PgScrubber::select_range()
 
 void PgScrubber::select_range_n_notify()
 {
-  get_counters_set().inc(scrbcnt_chunks_selected);
+  get_labeled_counters()->inc(scrbcnt_chunks_selected);
   auto num_chunk_objects = select_range();
   if (num_chunk_objects.has_value()) {
     // the next chunk to handle is not blocked
@@ -1010,7 +1010,7 @@ void PgScrubber::select_range_n_notify()
     // we will wait for the objects range to become available for scrubbing
     dout(10) << __func__ << ": selected chunk is busy" << dendl;
     m_osds->queue_scrub_chunk_busy(m_pg, Scrub::scrub_prio_t::low_priority);
-    get_counters_set().inc(scrbcnt_chunks_busy);
+    get_labeled_counters()->inc(scrbcnt_chunks_busy);
   }
 }
 
@@ -1042,7 +1042,7 @@ bool PgScrubber::write_blocked_by_scrub(const hobject_t& soid)
     return false;
   }
 
-  get_counters_set().inc(scrbcnt_write_blocked);
+  get_labeled_counters()->inc(scrbcnt_write_blocked);
   dout(20) << __func__ << " " << soid << " can preempt? "
 	   << preemption_data.is_preemptable() << " already preempted? "
 	   << preemption_data.was_preempted() << dendl;
@@ -2525,11 +2525,22 @@ void PgScrubber::set_scrub_duration(std::chrono::milliseconds duration)
   });
 }
 
-PerfCounters& PgScrubber::get_counters_set() const
+PerfCounters* PgScrubber::get_osd_perf_counters() const
 {
-  return *m_osds->get_scrub_services().get_perf_counters(
+  return m_osds->logger;
+}
+
+const Scrub::ScrubCounterSet& PgScrubber::get_unlabeled_counters() const
+{
+  return m_pg->pool.info.is_replicated() ? io_counters_replicated
+					 : io_counters_ec;
+}
+
+PerfCounters* PgScrubber::get_labeled_counters() const
+{
+  return m_osds->get_scrub_services().get_perf_counters(
       (m_pg->pool.info.is_replicated() ? pg_pool_t::TYPE_REPLICATED
-				       : pg_pool_t::TYPE_ERASURE),
+                                       : pg_pool_t::TYPE_ERASURE),
       (m_is_deep ? scrub_level_t::deep : scrub_level_t::shallow));
 }