Merge pull request ceph#45962 from ronen-fr/wip-rf-sbe-test2

test/osd: test environment for the scrubber "backend"

Reviewed-by: Samuel Just <[email protected]>
Reviewed-by: Matan Breizman <[email protected]>

Author: ronen-fr

src/crimson/osd/ops_executer.cc

@@ -611,7 +611,7 @@ OpsExecuter::do_execute_op(OSDOp& osd_op)
     });
   case CEPH_OSD_OP_OMAPSETVALS:
 #if 0
-    if (!pg.get_pool().info.supports_omap()) {
+    if (!pg.get_pgpool().info.supports_omap()) {
       return crimson::ct_error::operation_not_supported::make();
     }
 #endif
@@ -620,7 +620,7 @@ OpsExecuter::do_execute_op(OSDOp& osd_op)
     }, true);
   case CEPH_OSD_OP_OMAPSETHEADER:
 #if 0
-    if (!pg.get_pool().info.supports_omap()) {
+    if (!pg.get_pgpool().info.supports_omap()) {
       return crimson::ct_error::operation_not_supported::make();
     }
 #endif
@@ -630,7 +630,7 @@ OpsExecuter::do_execute_op(OSDOp& osd_op)
     }, true);
   case CEPH_OSD_OP_OMAPRMKEYRANGE:
 #if 0
-    if (!pg.get_pool().info.supports_omap()) {
+    if (!pg.get_pgpool().info.supports_omap()) {
       return crimson::ct_error::operation_not_supported::make();
     }
 #endif
@@ -639,7 +639,7 @@ OpsExecuter::do_execute_op(OSDOp& osd_op)
     }, true);
   case CEPH_OSD_OP_OMAPRMKEYS:
     /** TODO: Implement supports_omap()
-    if (!pg.get_pool().info.supports_omap()) {
+    if (!pg.get_pgpool().info.supports_omap()) {
       return crimson::ct_error::operation_not_supported::make();
     }*/
     return do_write_op([&osd_op] (auto& backend, auto& os, auto& txn) {
@@ -714,7 +714,7 @@ std::vector<pg_log_entry_t> OpsExecuter::prepare_transaction(
 
 // Defined here because there is a circular dependency between OpsExecuter and PG
 uint32_t OpsExecuter::get_pool_stripe_width() const {
-  return pg->get_pool().info.get_stripe_width();
+  return pg->get_pgpool().info.get_stripe_width();
 }
 
 // Defined here because there is a circular dependency between OpsExecuter and PG
@@ -908,7 +908,7 @@ static PG::interruptible_future<> do_pgnls(
   }
   const auto pg_start = pg.get_pgid().pgid.get_hobj_start();
   const auto pg_end = \
-    pg.get_pgid().pgid.get_hobj_end(pg.get_pool().info.get_pg_num());
+    pg.get_pgid().pgid.get_hobj_end(pg.get_pgpool().info.get_pg_num());
   return do_pgnls_common(pg_start,
                          pg_end,
                          pg.get_backend(),
@@ -952,7 +952,7 @@ static PG::interruptible_future<> do_pgnls_filtered(
   return seastar::do_with(std::move(filter),
     [&, lower_bound=std::move(lower_bound)](auto&& filter) {
       const auto pg_start = pg.get_pgid().pgid.get_hobj_start();
-      const auto pg_end = pg.get_pgid().pgid.get_hobj_end(pg.get_pool().info.get_pg_num());
+      const auto pg_end = pg.get_pgid().pgid.get_hobj_end(pg.get_pgpool().info.get_pg_num());
       return do_pgnls_common(pg_start,
                              pg_end,
                              pg.get_backend(),
@@ -1037,7 +1037,7 @@ static PG::interruptible_future<> do_pgls(
   }
   const auto pg_start = pg.get_pgid().pgid.get_hobj_start();
   const auto pg_end =
-    pg.get_pgid().pgid.get_hobj_end(pg.get_pool().info.get_pg_num());
+    pg.get_pgid().pgid.get_hobj_end(pg.get_pgpool().info.get_pg_num());
   return do_pgls_common(pg_start,
 			pg_end,
 			pg.get_backend(),
@@ -1081,7 +1081,7 @@ static PG::interruptible_future<> do_pgls_filtered(
   return seastar::do_with(std::move(filter),
     [&, lower_bound=std::move(lower_bound)](auto&& filter) {
       const auto pg_start = pg.get_pgid().pgid.get_hobj_start();
-      const auto pg_end = pg.get_pgid().pgid.get_hobj_end(pg.get_pool().info.get_pg_num());
+      const auto pg_end = pg.get_pgid().pgid.get_hobj_end(pg.get_pgpool().info.get_pg_num());
       return do_pgls_common(pg_start,
                             pg_end,
                             pg.get_backend(),

src/crimson/osd/pg.cc

@@ -331,7 +331,7 @@ void PG::prepare_write(pg_info_t &info,
   }
   pglog.write_log_and_missing(
     t, &km, coll_ref->get_cid(), pgmeta_oid,
-    peering_state.get_pool().info.require_rollback());
+    peering_state.get_pgpool().info.require_rollback());
   if (!km.empty()) {
     t.omap_setkeys(coll_ref->get_cid(), pgmeta_oid, km);
   }
@@ -628,15 +628,15 @@ PG::do_osd_ops_execute(
     // check for full
     if ((ox->delta_stats.num_bytes > 0 ||
       ox->delta_stats.num_objects > 0) &&
-      get_pool().info.has_flag(pg_pool_t::FLAG_FULL)) {
+      get_pgpool().info.has_flag(pg_pool_t::FLAG_FULL)) {
       const auto& m = ox->get_message();
       if (m.get_reqid().name.is_mds() ||   // FIXME: ignore MDS for now
         m.has_flag(CEPH_OSD_FLAG_FULL_FORCE)) {
         logger().info(" full, but proceeding due to FULL_FORCE or MDS");
       } else if (m.has_flag(CEPH_OSD_FLAG_FULL_TRY)) {
         // they tried, they failed.
         logger().info(" full, replying to FULL_TRY op");
-        if (get_pool().info.has_flag(pg_pool_t::FLAG_FULL_QUOTA))
+        if (get_pgpool().info.has_flag(pg_pool_t::FLAG_FULL_QUOTA))
           return interruptor::make_ready_future<OpsExecuter::rep_op_fut_tuple>(
             seastar::now(),
             OpsExecuter::osd_op_ierrorator::future<>(

src/crimson/osd/pg.h

@@ -462,8 +462,8 @@ class PG : public boost::intrusive_ref_counter<
     return get_info().history.same_interval_since;
   }
 
-  const auto& get_pool() const {
-    return peering_state.get_pool();
+  const auto& get_pgpool() const {
+    return peering_state.get_pgpool();
   }
   pg_shard_t get_primary() const {
     return peering_state.get_primary();
@@ -702,7 +702,7 @@ class PG : public boost::intrusive_ref_counter<
   interruptible_future<std::tuple<bool, int>> already_complete(const osd_reqid_t& reqid);
   int get_recovery_op_priority() const {
     int64_t pri = 0;
-    get_pool().info.opts.get(pool_opts_t::RECOVERY_OP_PRIORITY, &pri);
+    get_pgpool().info.opts.get(pool_opts_t::RECOVERY_OP_PRIORITY, &pri);
     return  pri > 0 ? pri : crimson::common::local_conf()->osd_recovery_op_priority;
   }
   seastar::future<> mark_unfound_lost(int) {

src/include/types_fmt.h

@@ -43,6 +43,22 @@ struct fmt::formatter<std::list<A>> {
   }
 };
 
+template <class A>
+struct fmt::formatter<std::vector<A>> {
+  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+
+  template <typename FormatContext>
+  auto format(const std::vector<A>& l, FormatContext& ctx)
+  {
+    std::string_view sep = "[";
+    for (const auto& e : l) {
+      fmt::format_to(ctx.out(), "{}{}", sep, e);
+      sep = ",";
+    }
+    return fmt::format_to(ctx.out(), "]");
+  }
+};
+
 template <class A>
 struct fmt::formatter<std::set<A>> {
   constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }

src/osd/ECBackend.h

@@ -675,7 +675,8 @@ class ECBackend : public PGBackend {
     ScrubMap &map,
     ScrubMapBuilder &pos,
     ScrubMap::object &o) override;
-  uint64_t be_get_ondisk_size(uint64_t logical_size) override {
+
+  uint64_t be_get_ondisk_size(uint64_t logical_size) const final {
     return sinfo.logical_to_next_chunk_offset(logical_size);
   }
   void _failed_push(const hobject_t &hoid,

src/osd/OSD.cc

@@ -9140,7 +9140,7 @@ void OSD::split_pgs(
       *i,
       split_bits,
       i->ps(),
-      &child->get_pool().info,
+      &child->get_pgpool().info,
       rctx.transaction);
     parent->split_into(
       i->pgid,

src/osd/PG.cc

@@ -215,7 +215,7 @@ PG::PG(OSDService *o, OSDMapRef curmap,
     curmap,
     this,
     this),
-  pool(recovery_state.get_pool()),
+  pool(recovery_state.get_pgpool()),
   info(recovery_state.get_info())
 {
 #ifdef PG_DEBUG_REFS

src/osd/PG.h

@@ -163,18 +163,9 @@ class PGRecoveryStats {
  *
  */
 
-/// Facilitating scrub-realated object access to private PG data
-class ScrubberPasskey {
-private:
-  friend class Scrub::ReplicaReservations;
-  friend class PrimaryLogScrub;
-  friend class PgScrubber;
-  ScrubberPasskey() {}
-  ScrubberPasskey(const ScrubberPasskey&) = default;
-  ScrubberPasskey& operator=(const ScrubberPasskey&) = delete;
-};
-
-class PG : public DoutPrefixProvider, public PeeringState::PeeringListener {
+class PG : public DoutPrefixProvider,
+	   public PeeringState::PeeringListener,
+	   public Scrub::PgScrubBeListener {
   friend struct NamedState;
   friend class PeeringState;
   friend class PgScrubber;
@@ -245,7 +236,7 @@ class PG : public DoutPrefixProvider, public PeeringState::PeeringListener {
     return pg_id;
   }
 
-  const PGPool& get_pool() const {
+  const PGPool& get_pgpool() const final {
     return pool;
   }
   uint64_t get_last_user_version() const {
@@ -261,6 +252,11 @@ class PG : public DoutPrefixProvider, public PeeringState::PeeringListener {
     return info.history.same_interval_since;
   }
 
+  bool is_waiting_for_unreadable_object() const final
+  {
+    return !waiting_for_unreadable_object.empty();
+  }
+
   static void set_last_scrub_stamp(
     utime_t t, pg_history_t &history, pg_stat_t &stats) {
     stats.last_scrub_stamp = t;
@@ -346,7 +342,7 @@ class PG : public DoutPrefixProvider, public PeeringState::PeeringListener {
   int get_acting_primary() const {
     return recovery_state.get_acting_primary();
   }
-  pg_shard_t get_primary() const {
+  pg_shard_t get_primary() const final {
     return recovery_state.get_primary();
   }
   const std::vector<int> get_up() const {
@@ -1408,20 +1404,29 @@ class PG : public DoutPrefixProvider, public PeeringState::PeeringListener {
   const pg_info_t &info;
 
 
-// ScrubberPasskey getters:
+// ScrubberPasskey getters/misc:
 public:
-  const pg_info_t& get_pg_info(ScrubberPasskey) const {
-    return info;
-  }
-
-  OSDService* get_pg_osd(ScrubberPasskey) const {
-    return osd;
-  }
-
-  requested_scrub_t& get_planned_scrub(ScrubberPasskey) {
-    return m_planned_scrub;
-  }
-
+ const pg_info_t& get_pg_info(ScrubberPasskey) const final { return info; }
+
+ OSDService* get_pg_osd(ScrubberPasskey) const { return osd; }
+
+ requested_scrub_t& get_planned_scrub(ScrubberPasskey)
+ {
+   return m_planned_scrub;
+ }
+
+ void force_object_missing(ScrubberPasskey,
+                           const std::set<pg_shard_t>& peer,
+                           const hobject_t& oid,
+                           eversion_t version) final
+ {
+   recovery_state.force_object_missing(peer, oid, version);
+ }
+
+ uint64_t logical_to_ondisk_size(uint64_t logical_size) const final
+ {
+   return get_pgbackend()->be_get_ondisk_size(logical_size);
+ }
 };
 
 #endif

src/osd/PGBackend.h

@@ -586,7 +586,8 @@ typedef std::shared_ptr<const OSDMap> OSDMapRef;
      ScrubMapBuilder &pos);
 
    virtual uint64_t be_get_ondisk_size(
-     uint64_t logical_size) = 0;
+     uint64_t logical_size) const = 0;
+
    virtual int be_deep_scrub(
      const hobject_t &oid,
      ScrubMap &map,

src/osd/PeeringState.h

@@ -20,6 +20,7 @@
 #include "PGStateUtils.h"
 #include "PGPeeringEvent.h"
 #include "osd_types.h"
+#include "osd_types_fmt.h"
 #include "os/ObjectStore.h"
 #include "OSDMap.h"
 #include "MissingLoc.h"
@@ -57,6 +58,22 @@ struct PGPool {
   }
 };
 
+template <>
+struct fmt::formatter<PGPool> {
+  template <typename ParseContext>
+  constexpr auto parse(ParseContext& ctx) { return ctx.begin(); }
+
+  template <typename FormatContext>
+  auto format(const PGPool& pool, FormatContext& ctx)
+  {
+    return fmt::format_to(ctx.out(),
+                          "{}/{}({})",
+                          pool.id,
+                          pool.name,
+                          pool.info);
+  }
+};
+
 struct PeeringCtx;
 
 // [primary only] content recovery state
@@ -2136,7 +2153,7 @@ class PeeringState : public MissingLoc::MappingInfo {
     return last_rollback_info_trimmed_to_applied;
   }
   /// Returns stable reference to internal pool structure
-  const PGPool &get_pool() const {
+  const PGPool &get_pgpool() const {
     return pool;
   }
   /// Returns reference to current osdmap