Merge pull request ceph#55131 from ronen-fr/wip-rf-reserver

osd/scrub: partial implementation of scrub reserver
Reviewed-by: Samuel Just <[email protected]>-

Author: ronen-fr

src/common/AsyncReserver.h

@@ -264,6 +264,39 @@ class AsyncReserver {
     do_queues();
   }
 
+  /**
+   * The synchronous version of request_reservation
+   * Used to handle requests from OSDs that do not support the async interface
+   * to scrub replica reservations, but still must count towards the max
+   * active reservations.
+   */
+  bool request_reservation_or_fail(
+      T item,		     ///< [in] reservation key
+      Context *on_reserved   ///< [in] callback to be called on reservation
+  )
+  {
+    std::lock_guard l(lock);
+    ceph_assert(!queue_pointers.count(item) && !in_progress.count(item));
+
+    if (in_progress.size() >= max_allowed) {
+      rdout(10) << fmt::format("{}: request: {} denied", __func__, item)
+		<< dendl;
+      return false;
+    }
+
+    const unsigned prio = UINT_MAX;
+    Reservation r(item, prio, on_reserved, nullptr);
+    queues[prio].push_back(r);
+    queue_pointers.insert(std::make_pair(
+	item, std::make_pair(prio, --(queues[prio]).end())));
+    do_queues();
+    // the new request should be in_progress now
+    ceph_assert(in_progress.count(item));
+    rdout(10) << fmt::format("{}: request: {} granted", __func__, item)
+	      << dendl;
+    return true;
+  }
+
   /**
    * Cancels reservation
    *

src/messages/MOSDScrubReserve.h

@@ -19,7 +19,7 @@
 
 class MOSDScrubReserve : public MOSDFastDispatchOp {
 private:
-  static constexpr int HEAD_VERSION = 2;
+  static constexpr int HEAD_VERSION = 3;
   static constexpr int COMPAT_VERSION = 1;
 public:
   using reservation_nonce_t = uint32_t;
@@ -35,6 +35,9 @@ class MOSDScrubReserve : public MOSDFastDispatchOp {
   int32_t type;
   pg_shard_t from;
   reservation_nonce_t reservation_nonce{0};
+  /// 'false' if the (legacy) primary is expecting an immediate
+  /// granted / denied response
+  bool wait_for_resources{false};
 
   epoch_t get_map_epoch() const override {
     return map_epoch;
@@ -48,12 +51,12 @@ class MOSDScrubReserve : public MOSDFastDispatchOp {
       map_epoch(0), type(-1) {}
   MOSDScrubReserve(spg_t pgid,
 		   epoch_t map_epoch,
-		   int type,
+		   ReserveMsgOp type_code,
 		   pg_shard_t from,
 		   reservation_nonce_t nonce)
     : MOSDFastDispatchOp{MSG_OSD_SCRUB_RESERVE, HEAD_VERSION, COMPAT_VERSION},
       pgid(pgid), map_epoch(map_epoch),
-      type(type), from(from), reservation_nonce{nonce} {}
+      type(static_cast<int32_t>(type_code)), from(from), reservation_nonce{nonce} {}
 
   std::string_view get_type_name() const {
     return "MOSDScrubReserve";
@@ -63,7 +66,7 @@ class MOSDScrubReserve : public MOSDFastDispatchOp {
     out << "MOSDScrubReserve(" << pgid << " ";
     switch (type) {
     case REQUEST:
-      out << "REQUEST ";
+      out << (wait_for_resources ? "QREQUEST " : "REQUEST ");
       break;
     case GRANT:
       out << "GRANT ";
@@ -89,10 +92,16 @@ class MOSDScrubReserve : public MOSDFastDispatchOp {
     decode(from, p);
     if (header.version >= 2) {
       decode(reservation_nonce, p);
+      if (header.version >= 3) {
+        decode(wait_for_resources, p);
+      } else {
+        wait_for_resources = false;
+      }
     } else {
       // a zero nonce (identifying legacy senders) is ignored when
       // checking the request for obsolescence
       reservation_nonce = 0;
+      wait_for_resources = false;
     }
   }
 
@@ -103,6 +112,7 @@ class MOSDScrubReserve : public MOSDFastDispatchOp {
     encode(type, payload);
     encode(from, payload);
     encode(reservation_nonce, payload);
+    encode(wait_for_resources, payload);
   }
 private:
   template<class T, typename... Args>

src/osd/scrubber/osd_scrub.cc

@@ -473,6 +473,11 @@ bool OsdScrub::inc_scrubs_remote(pg_t pgid)
   return m_resource_bookkeeper.inc_scrubs_remote(pgid);
 }
 
+void OsdScrub::enqueue_remote_reservation(pg_t pgid)
+{
+  m_resource_bookkeeper.enqueue_remote_reservation(pgid);
+}
+
 void OsdScrub::dec_scrubs_remote(pg_t pgid)
 {
   m_resource_bookkeeper.dec_scrubs_remote(pgid);

src/osd/scrubber/osd_scrub.h

@@ -70,6 +70,7 @@ class OsdScrub {
       bool is_high_priority);
   void dec_scrubs_local();
   bool inc_scrubs_remote(pg_t pgid);
+  void enqueue_remote_reservation(pg_t pgid);
   void dec_scrubs_remote(pg_t pgid);
 
   // counting the number of PGs stuck while scrubbing, waiting for objects