Merge pull request ceph#62288 from athanatos/sjust/wip-crimson-recovery-69412-replica

crimson: take read lock on replica during final push commit

Reviewed-by: Matan Breizman <[email protected]>

Author: athanatos

src/crimson/osd/object_context.h

@@ -107,12 +107,22 @@ class ObjectContext : public ceph::common::intrusive_lru_base<
     ceph_assert(is_head());
     obs = std::move(_obs);
     ssc = std::move(_ssc);
+    // ObjectContextLoader::load_and_lock* rely on this to determine whether to
+    // start loading from disk.  As this method fills in the metadata, such
+    // loading will not be necessary in the future. loading_started will already
+    // be set if this is invoked from ObjectContextLoader::load_and_lock*
+    loading_started = true;
     fully_loaded = true;
   }
 
   void set_clone_state(ObjectState &&_obs) {
     ceph_assert(!is_head());
     obs = std::move(_obs);
+    // ObjectContextLoader::load_and_lock* rely on this to determine whether to
+    // start loading from disk.  As this method fills in the metadata, such
+    // loading will not be necessary in the future. loading_started will already
+    // be set if this is invoked from ObjectContextLoader::load_and_lock*
+    loading_started = true;
     fully_loaded = true;
   }
 

src/crimson/osd/object_context_loader.cc

@@ -22,8 +22,9 @@ ObjectContextLoader::load_and_lock_head(Manager &manager, RWState::State lock_ty
     auto [obc, _] = obc_registry.get_cached_obc(manager.target);
     manager.set_state_obc(manager.head_state, obc);
   }
-  ceph_assert(manager.target_state.is_empty());
-  manager.set_state_obc(manager.target_state, manager.head_state.obc);
+  if (manager.target_state.is_empty()) {
+    manager.set_state_obc(manager.target_state, manager.head_state.obc);
+  }
 
   if (manager.target_state.obc->loading_started) {
     co_await manager.target_state.lock_to(lock_type);
@@ -45,7 +46,6 @@ ObjectContextLoader::load_and_lock_clone(
   auto releaser = manager.get_releaser();
 
   ceph_assert(!manager.target.is_head());
-  ceph_assert(manager.target_state.is_empty());
 
   if (manager.head_state.is_empty()) {
     auto [obc, _] = obc_registry.get_cached_obc(manager.target.get_head());
@@ -65,6 +65,9 @@ ObjectContextLoader::load_and_lock_clone(
   }
 
   if (manager.options.resolve_clone) {
+    // target_state must be empty because we won't know which object to load
+    // until now
+    ceph_assert(manager.target_state.is_empty());
     auto resolved_oid = resolve_oid(
       manager.head_state.obc->get_head_ss(),
       manager.target);
@@ -91,6 +94,8 @@ ObjectContextLoader::load_and_lock_clone(
      * actually can mutate a clone do not set resolve_clone, so target will not
      * become head here.
      */
+    ceph_assert(manager.options.resolve_clone);
+    ceph_assert(manager.target_state.is_empty());
     manager.set_state_obc(manager.target_state, manager.head_state.obc);
     if (lock_type != manager.head_state.state) {
       // This case isn't actually possible at the moment for the above reason.
@@ -101,11 +106,20 @@ ObjectContextLoader::load_and_lock_clone(
       manager.head_state.state = RWState::RWNONE;
     }
   } else {
-    auto [obc, _] = obc_registry.get_cached_obc(manager.target);
-    manager.set_state_obc(manager.target_state, obc);
+    // caller may have already populated this if !resolve_clone
+    if (manager.target_state.is_empty()) {
+      auto [obc, _] = obc_registry.get_cached_obc(manager.target);
+      manager.set_state_obc(manager.target_state, obc);
+    }
 
     if (manager.target_state.obc->loading_started) {
       co_await manager.target_state.lock_to(RWState::RWREAD);
+      if (!manager.target_state.obc->ssc) {
+	// A cached clone obc may have a null ssc if created via
+	// create_cached_obc_from_push_data.  This interface
+	// is responsible for fixing that if found.
+	manager.target_state.obc->ssc = manager.head_state.obc->ssc;
+      }
     } else {
       manager.target_state.lock_excl_sync();
       manager.target_state.obc->loading_started = true;
@@ -115,6 +129,8 @@ ObjectContextLoader::load_and_lock_clone(
     }
   }
 
+  ceph_assert(manager.target_state.obc->ssc);
+  ceph_assert(manager.head_state.obc->ssc);
   releaser.cancel();
 }
 

src/crimson/osd/object_context_loader.h

@@ -136,6 +136,7 @@ class ObjectContextLoader {
     friend ObjectContextLoader;
 
     void set_state_obc(state_t &s, ObjectContextRef _obc) {
+      ceph_assert(s.is_empty());
       s.obc = std::move(_obc);
       s.obc->append_to(loader.obc_set_accessing);
     }
@@ -210,6 +211,37 @@ class ObjectContextLoader {
     }
   };
 
+  /**
+   * create_cached_obc_from_push_data
+   *
+   * Creates a fresh cached obc from passed oi and ssc.
+   * Overwrites any obc already in cache for this object.
+   *
+   * Note, this interface may be used to create a clone obc
+   * with a null ssc.  The capability is useful when handling
+   * a clone push on a replica -- we don't necessarily have
+   * a valid local copy of the head since the primary may not
+   * push the head first.  That obc with a null ssc may
+   * remain in the cache.  ObjectContextLoader::load_and_lock_clone
+   * will fix the ssc member if null, but users of any other
+   * access mechanism must be aware that ssc on a clone obc may be
+   * null.
+   */
+  ObjectContextRef create_cached_obc_from_push_data(
+    const object_info_t &oi,
+    SnapSetContextRef ssc) {
+    auto obc = obc_registry.get_cached_obc(oi.soid).first;
+    if (oi.soid.is_head()) {
+      ceph_assert(ssc); // head, ssc may not be null
+      obc->set_head_state(ObjectState(oi, true), SnapSetContextRef(ssc));
+    } else {
+      // clone, ssc may be null
+      obc->set_clone_state(ObjectState(oi, true));
+      obc->set_clone_ssc(SnapSetContextRef(ssc));
+    }
+    return obc;
+  }
+
   Orderer get_obc_orderer(const hobject_t &oid) {
     Orderer ret;
     std::tie(ret.orderer_obc, std::ignore) =

src/crimson/osd/pg.cc

@@ -1315,9 +1315,10 @@ void PG::log_operation(
 void PG::replica_clear_repop_obc(
   const std::vector<pg_log_entry_t> &logv) {
   LOG_PREFIX(PG::replica_clear_repop_obc);
-  DEBUGDPP("clearing obc for {} log entries", logv.size());
+  DEBUGDPP("clearing obc for {} log entries", *this, logv.size());
   for (auto &&e: logv) {
     DEBUGDPP("clearing entry for {} from: {} to: {}",
+	     *this,
 	     e.soid,
 	     e.soid.get_object_boundary(),
 	     e.soid.get_head());

src/crimson/osd/recovery_backend.cc

@@ -50,6 +50,8 @@ void RecoveryBackend::clean_up(ceph::os::Transaction& t,
   });
   clear_temp_objs();
 
+  replica_push_targets.clear();
+
   for (auto& [soid, recovery_waiter] : recovering) {
     if ((recovery_waiter->pull_info
          && recovery_waiter->pull_info->is_complete())

src/crimson/osd/recovery_backend.h

@@ -272,6 +272,14 @@ class RecoveryBackend {
   virtual interruptible_future<> handle_backfill_op(
     Ref<MOSDFastDispatchOp> m,
     crimson::net::ConnectionXcoreRef conn);
+
+  /**
+   * replica_push_targets
+   *
+   * Holds obc on replica for in-progress pushes, see
+   * ReplicatedRecoveryBackend::handle_push
+   */
+  std::map<hobject_t, crimson::osd::ObjectContextRef> replica_push_targets;
 private:
   void handle_backfill_finish(
     MOSDPGBackfill& m,