nvmeofgw: fix host issue during redeploy

leonidc · leonidc · commit abdb740458ec · 2025-03-19T13:45:29.000+02:00
This commit fixes the issue when during redeploy hosts might stuck due to many failover/failbacks during very short time frame Fixes: https://tracker.ceph.com/issues/70497 Signed-off-by: Leonid Chernin <leonidc@il.ibm.com>
diff --git a/src/common/options/mon.yaml.in b/src/common/options/mon.yaml.in
@@ -77,7 +77,14 @@ options:
   level: advanced
   desc: Period in seconds from last beacon to monitor marking a  NVMeoF gateway as
     failed
-  default: 10
+  default: 15
+  services:
+  - mon
+- name: mon_nvmeofgw_skip_failovers_interval
+  type: secs
+  level: advanced
+  desc: Period in seconds in which no failovers are performed in GW's pool-group
+  default: 12
   services:
   - mon
 - name: mon_nvmeofgw_set_group_id_retry
diff --git a/src/mon/NVMeofGwMap.cc b/src/mon/NVMeofGwMap.cc
@@ -285,6 +285,16 @@ void NVMeofGwMap::track_deleting_gws(const NvmeGroupKey& group_key,
   }
 }
 
+void NVMeofGwMap::skip_failovers_for_group(const NvmeGroupKey& group_key)
+{
+  const auto skip_failovers = g_conf().get_val<std::chrono::seconds>
+    ("mon_nvmeofgw_skip_failovers_interval");
+  for (auto& gw_created: created_gws[group_key]) {
+    gw_created.second.allow_failovers_ts = std::chrono::system_clock::now()
+        + skip_failovers;
+  }
+}
+
 int NVMeofGwMap::process_gw_map_gw_no_subsys_no_listeners(
   const NvmeGwId &gw_id, const NvmeGroupKey& group_key, bool &propose_pending)
 {
@@ -523,9 +533,14 @@ void  NVMeofGwMap::find_failover_candidate(
 #define MIN_NUM_ANA_GROUPS 0xFFF
   int min_num_ana_groups_in_gw = 0;
   int current_ana_groups_in_gw = 0;
+  std::chrono::system_clock::time_point now = std::chrono::system_clock::now();
   NvmeGwId min_loaded_gw_id = ILLEGAL_GW_ID;
   auto& gws_states = created_gws[group_key];
   auto gw_state = gws_states.find(gw_id);
+  if (gw_state->second.allow_failovers_ts > now) {
+    dout(4) << "gw " << gw_id << " skip-failovers is set " << dendl;
+    return;
+  }
 
   // this GW may handle several ANA groups and  for each
   // of them need to found the candidate GW
diff --git a/src/mon/NVMeofGwMap.h b/src/mon/NVMeofGwMap.h
@@ -87,6 +87,7 @@ class NVMeofGwMap
        const NvmeGroupKey& group_key, bool &map_modified);
   void gw_performed_startup(const NvmeGwId &gw_id,
        const NvmeGroupKey& group_key, bool &propose_pending);
+  void skip_failovers_for_group(const NvmeGroupKey& group_key);
 private:
   int  do_delete_gw(const NvmeGwId &gw_id, const NvmeGroupKey& group_key);
   int  do_erase_gw_id(const NvmeGwId &gw_id,
diff --git a/src/mon/NVMeofGwMon.cc b/src/mon/NVMeofGwMon.cc
@@ -153,9 +153,35 @@ version_t NVMeofGwMon::get_trim_to() const
   return 0;
 }
 
+/**
+ * restore_pending_map_info
+ * function called during new paxos epochs
+ * function called to restore in pending map all data that is not serialized
+ * to paxos peons. Othervise it would be overriden in "pending_map = map"
+ * currently  just "allow_failovers_ts" variable is restored
+ */
+void NVMeofGwMon::restore_pending_map_info(NVMeofGwMap & tmp_map) {
+  std::chrono::system_clock::time_point now = std::chrono::system_clock::now();
+  for (auto& created_map_pair: tmp_map.created_gws) {
+    auto group_key = created_map_pair.first;
+    NvmeGwMonStates& gw_created_map = created_map_pair.second;
+    for (auto& gw_created_pair: gw_created_map) {
+      auto gw_id = gw_created_pair.first;
+      if (gw_created_pair.second.allow_failovers_ts > now) {
+        // restore not persistent information upon new epochs
+        dout(10) << " restore skip-failovers timeout for gw  " << gw_id  << dendl;
+        pending_map.created_gws[group_key][gw_id].allow_failovers_ts =
+          gw_created_pair.second.allow_failovers_ts;
+      }
+    }
+  }
+}
+
 void NVMeofGwMon::create_pending()
 {
+  NVMeofGwMap tmp_map = pending_map;
   pending_map = map;// deep copy of the object
+  restore_pending_map_info(tmp_map);
   pending_map.epoch++;
   dout(10) << " pending " << pending_map  << dendl;
 }
@@ -641,11 +667,12 @@ bool NVMeofGwMon::prepare_beacon(MonOpRequestRef op)
       if (pending_map.created_gws[group_key][gw_id].availability ==
 	  gw_availability_t::GW_AVAILABLE) {
 	dout(1) << " Warning :GW marked as Available in the NVmeofGwMon "
-		<< "database, performed full startup - Apply GW!"
+		<< "database, performed full startup - Apply it but don't allow failover!"
 		<< gw_id << dendl;
 	 process_gw_down(gw_id, group_key, gw_propose, avail);
-	 LastBeacon lb = {gw_id, group_key};
-	 last_beacon[lb] = now; //Update last beacon
+	 pending_map.skip_failovers_for_group(group_key);
+	 dout(4) << "set skip-failovers for gw's group " << gw_id << " group "
+	 << group_key << dendl;
       } else if (
 	pending_map.created_gws[group_key][gw_id].performed_full_startup ==
 	false) {
@@ -654,6 +681,8 @@ bool NVMeofGwMon::prepare_beacon(MonOpRequestRef op)
 	pending_map.created_gws[group_key][gw_id].addr_vect =
 	    entity_addrvec_t(con->get_peer_addr());
       }
+      LastBeacon lb = {gw_id, group_key};
+      last_beacon[lb] = now; //Update last beacon
       goto set_propose;
     }
   // gw already created
@@ -774,15 +803,19 @@ bool NVMeofGwMon::prepare_beacon(MonOpRequestRef op)
           % beacons_till_ack) == 0))|| (!apply_ack_logic) ) {
     send_ack = true;
     if (apply_ack_logic) {
-      dout(10) << "ack sent: beacon index "
+      dout(20) << "ack sent: beacon index "
        << pending_map.created_gws[group_key][gw_id].beacon_index
        << " gw " << gw_id <<dendl;
     }
   }
   if (send_ack && ((!gw_propose && epoch_filter_enabled) ||
-                    (propose && !epoch_filter_enabled)) ) {
-          // if epoch-filter-bit: send ack to beacon in case no propose
-          //or if changed something not relevant to gw-epoch
+                    (propose && !epoch_filter_enabled) ||
+                    (avail == gw_availability_t::GW_CREATED)) ) {
+          /* always send beacon ack to gw in Created state,
+           * it should be temporary state
+           * if epoch-filter-bit: send ack to beacon in case no propose
+           * or if changed something not relevant to gw-epoch
+          */
     if (gw_created) {
       // respond with a map slice correspondent to the same GW
       ack_map.created_gws[group_key][gw_id] = map.created_gws[group_key][gw_id];
diff --git a/src/mon/NVMeofGwMon.h b/src/mon/NVMeofGwMon.h
@@ -98,6 +98,7 @@ class NVMeofGwMon: public PaxosService,
        NvmeGwId &gw_id, NvmeGroupKey& group_key);
   epoch_t get_ack_map_epoch(bool gw_created, const NvmeGroupKey& group_key);
   void recreate_gw_epoch();
+  void restore_pending_map_info(NVMeofGwMap & tmp_map);
 };
 
 #endif /* MON_NVMEGWMONITOR_H_ */
diff --git a/src/mon/NVMeofGwTypes.h b/src/mon/NVMeofGwTypes.h
@@ -131,6 +131,28 @@ struct NvmeGwMonState {
   //ceph entity address allocated for the GW-client that represents this GW-id
   entity_addrvec_t addr_vect;
   uint16_t beacon_index = 0;
+  /**
+   * during redeploy action and maybe other emergency use-cases gw performs scenario
+   * that we call fast-reboot. It quickly reboots(due to redeploy f.e) and sends the
+   * first beacon to monitor in "Created" state while according to monitor FSM it
+   * still appears "Available".
+   * This lost of synchronizarion with GW is detected by monitor. After fast reboot, the monitor
+   * still considers this GW as not eligible for owning any ANA group until it becomes
+   * in Available state (sends the next beacon that includes the subsystems information).
+   * In this specific fast-reboot case, we prefer to avoid failing over the ANA groups
+   * that were owned by this GW for a short time frame, assuming that this GW will be
+   * in Available State in few seconds. Doing too many failovers and failbacks
+   * in a very short times frame, on many GWs, is causing a lot of pain to the
+   * initiators, up to the point that they might stuck.
+   *
+   * So was decided to set to GW new timeout varible "allow_failovers_ts" - no failovers
+   * are performed to GW's pool-group during 12 seconds from the time it is set.
+   * this variable is not persistent - not serialized to peers, so need to prevent
+   * it from being overriden by new epochs in monitor's function create_pending -
+   * function restore_pending_map_info is called for this purpose
+  */
+  std::chrono::system_clock::time_point allow_failovers_ts =
+             std::chrono::system_clock::now();
   NvmeGwMonState(): ana_grp_id(REDUNDANT_GW_ANA_GROUP_ID) {}
 
   NvmeGwMonState(NvmeAnaGrpId id)
