Merge pull request ceph#60871 from leonidc/leonidc-epoch-filter

Epoch filtering

Reviewed-by: Samuel Just <[email protected]>
Reviewed-by: Aviv Caro <[email protected]>
Reviewed-by: Ronen Friedman <[email protected]>

Author: ronen-fr

src/common/options/mon.yaml.in

@@ -89,6 +89,11 @@ options:
     receives a monitor group ID assignment before the gateway is fully up during
     initialization, a retry is required.
   default: 1000
+- name: mon_nvmeofgw_beacons_till_ack
+  type: uint
+  level: advanced
+  default: 15
+  desc: Number of beacons from MonClient before NVMeofGwMon sends ack-map to it
   services:
   - mon
 - name: mon_nvmeofgw_delete_grace

src/include/ceph_features.h

@@ -148,6 +148,7 @@ DEFINE_CEPH_FEATURE(42, 1, MSGR_KEEPALIVE2)  // 4.3 (for consistency)
 DEFINE_CEPH_FEATURE(43, 1, OSD_POOLRESEND)   // 4.13
 DEFINE_CEPH_FEATURE(44, 2, NVMEOFHA)
 DEFINE_CEPH_FEATURE_RETIRED(45, 1, OSD_SET_ALLOC_HINT, JEWEL, LUMINOUS)
+DEFINE_CEPH_FEATURE(45, 2, NVMEOFHAMAP)
 // available
 DEFINE_CEPH_FEATURE(46, 1, OSD_FADVISE_FLAGS)
 DEFINE_CEPH_FEATURE_RETIRED(46, 1, OSD_REPOP, JEWEL, LUMINOUS) // overlap
@@ -226,6 +227,7 @@ DEFINE_CEPH_FEATURE_RETIRED(63, 1, RESERVED_BROKEN, LUMINOUS, QUINCY) // client-
 	 CEPH_FEATURE_MSGR_KEEPALIVE2 |	\
 	 CEPH_FEATURE_OSD_POOLRESEND |	\
 	 CEPH_FEATUREMASK_NVMEOFHA | \
+	 CEPH_FEATUREMASK_NVMEOFHAMAP | \
 	 CEPH_FEATURE_OSD_FADVISE_FLAGS |     \
 	 CEPH_FEATURE_MDS_QUOTA | \
          CEPH_FEATURE_CRUSH_V4 |	     \

src/mon/NVMeofGwMap.cc

@@ -83,6 +83,14 @@ int NVMeofGwMap::cfg_add_gw(
   const NvmeGwId &gw_id, const NvmeGroupKey& group_key)
 {
   std::set<NvmeAnaGrpId> allocated;
+  if (HAVE_FEATURE(mon->get_quorum_con_features(), NVMEOFHAMAP)) {
+    auto gw_epoch_it = gw_epoch.find(group_key);
+    if (gw_epoch_it == gw_epoch.end()) {
+      gw_epoch[group_key] = epoch;
+      dout(10) << "Allocated first gw_epoch : group_key "
+          << group_key << " epoch " << gw_epoch[group_key] << dendl;
+    }
+  }
   for (auto& itr: created_gws[group_key]) {
     allocated.insert(itr.second.ana_grp_id);
     if (itr.first == gw_id) {
@@ -190,8 +198,10 @@ int  NVMeofGwMap::do_erase_gw_id(const NvmeGwId &gw_id,
     fsm_timers.erase(group_key);
 
   created_gws[group_key].erase(gw_id);
-  if (created_gws[group_key].size() == 0)
+  if (created_gws[group_key].size() == 0) {
     created_gws.erase(group_key);
+    gw_epoch.erase(group_key);
+  }
   return 0;
 }
 
@@ -221,6 +231,23 @@ int NVMeofGwMap::do_delete_gw(
   return -EINVAL;
 }
 
+void  NVMeofGwMap::gw_performed_startup(const NvmeGwId &gw_id,
+      const NvmeGroupKey& group_key, bool &propose_pending)
+{
+  dout(4) << "GW  performed the full startup " << gw_id << dendl;
+  propose_pending = true;
+  increment_gw_epoch( group_key);
+}
+
+void NVMeofGwMap::increment_gw_epoch(const NvmeGroupKey& group_key)
+{
+  if (HAVE_FEATURE(mon->get_quorum_con_features(), NVMEOFHAMAP)) {
+    gw_epoch[group_key] ++;
+    dout(4) << "incremented epoch of " << group_key
+         << " " << gw_epoch[group_key] << dendl;
+  }
+}
+
 int NVMeofGwMap::get_num_namespaces(const NvmeGwId &gw_id,
     const NvmeGroupKey& group_key,  const BeaconSubsystems&  subs)
 {
@@ -273,7 +300,10 @@ int NVMeofGwMap::process_gw_map_gw_no_subsys_no_listeners(
     gw_id, group_key, state_itr.second,state_itr.first, propose_pending);
     }
     propose_pending = true; // map should reflect that gw becames Created
-    if (propose_pending) validate_gw_map(group_key);
+    if (propose_pending) {
+      validate_gw_map(group_key);
+      increment_gw_epoch(group_key);
+    }
   } else {
     dout(1)  << __FUNCTION__ << "ERROR GW-id was not found in the map "
          << gw_id << dendl;
@@ -299,7 +329,10 @@ int NVMeofGwMap::process_gw_map_gw_down(
       state_itr.second = gw_states_per_group_t::GW_STANDBY_STATE;
     }
     propose_pending = true; // map should reflect that gw becames Unavailable
-    if (propose_pending) validate_gw_map(group_key);
+    if (propose_pending) {
+      validate_gw_map(group_key);
+      increment_gw_epoch(group_key);
+    }
   } else {
     dout(1)  << __FUNCTION__ << "ERROR GW-id was not found in the map "
 	     << gw_id << dendl;
@@ -338,7 +371,10 @@ void NVMeofGwMap::process_gw_map_ka(
 	state_itr.first, last_osd_epoch, propose_pending);
     }
   }
-  if (propose_pending) validate_gw_map(group_key);
+  if (propose_pending) {
+    validate_gw_map(group_key);
+    increment_gw_epoch(group_key);
+  }
 }
 
 void NVMeofGwMap::handle_abandoned_ana_groups(bool& propose)
@@ -387,6 +423,7 @@ void NVMeofGwMap::handle_abandoned_ana_groups(bool& propose)
     }
     if (propose) {
       validate_gw_map(group_key);
+      increment_gw_epoch(group_key);
     }
   }
 }
@@ -751,7 +788,10 @@ void NVMeofGwMap::fsm_handle_gw_delete(
 	    << "for GW " << gw_id  << dendl;
   }
   }
-  if (map_modified) validate_gw_map(group_key);
+  if (map_modified) {
+    validate_gw_map(group_key);
+    increment_gw_epoch(group_key);
+  }
 }
 
 void NVMeofGwMap::fsm_handle_to_expired(
@@ -817,7 +857,10 @@ void NVMeofGwMap::fsm_handle_to_expired(
     //another Trigger for GW down (failover)
     process_gw_map_gw_down(gw_id, group_key, map_modified);
   }
-  if (map_modified) validate_gw_map(group_key);
+  if (map_modified) {
+    validate_gw_map(group_key);
+    increment_gw_epoch(group_key);
+  }
 }
 
 struct CMonRequestProposal : public Context {

src/mon/NVMeofGwMap.h

@@ -45,6 +45,21 @@ class NVMeofGwMap
 
   // map that handles timers started by all Gateway FSMs
   std::map<NvmeGroupKey, NvmeGwTimers> fsm_timers;
+  /**
+   * gw_epoch
+   *
+   * Mapping from NvmeGroupKey -> epoch_t e such that e is the most recent
+   * map epoch which affects NvmeGroupKey.
+   *
+   * The purpose of this map is to allow us to determine whether a particular
+   * gw needs to be sent the current map.  If a gw with NvmeGroupKey key already
+   * has map epoch e, we only need to send a new map if gw_epoch[key] > e.  See
+   * check_sub for this logic.
+   *
+   * Map mutators generally need to invoke increment_gw_epoch(group_key) when
+   * updating the map with a change affecting gws in group_key.
+   */
+  std::map<NvmeGroupKey, epoch_t> gw_epoch;
 
   void to_gmap(std::map<NvmeGroupKey, NvmeGwMonClientStates>& Gmap) const;
   void track_deleting_gws(const NvmeGroupKey& group_key,
@@ -70,6 +85,8 @@ class NVMeofGwMap
     NvmeAnaGrpId anagrpid, uint8_t value);
   void handle_gw_performing_fast_reboot(const NvmeGwId &gw_id,
        const NvmeGroupKey& group_key, bool &map_modified);
+  void gw_performed_startup(const NvmeGwId &gw_id,
+       const NvmeGroupKey& group_key, bool &propose_pending);
 private:
   int  do_delete_gw(const NvmeGwId &gw_id, const NvmeGroupKey& group_key);
   int  do_erase_gw_id(const NvmeGwId &gw_id,
@@ -115,6 +132,7 @@ class NVMeofGwMap
     NvmeAnaGrpId anagrpid);
   void validate_gw_map(
     const NvmeGroupKey& group_key);
+  void increment_gw_epoch(const NvmeGroupKey& group_key);
 
 public:
   int blocklist_gw(
@@ -123,21 +141,31 @@ class NVMeofGwMap
 
   void encode(ceph::buffer::list &bl, uint64_t features) const {
     using ceph::encode;
-    ENCODE_START(1, 1, bl);
+    uint8_t version = 1;
+    if (HAVE_FEATURE(features, NVMEOFHAMAP)) {
+       version = 2;
+    }
+    ENCODE_START(version, version, bl);
     encode(epoch, bl);// global map epoch
 
     encode(created_gws, bl, features); //Encode created GWs
     encode(fsm_timers, bl, features);
+    if (version >= 2) {
+      encode(gw_epoch, bl);
+    }
     ENCODE_FINISH(bl);
   }
 
   void decode(ceph::buffer::list::const_iterator &bl) {
     using ceph::decode;
-    DECODE_START(1, bl);
-    decode(epoch, bl);
+    DECODE_START(2, bl);
 
+    decode(epoch, bl);
     decode(created_gws, bl);
     decode(fsm_timers, bl);
+    if (struct_v >= 2) {
+      decode(gw_epoch, bl);
+    }
     DECODE_FINISH(bl);
   }