Merge pull request ceph#62398 from AliMasarweh/wip-alimasa-rgw-standalone-zone

RGW/standalone: refactor rgw_zone.h with configstore

Reviewed-by: Casey Bodley <[email protected]>
Reviewed-by: Daniel Gryniewicz <[email protected]>

Author: cbodley

src/rgw/driver/dbstore/config/sqlite.cc

@@ -872,6 +872,14 @@ int SQLiteConfigStore::delete_period(const DoutPrefixProvider* dpp,
   return 0;
 }
 
+int SQLiteConfigStore::update_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y,
+                                           std::string_view period_id, uint32_t epoch)
+{
+  Prefix prefix{*dpp, "dbconfig:sqlite:read_latest_epoch "}; dpp = &prefix;
+  // TODO: implement it later
+  return 0;
+}
+
 int SQLiteConfigStore::list_period_ids(const DoutPrefixProvider* dpp,
                                        optional_yield y,
                                        const std::string& marker,

src/rgw/driver/dbstore/config/sqlite.h

@@ -79,6 +79,8 @@ class SQLiteConfigStore : public sal::ConfigStore {
                       optional_yield y, const std::string& marker,
                       std::span<std::string> entries,
                       sal::ListResult<std::string>& result) override;
+  int update_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y,
+                          std::string_view period_id, uint32_t epoch) override;
 
   int write_default_zonegroup_id(const DoutPrefixProvider* dpp,
                                  optional_yield y, bool exclusive,

src/rgw/driver/immutable_config/store.cc

@@ -137,6 +137,12 @@ int ImmutableConfigStore::list_period_ids(const DoutPrefixProvider* dpp,
   return 0;
 }
 
+int ImmutableConfigStore::update_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y,
+                                              std::string_view period_id, uint32_t epoch)
+{
+  return -EROFS;
+}
+
 
 // ZoneGroup
 

src/rgw/driver/immutable_config/store.h

@@ -78,6 +78,8 @@ class ImmutableConfigStore : public ConfigStore {
                               optional_yield y, const std::string& marker,
                               std::span<std::string> entries,
                               ListResult<std::string>& result) override;
+  virtual int update_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y,
+                                  std::string_view period_id, uint32_t epoch) override;
 
   // ZoneGroup
   virtual int write_default_zonegroup_id(const DoutPrefixProvider* dpp,

src/rgw/driver/rados/config/period.cc

@@ -79,9 +79,8 @@ static int delete_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y,
   return impl->remove(dpp, y, pool, latest_oid, objv);
 }
 
-static int update_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y,
-                               ConfigImpl* impl, std::string_view period_id,
-                               uint32_t epoch)
+int RadosConfigStore::update_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y,
+                                          std::string_view period_id, uint32_t epoch)
 {
   static constexpr int MAX_RETRIES = 20;
 
@@ -91,7 +90,7 @@ static int update_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y,
     bool exclusive = false;
 
     // read existing epoch
-    int r = read_latest_epoch(dpp, y, impl, period_id, existing_epoch, &objv);
+    int r = read_latest_epoch(dpp, y, impl.get(), period_id, existing_epoch, &objv);
     if (r == -ENOENT) {
       // use an exclusive create to set the epoch atomically
       exclusive = true;
@@ -111,7 +110,7 @@ static int update_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y,
           << " -> " << epoch << " on period=" << period_id << dendl;
     }
 
-    r = write_latest_epoch(dpp, y, impl, exclusive, period_id, epoch, &objv);
+    r = write_latest_epoch(dpp, y, impl.get(), exclusive, period_id, epoch, &objv);
     if (r == -EEXIST) {
       continue; // exclusive create raced with another update, retry
     } else if (r == -ECANCELED) {
@@ -149,7 +148,8 @@ int RadosConfigStore::create_period(const DoutPrefixProvider* dpp,
     return r;
   }
 
-  (void) update_latest_epoch(dpp, y, impl.get(), info.get_id(), info.get_epoch());
+  // non const RGWPeriod
+  (void) this->update_latest_epoch(dpp, y, info.get_id(), info.get_epoch());
   return 0;
 }
 
@@ -182,8 +182,7 @@ int RadosConfigStore::delete_period(const DoutPrefixProvider* dpp,
   // read the latest_epoch
   uint32_t latest_epoch = 0;
   RGWObjVersionTracker latest_objv;
-  int r = read_latest_epoch(dpp, y, impl.get(), period_id,
-                            latest_epoch, &latest_objv);
+  int r = read_latest_epoch(dpp, y, impl.get(), period_id, latest_epoch, &latest_objv);
   if (r < 0 && r != -ENOENT) { // just delete epoch=0 on ENOENT
     ldpp_dout(dpp, 0) << "failed to read latest epoch for period "
         << period_id << ": " << cpp_strerror(r) << dendl;

src/rgw/driver/rados/config/store.h

@@ -85,6 +85,8 @@ class RadosConfigStore : public sal::ConfigStore {
                               optional_yield y, const std::string& marker,
                               std::span<std::string> entries,
                               sal::ListResult<std::string>& result) override;
+  virtual int update_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y,
+                                  std::string_view period_id, uint32_t epoch) override;
 
   // ZoneGroup
   virtual int write_default_zonegroup_id(const DoutPrefixProvider* dpp,

src/rgw/driver/rados/rgw_data_sync.cc

@@ -3232,7 +3232,7 @@ void RGWRemoteDataLog::wakeup(int shard_id, bc::flat_set<rgw_data_notify_entry>&
   data_sync_cr->wakeup(shard_id, entries);
 }
 
-int RGWRemoteDataLog::run_sync(const DoutPrefixProvider *dpp, int num_shards)
+int RGWRemoteDataLog::run_sync(const DoutPrefixProvider *dpp, int num_shards, rgw::sal::ConfigStore* cfgstore)
 {
   // construct and start bid manager for data sync fairness
   const auto& control_pool = sc.env->driver->svc()->zone->get_zone_params().control_pool;

src/rgw/driver/rados/rgw_data_sync.h

@@ -416,7 +416,7 @@ class RGWRemoteDataLog : public RGWCoroutinesManager {
   int read_recovering_shards(const DoutPrefixProvider *dpp, const int num_shards, std::set<int>& recovering_shards);
   int read_shard_status(const DoutPrefixProvider *dpp, int shard_id, std::set<std::string>& lagging_buckets,std::set<std::string>& recovering_buckets, rgw_data_sync_marker* sync_marker, const int max_entries);
   int init_sync_status(const DoutPrefixProvider *dpp, int num_shards);
-  int run_sync(const DoutPrefixProvider *dpp, int num_shards);
+  int run_sync(const DoutPrefixProvider *dpp, int num_shards, rgw::sal::ConfigStore* cfgstore);
 
   void wakeup(int shard_id, bc::flat_set<rgw_data_notify_entry>& entries);
 };
@@ -483,7 +483,7 @@ class RGWDataSyncStatusManager : public DoutPrefixProvider {
     return source_log.read_source_log_shards_next(dpp, shard_markers, result);
   }
 
-  int run(const DoutPrefixProvider *dpp) { return source_log.run_sync(dpp, num_shards); }
+  int run(const DoutPrefixProvider *dpp, rgw::sal::ConfigStore* cfgstore) { return source_log.run_sync(dpp, num_shards, cfgstore); }
 
   void wakeup(int shard_id, bc::flat_set<rgw_data_notify_entry>& entries) { return source_log.wakeup(shard_id, entries); }
 

src/rgw/driver/rados/rgw_period.cc

@@ -2,6 +2,8 @@
 // vim: ts=8 sw=2 smarttab ft=cpp
 
 #include "rgw_sync.h"
+#include "rgw_sal.h"
+#include "rgw_sal_config.h"
 
 #include "services/svc_zone.h"
 
@@ -27,115 +29,6 @@ int RGWPeriod::get_zonegroup(RGWZoneGroup& zonegroup,
   return -ENOENT;
 }
 
-int RGWPeriod::get_latest_epoch(const DoutPrefixProvider *dpp, epoch_t& latest_epoch, optional_yield y)
-{
-  RGWPeriodLatestEpochInfo info;
-
-  int ret = read_latest_epoch(dpp, info, y);
-  if (ret < 0) {
-    return ret;
-  }
-
-  latest_epoch = info.epoch;
-
-  return 0;
-}
-
-int RGWPeriod::delete_obj(const DoutPrefixProvider *dpp, optional_yield y)
-{
-  rgw_pool pool(get_pool(cct));
-
-  // delete the object for each period epoch
-  for (epoch_t e = 1; e <= epoch; e++) {
-    RGWPeriod p{get_id(), e};
-    rgw_raw_obj oid{pool, p.get_period_oid()};
-    auto sysobj = sysobj_svc->get_obj(oid);
-    int ret = sysobj.wop().remove(dpp, y);
-    if (ret < 0) {
-      ldpp_dout(dpp, 0) << "WARNING: failed to delete period object " << oid
-          << ": " << cpp_strerror(-ret) << dendl;
-    }
-  }
-
-  // delete the .latest_epoch object
-  rgw_raw_obj oid{pool, get_period_oid_prefix() + get_latest_epoch_oid()};
-  auto sysobj = sysobj_svc->get_obj(oid);
-  int ret = sysobj.wop().remove(dpp, y);
-  if (ret < 0) {
-    ldpp_dout(dpp, 0) << "WARNING: failed to delete period object " << oid
-        << ": " << cpp_strerror(-ret) << dendl;
-  }
-  return ret;
-}
-
-int RGWPeriod::update(const DoutPrefixProvider *dpp, optional_yield y)
-{
-  auto zone_svc = sysobj_svc->get_zone_svc();
-  ldpp_dout(dpp, 20) << __func__ << " realm " << realm_id << " period " << get_id() << dendl;
-  list<string> zonegroups;
-  int ret = zone_svc->list_zonegroups(dpp, zonegroups);
-  if (ret < 0) {
-    ldpp_dout(dpp, 0) << "ERROR: failed to list zonegroups: " << cpp_strerror(-ret) << dendl;
-    return ret;
-  }
-
-  // clear zone short ids of removed zones. period_map.update() will add the
-  // remaining zones back
-  period_map.short_zone_ids.clear();
-
-  for (auto& iter : zonegroups) {
-    RGWZoneGroup zg(string(), iter);
-    ret = zg.init(dpp, cct, sysobj_svc, y);
-    if (ret < 0) {
-      ldpp_dout(dpp, 0) << "WARNING: zg.init() failed: " << cpp_strerror(-ret) << dendl;
-      continue;
-    }
-
-    if (zg.realm_id != realm_id) {
-      ldpp_dout(dpp, 20) << "skipping zonegroup " << zg.get_name() << " zone realm id " << zg.realm_id << ", not on our realm " << realm_id << dendl;
-      continue;
-    }
-
-    if (zg.master_zone.empty()) {
-      ldpp_dout(dpp, 0) << "ERROR: zonegroup " << zg.get_name() << " should have a master zone " << dendl;
-      return -EINVAL;
-    }
-
-    if (zg.zones.find(zg.master_zone) == zg.zones.end()) {
-      ldpp_dout(dpp, 0) << "ERROR: zonegroup " << zg.get_name()
-                   << " has a non existent master zone "<< dendl;
-      return -EINVAL;
-    }
-
-    if (zg.is_master_zonegroup()) {
-      master_zonegroup = zg.get_id();
-      master_zone = zg.master_zone;
-    }
-
-    int ret = period_map.update(zg, cct);
-    if (ret < 0) {
-      return ret;
-    }
-  }
-
-  ret = period_config.read(dpp, sysobj_svc, realm_id, y);
-  if (ret < 0 && ret != -ENOENT) {
-    ldpp_dout(dpp, 0) << "ERROR: failed to read period config: "
-        << cpp_strerror(ret) << dendl;
-    return ret;
-  }
-  return 0;
-}
-
-void RGWPeriod::fork()
-{
-  ldout(cct, 20) << __func__ << " realm " << realm_id << " period " << id << dendl;
-  predecessor_uuid = id;
-  id = get_staging_id(realm_id);
-  period_map.reset();
-  realm_epoch++;
-}
-
 static int read_sync_status(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver, rgw_meta_sync_status *sync_status)
 {
   rgw::sal::RadosStore* rados_store = static_cast<rgw::sal::RadosStore*>(driver);
@@ -201,99 +94,6 @@ int RGWPeriod::update_sync_status(const DoutPrefixProvider *dpp,
   return 0;
 }
 
-int RGWPeriod::commit(const DoutPrefixProvider *dpp,
-		      rgw::sal::Driver* driver,
-		      RGWRealm& realm, const RGWPeriod& current_period,
-                      std::ostream& error_stream, optional_yield y,
-		      bool force_if_stale)
-{
-  auto zone_svc = sysobj_svc->get_zone_svc();
-  ldpp_dout(dpp, 20) << __func__ << " realm " << realm.get_id() << " period " << current_period.get_id() << dendl;
-  // gateway must be in the master zone to commit
-  if (master_zone != zone_svc->get_zone_params().get_id()) {
-    error_stream << "Cannot commit period on zone "
-        << zone_svc->get_zone_params().get_id() << ", it must be sent to "
-        "the period's master zone " << master_zone << '.' << std::endl;
-    return -EINVAL;
-  }
-  // period predecessor must match current period
-  if (predecessor_uuid != current_period.get_id()) {
-    error_stream << "Period predecessor " << predecessor_uuid
-        << " does not match current period " << current_period.get_id()
-        << ". Use 'period pull' to get the latest period from the master, "
-        "reapply your changes, and try again." << std::endl;
-    return -EINVAL;
-  }
-  // realm epoch must be 1 greater than current period
-  if (realm_epoch != current_period.get_realm_epoch() + 1) {
-    error_stream << "Period's realm epoch " << realm_epoch
-        << " does not come directly after current realm epoch "
-        << current_period.get_realm_epoch() << ". Use 'realm pull' to get the "
-        "latest realm and period from the master zone, reapply your changes, "
-        "and try again." << std::endl;
-    return -EINVAL;
-  }
-  // did the master zone change?
-  if (master_zone != current_period.get_master_zone()) {
-    // store the current metadata sync status in the period
-    int r = update_sync_status(dpp, driver, current_period, error_stream, force_if_stale);
-    if (r < 0) {
-      ldpp_dout(dpp, 0) << "failed to update metadata sync status: "
-          << cpp_strerror(-r) << dendl;
-      return r;
-    }
-    // create an object with a new period id
-    r = create(dpp, y, true);
-    if (r < 0) {
-      ldpp_dout(dpp, 0) << "failed to create new period: " << cpp_strerror(-r) << dendl;
-      return r;
-    }
-    // set as current period
-    r = realm.set_current_period(dpp, *this, y);
-    if (r < 0) {
-      ldpp_dout(dpp, 0) << "failed to update realm's current period: "
-          << cpp_strerror(-r) << dendl;
-      return r;
-    }
-    ldpp_dout(dpp, 4) << "Promoted to master zone and committed new period "
-        << id << dendl;
-    return 0;
-  }
-  // period must be based on current epoch
-  if (epoch != current_period.get_epoch()) {
-    error_stream << "Period epoch " << epoch << " does not match "
-        "predecessor epoch " << current_period.get_epoch()
-        << ". Use 'period pull' to get the latest epoch from the master zone, "
-        "reapply your changes, and try again." << std::endl;
-    return -EINVAL;
-  }
-  // set period as next epoch
-  set_id(current_period.get_id());
-  set_epoch(current_period.get_epoch() + 1);
-  set_predecessor(current_period.get_predecessor());
-  realm_epoch = current_period.get_realm_epoch();
-  // write the period to rados
-  int r = store_info(dpp, false, y);
-  if (r < 0) {
-    ldpp_dout(dpp, 0) << "failed to store period: " << cpp_strerror(-r) << dendl;
-    return r;
-  }
-  // set as latest epoch
-  r = update_latest_epoch(dpp, epoch, y);
-  if (r < 0) {
-    ldpp_dout(dpp, 0) << "failed to set latest epoch: " << cpp_strerror(-r) << dendl;
-    return r;
-  }
-  r = reflect(dpp, y);
-  if (r < 0) {
-    ldpp_dout(dpp, 0) << "failed to update local objects: " << cpp_strerror(-r) << dendl;
-    return r;
-  }
-  ldpp_dout(dpp, 4) << "Committed new epoch " << epoch
-      << " for period " << id << dendl;
-  return 0;
-}
-
 void RGWPeriod::generate_test_instances(list<RGWPeriod*> &o)
 {
   RGWPeriod *z = new RGWPeriod;