rgw/cloud-restore: Support restoration of objects transitioned to Glacier/Tape endpoint

Restoration of objects from certain cloud services (like Glacier/Tape) could
take significant amount of time (even days). Hence store the state of such restore requests
and periodically process them.

Brief summary of changes

* Refactored existing restore code to consolidate and move all restore processing into rgw_restore* file/class

* RGWRestore class is defined to manage the restoration of objects.

* Lastly, for SAL_RADOS, FIFO is used to store and read restore entries.

Currently, this PR handles storing state of restore requests sent to cloud-glacier tier-type which need async processing.
The changes are tested with AWS Glacier Flexible Retrieval with tier_type Expedited and Standard.

Reviewed-by: Matt Benjamin <[email protected]>
Reviewed-by: Adam Emerson <[email protected]>
Reviewed-by: Jiffin Tony Thottan <[email protected]>
Reviewed-by: Daniel Gryniewicz <[email protected]>

Signed-off-by: Soumya Koduri <[email protected]>

Author: soumyakoduri

src/common/options/rgw.yaml.in

@@ -230,6 +230,7 @@ options:
   see_also:
   - rgw_enable_gc_threads
   - rgw_enable_lc_threads
+  - rgw_enable_restore_threads
   with_legacy: true
 - name: rgw_enable_gc_threads
   type: bool
@@ -246,6 +247,7 @@ options:
   see_also:
   - rgw_enable_quota_threads
   - rgw_enable_lc_threads
+  - rgw_enable_restore_threads
   with_legacy: true
 - name: rgw_enable_lc_threads
   type: bool
@@ -263,6 +265,24 @@ options:
   see_also:
   - rgw_enable_gc_threads
   - rgw_enable_quota_threads
+  - rgw_enable_restore_threads
+  with_legacy: true
+- name: rgw_enable_restore_threads
+  type: bool
+  level: advanced
+  desc: Enables the objects' restore maintenance thread.
+  long_desc: The objects restore maintenance thread is responsible for all the objects
+    restoration related maintenance work. The thread itself can be disabled, but in order
+    for the restore from the cloud to work correctly, at least one RGW in each zone needs
+    to have this thread running. Having the thread enabled on multiple RGW processes within
+    the same zone can spread some of the maintenance work between them.
+  default: true
+  services:
+  - rgw
+  see_also:
+  - rgw_enable_gc_threads
+  - rgw_enable_quota_threads
+  - rgw_enable_lc_threads
   with_legacy: true
 - name: rgw_data
   type: str
@@ -475,6 +495,35 @@ options:
   services:
   - rgw
   with_legacy: true
+- name: rgw_restore_max_objs
+  type: int
+  level: advanced
+  desc: Number of shards for restore processing
+  long_desc: Number of RADOS objects to use for storing restore entries which are in progress. This affects concurrency of restore maintenance, as shards can be processed in parallel.
+  default: 32
+  services:
+  - rgw
+  with_legacy: true
+- name: rgw_restore_lock_max_time
+  type: int
+  level: dev
+  default: 90
+  services:
+  - rgw
+  see_also:
+  with_legacy: true
+- name: rgw_restore_processor_period
+  type: int
+  level: advanced
+  desc: Restore cycle run time
+  long_desc: The amount of time between the start of consecutive runs of the restore
+    processing threads. If the thread runs takes more than this period, it will
+    not wait before running again.
+  fmt_desc: The cycle time for restore state processing.
+  default: 15_min
+  services:
+  - rgw
+  with_legacy: true
 - name: rgw_mp_lock_max_time
   type: int
   level: advanced
@@ -1270,6 +1319,14 @@ options:
   services:
   - rgw
   with_legacy: true
+- name: rgw_nfs_run_restore_threads
+  type: bool
+  level: advanced
+  desc: run objects' restore threads in librgw (default off)
+  default: false
+  services:
+  - rgw
+  with_legacy: true
 - name: rgw_nfs_run_sync_thread
   type: bool
   level: advanced

src/common/subsys.h

@@ -66,6 +66,7 @@ SUBSYS(rgw_access, 1, 5)
 SUBSYS(rgw_dbstore, 1, 5)
 SUBSYS(rgw_flight, 1, 5)
 SUBSYS(rgw_lifecycle, 1, 5)
+SUBSYS(rgw_restore, 1, 5)
 SUBSYS(rgw_notification, 1, 5)
 SUBSYS(javaclient, 1, 5)
 SUBSYS(asok, 1, 5)

src/rgw/CMakeLists.txt

@@ -66,6 +66,7 @@ set(librgw_common_srcs
   rgw_ldap.cc
   rgw_lc.cc
   rgw_lc_s3.cc
+  rgw_restore.cc
   rgw_metadata.cc
   rgw_multi.cc
   rgw_multi_del.cc

src/rgw/driver/daos/rgw_sal_daos.cc

@@ -1028,15 +1028,13 @@ int DaosObject::transition_to_cloud(
 
 int DaosObject::restore_obj_from_cloud(Bucket* bucket,
           rgw::sal::PlacementTier* tier,
-          rgw_placement_rule& placement_rule,
-          rgw_bucket_dir_entry& o,
 	  CephContext* cct,
           RGWObjTier& tier_config,
           uint64_t olh_epoch,
           std::optional<uint64_t> days,
+	  bool& in_progress,
           const DoutPrefixProvider* dpp, 
-          optional_yield y,
-          uint32_t flags)
+          optional_yield y)
 {
   return DAOS_NOT_IMPLEMENTED_LOG(dpp);
 }
@@ -2321,6 +2319,12 @@ std::unique_ptr<Lifecycle> DaosStore::get_lifecycle(void) {
   return 0;
 }
 
+std::unique_ptr<Restore> DaosStore::get_restore(const int n_objs,
+				const std::vector<std::string_view>& obj_names) {
+  DAOS_NOT_IMPLEMENTED_LOG(nullptr);
+  return 0;
+}
+
 bool DaosStore::process_expired_objects(const DoutPrefixProvider *dpp,
 	       				optional_yield y) {
   DAOS_NOT_IMPLEMENTED_LOG(nullptr);

src/rgw/driver/daos/rgw_sal_daos.h

@@ -654,15 +654,13 @@ class DaosObject : public StoreObject {
                                   optional_yield y) override;
   virtual int restore_obj_from_cloud(Bucket* bucket,
 			   rgw::sal::PlacementTier* tier,
-			   rgw_placement_rule& placement_rule,
-			   rgw_bucket_dir_entry& o,
 			   CephContext* cct,
 			   RGWObjTier& tier_config,
 			   uint64_t olh_epoch,
 			   std::optional<uint64_t> days,
+		           bool& in_progress,
 			   const DoutPrefixProvider* dpp,
-			   optional_yield y,
-			   uint32_t flags) override;
+			   optional_yield y) override;
   virtual bool placement_rules_match(rgw_placement_rule& r1,
                                      rgw_placement_rule& r2) override;
   virtual int dump_obj_layout(const DoutPrefixProvider* dpp, optional_yield y,
@@ -937,6 +935,8 @@ class DaosStore : public StoreDriver {
   virtual std::string zone_unique_trans_id(const uint64_t unique_num) override;
   virtual int cluster_stat(RGWClusterStat& stats) override;
   virtual std::unique_ptr<Lifecycle> get_lifecycle(void) override;
+  virtual std::unique_ptr<Restore> get_restore(const int n_objs,
+		 const std::vector<std::string_view>& obj_names) override;
   virtual bool process_expired_objects(const DoutPrefixProvider *dpp, optional_yield y) override;
   virtual std::unique_ptr<Notification> get_notification(
       rgw::sal::Object* obj, rgw::sal::Object* src_obj, struct req_state* s,
@@ -953,6 +953,7 @@ class DaosStore : public StoreDriver {
       std::string& _req_id,
       optional_yield y) override;
   virtual RGWLC* get_rgwlc(void) override { return NULL; }
+  virtual RGWRestore* get_rgwrestore(void) override { return NULL; }
   virtual RGWCoroutinesManagerRegistry* get_cr_registry() override {
     return NULL;
   }

src/rgw/driver/motr/rgw_sal_motr.cc

@@ -3335,6 +3335,11 @@ std::unique_ptr<Lifecycle> MotrStore::get_lifecycle(void)
   return 0;
 }
 
+std::unique_ptr<Restore> MotrStore::get_restore(const int n_objs,
+			const std::vector<std::string_view>& obj_names) {
+  return 0;
+}
+
 bool MotrStore::process_expired_objects(const DoutPrefixProvider *dpp,
 	       				optional_yield y)
 {

src/rgw/driver/motr/rgw_sal_motr.h

@@ -1006,6 +1006,8 @@ class MotrStore : public StoreDriver {
     virtual int list_all_zones(const DoutPrefixProvider* dpp, std::list<std::string>& zone_ids) override;
     virtual int cluster_stat(RGWClusterStat& stats) override;
     virtual std::unique_ptr<Lifecycle> get_lifecycle(void) override;
+    virtual std::unique_ptr<Restore> get_restore(const int n_objs,
+		   const std::vector<std::string_view>& obj_names) override;
     virtual bool process_expired_objects(const DoutPrefixProvider *dpp, optional_yield y) override;
     virtual std::unique_ptr<Notification> get_notification(rgw::sal::Object* obj, rgw::sal::Object* src_obj,
         req_state* s, rgw::notify::EventType event_type, optional_yield y, const std::string* object_name=nullptr) override;
@@ -1020,6 +1022,7 @@ class MotrStore : public StoreDriver {
         std::string& _req_id,
         optional_yield y) override;
     virtual RGWLC* get_rgwlc(void) override { return NULL; }
+    virtual RGWRestore* get_rgwrestore(void) override { return NULL; }
     virtual RGWCoroutinesManagerRegistry* get_cr_registry() override { return NULL; }
 
     virtual int log_usage(const DoutPrefixProvider *dpp, std::map<rgw_user_bucket, RGWUsageBatch>& usage_info) override;

src/rgw/driver/posix/rgw_sal_posix.cc

@@ -3071,15 +3071,11 @@ int POSIXObject::transition_to_cloud(Bucket* bucket,
 
 int POSIXObject::restore_obj_from_cloud(Bucket* bucket,
           rgw::sal::PlacementTier* tier,
-          rgw_placement_rule& placement_rule,
-          rgw_bucket_dir_entry& o,
 	  CephContext* cct,
-          RGWObjTier& tier_config,
-          uint64_t olh_epoch,
           std::optional<uint64_t> days,
+          bool& in_progress,
           const DoutPrefixProvider* dpp, 
-          optional_yield y,
-          uint32_t flags)
+          optional_yield y)
 {
   return -ERR_NOT_IMPLEMENTED;
 }

src/rgw/driver/posix/rgw_sal_posix.h

@@ -697,15 +697,11 @@ class POSIXObject : public StoreObject {
 			 optional_yield y) override;
   virtual int restore_obj_from_cloud(Bucket* bucket,
 			   rgw::sal::PlacementTier* tier,
-			   rgw_placement_rule& placement_rule,
-			   rgw_bucket_dir_entry& o,
 			   CephContext* cct,
-		           RGWObjTier& tier_config,
-			   uint64_t olh_epoch,
 			   std::optional<uint64_t> days,
+		           bool& in_progress,
 			   const DoutPrefixProvider* dpp,
-			   optional_yield y,
-		           uint32_t flags) override;
+			   optional_yield y) override;
   virtual bool placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) override;
   virtual int dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f) override;
   virtual int swift_versioning_restore(const ACLOwner& owner, const rgw_user& remote_user, bool& restored,

src/rgw/driver/rados/rgw_lc_tier.cc

@@ -260,6 +260,7 @@ int rgw_cloud_tier_restore_object(RGWLCCloudTierCtx& tier_ctx,
                          uint64_t& accounted_size, rgw::sal::Attrs& attrs,
                          std::optional<uint64_t> days,
                          RGWZoneGroupTierS3Glacier& glacier_params,
+			 bool& in_progress,
                          void* cb) {
   RGWRESTConn::get_obj_params req_params;
   std::string target_obj_name;
@@ -276,25 +277,23 @@ int rgw_cloud_tier_restore_object(RGWLCCloudTierCtx& tier_ctx,
     target_obj_name += get_key_instance(tier_ctx.obj->get_key());
   }
 
-  if (glacier_params.glacier_restore_tier_type != GlacierRestoreTierType::Expedited) {
-    //XXX: Supporting STANDARD tier type is still in WIP
-    ldpp_dout(tier_ctx.dpp, -1) << __func__ << "ERROR: Only Expedited tier_type is supported " << dendl;
-    return -1;
-  }
+  if (!in_progress) { // first time. Send RESTORE req.
 
-  rgw_obj dest_obj(dest_bucket, rgw_obj_key(target_obj_name));
+    rgw_obj dest_obj(dest_bucket, rgw_obj_key(target_obj_name));
+    ret = cloud_tier_restore(tier_ctx.dpp, tier_ctx.conn, dest_obj, days, glacier_params);
 
-  ret = cloud_tier_restore(tier_ctx.dpp, tier_ctx.conn, dest_obj, days, glacier_params);
-  
-  ldpp_dout(tier_ctx.dpp, 20) << __func__ << "Restoring object=" << dest_obj << "returned ret = " << ret << dendl;
- 
-  if (ret < 0 ) { 
-    ldpp_dout(tier_ctx.dpp, -1) << __func__ << "ERROR: failed to restore object=" << dest_obj << "; ret = " << ret << dendl;
-    return ret;
+    ldpp_dout(tier_ctx.dpp, 20) << __func__ << "Restoring object=" << target_obj_name << "returned ret = " << ret << dendl;
+
+    if (ret < 0 ) {
+      ldpp_dout(tier_ctx.dpp, -1) << __func__ << "ERROR: failed to restore object=" << dest_obj << "; ret = " << ret << dendl;
+      return ret;
+    }
+    in_progress = true;
   }
 
   // now send HEAD request and verify if restore is complete on glacier/tape endpoint
-  bool restore_in_progress = false;
+  static constexpr int MAX_RETRIES = 10;
+  uint32_t retries = 0;
   do {
     ret = rgw_cloud_tier_get_object(tier_ctx, true, headers, nullptr, etag,
                                     accounted_size, attrs, nullptr);
@@ -304,8 +303,14 @@ int rgw_cloud_tier_restore_object(RGWLCCloudTierCtx& tier_ctx,
       return ret;
     }
 
-    restore_in_progress = is_restore_in_progress(tier_ctx.dpp, headers);
-  } while(restore_in_progress);
+    in_progress = is_restore_in_progress(tier_ctx.dpp, headers);
+
+  } while(retries++ < MAX_RETRIES && in_progress);
+
+  if (in_progress) {
+    ldpp_dout(tier_ctx.dpp, 20) << __func__ << "Restoring object=" << target_obj_name << " still in progress; returning " << dendl;
+    return 0;
+  } 
 
   // now do the actual GET
   ret = rgw_cloud_tier_get_object(tier_ctx, false, headers, pset_mtime, etag,