rgw/cloud-restore: Add new tier-type & options related to S3 Glacier

soumyakoduri · soumyakoduri · commit b1ddd68f7c7c · 2025-03-15T21:43:12.000+05:30
Unlike regular S3 cloud services, restoring objects from S3/Tape or AWS Glacier services would require special handling. We need to first restore the object using Glacier `RestoreObject` API and then download it using `GET`. https://docs.aws.amazon.com/cli/latest/reference/s3api/restore-object.html A new cloud tier-type `s3-glacier` is added to handle S3 Glacier endpoints along with below tier-config options - `glacier_restore_days` - lifetime of the restored copy on the Glacier endpoint ; default: 1 day `glacier_restore_tier_type` - Retrieval tier at which the restore will be processed. Only "Standard" (default) and "Expedited" options are supported. In addition, a new option `restore_storage_class` is added to configure the storage class the objects need to be restored to. Default value: STANDARD Design doc: https://docs.google.com/document/d/1rzLJAzHK6cLuzJswgoplgugNOFCKi8NrjPOGfgVrIFg/edit?tab=t.0#heading=h.sgrmb31roboc Signed-off-by: Soumya Koduri <skoduri@redhat.com>
diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h
@@ -377,8 +377,11 @@ class DaosPlacementTier : public StorePlacementTier {
   virtual ~DaosPlacementTier() = default;
 
   virtual const std::string& get_tier_type() { return tier.tier_type; }
+  virtual bool is_tier_type_s3() { return (tier.is_tier_type_s3()); }
   virtual const std::string& get_storage_class() { return tier.storage_class; }
   virtual bool retain_head_object() { return tier.retain_head_object; }
+  virtual bool allow_read_through() { return tier.allow_read_through; }
+  virtual uint64_t get_read_through_restore_days() { return tier.read_through_restore_days; }
   RGWZoneGroupPlacementTier& get_rt() { return tier; }
 };
 
diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h
@@ -419,8 +419,11 @@ class MotrPlacementTier: public StorePlacementTier {
   virtual ~MotrPlacementTier() = default;
 
   virtual const std::string& get_tier_type() { return tier.tier_type; }
+  virtual bool is_tier_type_s3() { return (tier.is_tier_type_s3()); }
   virtual const std::string& get_storage_class() { return tier.storage_class; }
   virtual bool retain_head_object() { return tier.retain_head_object; }
+  virtual bool allow_read_through() { return tier.allow_read_through; }
+  virtual uint64_t get_read_through_restore_days() { return tier.read_through_restore_days; }
   RGWZoneGroupPlacementTier& get_rt() { return tier; }
 };
 
diff --git a/src/rgw/driver/rados/.rgw_lc_tier.cc.swm b/src/rgw/driver/rados/.rgw_lc_tier.cc.swm
diff --git a/src/rgw/driver/rados/rgw_lc_tier.h b/src/rgw/driver/rados/rgw_lc_tier.h
@@ -22,6 +22,8 @@ struct RGWLCCloudTierCtx {
   rgw::sal::Driver *driver;
   RGWBucketInfo& bucket_info;
   std::string storage_class;
+  std::string restore_storage_class;
+  std::string tier_type;
 
   rgw::sal::Object *obj;
 
diff --git a/src/rgw/driver/rados/rgw_obj_manifest.cc b/src/rgw/driver/rados/rgw_obj_manifest.cc
@@ -347,7 +347,7 @@ void RGWObjManifest::dump(Formatter *f) const
   ::encode_json("tail_placement", tail_placement, f);
   ::encode_json("tier_type", tier_type, f);
 
-  if (tier_type == "cloud-s3") {
+  if (tier_type == "cloud-s3" || tier_type == "cloud-s3-glacier") {
     ::encode_json("tier_config", tier_config, f);
   }
 
diff --git a/src/rgw/driver/rados/rgw_obj_manifest.h b/src/rgw/driver/rados/rgw_obj_manifest.h
@@ -471,16 +471,24 @@ class RGWObjManifest {
       return tier_type;
   }
 
+  bool is_tier_type_s3() {
+      return (tier_type == "cloud-s3" || tier_type == "cloud-s3-glacier");
+  }
+
+  bool is_tier_type_s3_glacier() {
+      return (tier_type == "cloud-s3-glacier");
+  }
+
   inline void set_tier_type(std::string value) {
-      /* Only "cloud-s3" tier-type is supported for now */
-      if (value == "cloud-s3") {
+      /* Only "cloud-s3" & "cloud-s3-glacier" tier-type is supported for now */
+      if (value == "cloud-s3" || value == "cloud-s3-glacier") {
         tier_type = value;
       }
   }
 
   inline void set_tier_config(RGWObjTier t) {
-      /* Set only if tier_type set to "cloud-s3" */
-      if (tier_type != "cloud-s3")
+      /* Set only if tier_type set to "cloud-s3" or "cloud-s3-glacier" */
+      if (!is_tier_type_s3())
         return;
 
       tier_config.name = t.name;
@@ -489,7 +497,7 @@ class RGWObjManifest {
   }
 
   inline const void get_tier_config(RGWObjTier* t) {
-      if (tier_type != "cloud-s3")
+      if (!is_tier_type_s3())
         return;
 
       t->name = tier_config.name;
diff --git a/src/rgw/driver/rados/rgw_putobj_processor.cc b/src/rgw/driver/rados/rgw_putobj_processor.cc
@@ -45,9 +45,9 @@ int read_cloudtier_info_from_attrs(rgw::sal::Attrs& attrs, RGWObjCategory& categ
     auto i = attr_iter->second;
     string m = i.to_str();
 
-    if (m == "cloud-s3") {
+    if (m == "cloud-s3" || m == "cloud-s3-glacier") {
       category = RGWObjCategory::CloudTiered;
-      manifest.set_tier_type("cloud-s3");
+      manifest.set_tier_type(m);
 
       auto config_iter = attrs.find(RGW_ATTR_CLOUD_TIER_CONFIG);
       if (config_iter != attrs.end()) {
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
@@ -5287,15 +5287,16 @@ int RGWRados::restore_obj_from_cloud(RGWLCCloudTierCtx& tier_ctx,
   }
   boost::optional<RGWPutObj_Compress> compressor;
   CompressorRef plugin;
+  dest_placement.storage_class = tier_ctx.restore_storage_class;
   RGWRadosPutObj cb(dpp, cct, plugin, compressor, &processor, progress_cb, progress_data,
                     [&](map<string, bufferlist> obj_attrs) {
-                      // XXX: do we need filter() like in fetch_remote_obj() cb
+                      // XXX: do we need filter() lke in fetch_remote_obj() cb
                       dest_placement.inherit_from(dest_bucket_info.placement_rule);
                       /* For now we always restore to STANDARD storage-class.
                        * Later we will add support to take restore-target-storage-class
                        * for permanent restore
                        */
-                      dest_placement.storage_class = RGW_STORAGE_CLASS_STANDARD;
+  //                    dest_placement.storage_class = RGW_STORAGE_CLASS_STANDARD;
 
                       processor.set_tail_placement(dest_placement);
 
@@ -5420,7 +5421,7 @@ int RGWRados::restore_obj_from_cloud(RGWLCCloudTierCtx& tier_ctx,
     // set tier-config only for temp restored objects, as
     // permanent copies will be treated as regular objects
     {
-      t.append("cloud-s3");
+      t.append(tier_ctx.tier_type);
       encode(tier_config, t_tier);
       attrs[RGW_ATTR_CLOUD_TIER_TYPE] = t;
       attrs[RGW_ATTR_CLOUD_TIER_CONFIG] = t_tier;
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -2916,6 +2916,8 @@ int RadosObject::restore_obj_from_cloud(Bucket* bucket,
   const rgw::sal::ZoneGroup& zonegroup = store->get_zone()->get_zonegroup();
   int ret = 0;
   string src_storage_class = o.meta.storage_class; // or take src_placement also as input
+  // update tier_config in case tier params are updated
+  tier_config.tier_placement = rtier->get_rt();
 
   if (bucket_name.empty()) {
     bucket_name = "rgwx-" + zonegroup.get_name() + "-" + tier->get_storage_class() +
@@ -2933,6 +2935,8 @@ int RadosObject::restore_obj_from_cloud(Bucket* bucket,
   tier_ctx.multipart_min_part_size = rtier->get_rt().t.s3.multipart_min_part_size;
   tier_ctx.multipart_sync_threshold = rtier->get_rt().t.s3.multipart_sync_threshold;
   tier_ctx.storage_class = tier->get_storage_class();
+  tier_ctx.restore_storage_class = rtier->get_rt().restore_storage_class;
+  tier_ctx.tier_type = rtier->get_rt().tier_type;
 
   ldpp_dout(dpp, 20) << "Restoring object(" << o.key << ") from the cloud endpoint(" << endpoint << ")" << dendl;
 
@@ -3242,7 +3246,7 @@ int RadosObject::write_cloud_tier(const DoutPrefixProvider* dpp,
   tier_config.tier_placement = rtier->get_rt();
   tier_config.is_multipart_upload = is_multipart_upload;
 
-  pmanifest->set_tier_type("cloud-s3");
+  pmanifest->set_tier_type(rtier->get_rt().tier_type);
   pmanifest->set_tier_config(tier_config);
 
   /* check if its necessary */
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -43,8 +43,11 @@ class RadosPlacementTier: public StorePlacementTier {
   virtual ~RadosPlacementTier() = default;
 
   virtual const std::string& get_tier_type() { return tier.tier_type; }
+  virtual bool is_tier_type_s3() { return (tier.is_tier_type_s3()); }
   virtual const std::string& get_storage_class() { return tier.storage_class; }
   virtual bool retain_head_object() { return tier.retain_head_object; }
+  virtual bool allow_read_through() { return tier.allow_read_through; }
+  virtual uint64_t get_read_through_restore_days() { return tier.read_through_restore_days; }
   RGWZoneGroupPlacementTier& get_rt() { return tier; }
 };
 
diff --git a/src/rgw/driver/rados/rgw_zone.cc b/src/rgw/driver/rados/rgw_zone.cc
@@ -1370,10 +1370,17 @@ int RGWZoneGroupPlacementTier::update_params(const JSONFormattable& config)
     }
   }
 
-  if (tier_type == "cloud-s3") {
+  if (is_tier_type_s3()) {
     r = t.s3.update_params(config);
   }
 
+  if (config.exists("restore_storage_class")) {
+    restore_storage_class = config["restore_storage_class"];
+  }
+
+  if (is_tier_type_s3_glacier()) {
+    r = s3_glacier.update_params(config);
+  }
   return r;
 }
 
@@ -1389,10 +1396,18 @@ int RGWZoneGroupPlacementTier::clear_params(const JSONFormattable& config)
     read_through_restore_days = DEFAULT_READ_THROUGH_RESTORE_DAYS;
   }
 
-  if (tier_type == "cloud-s3") {
+  if (is_tier_type_s3()) {
     t.s3.clear_params(config);
   }
 
+  if (config.exists("restore_storage_class")) {
+    restore_storage_class = RGW_STORAGE_CLASS_STANDARD;
+  }
+
+  if (is_tier_type_s3_glacier()) {
+    s3_glacier.clear_params(config);
+  }
+
   return 0;
 }
 
@@ -1509,6 +1524,40 @@ int RGWZoneGroupPlacementTierS3::clear_params(const JSONFormattable& config)
   return 0;
 }
 
+int RGWZoneGroupTierS3Glacier::update_params(const JSONFormattable& config)
+{
+  int r = -1;
+
+  if (config.exists("glacier_restore_days")) {
+    r = conf_to_uint64(config, "glacier_restore_days", &glacier_restore_days);
+    if (r < 0) {
+      glacier_restore_days = DEFAULT_GLACIER_RESTORE_DAYS;
+    }
+  }
+  if (config.exists("glacier_restore_tier_type")) {
+    string s;
+    s = config["glacier_restore_tier_type"];
+    if (s != "Expedited") {
+      glacier_restore_tier_type = Standard;
+    } else {
+      glacier_restore_tier_type = Expedited;
+    }
+  }
+  return 0;
+}
+
+int RGWZoneGroupTierS3Glacier::clear_params(const JSONFormattable& config)
+{
+  if (config.exists("glacier_restore_days")) {
+    glacier_restore_days = DEFAULT_GLACIER_RESTORE_DAYS;
+  }
+  if (config.exists("glacier_restore_tier_type")) {
+    /* default */
+    glacier_restore_tier_type = Standard;
+  }
+  return 0;
+}
+
 void rgw_meta_sync_info::generate_test_instances(list<rgw_meta_sync_info*>& o)
 {
   auto info = new rgw_meta_sync_info;
diff --git a/src/rgw/radosgw-admin/radosgw-admin.cc b/src/rgw/radosgw-admin/radosgw-admin.cc
@@ -5929,16 +5929,16 @@ int main(int argc, const char **argv)
       /* Tier options */
       bool tier_class = false;
       std::string storage_class = rule.get_storage_class();
-      RGWZoneGroupPlacementTier t{storage_class};
+      RGWZoneGroupPlacementTier t;
       RGWZoneGroupPlacementTier *pt = &t;
 
 	  auto ptiter = target.tier_targets.find(storage_class);
 	  if (ptiter != target.tier_targets.end()) {
         pt = &ptiter->second;
         tier_class = true;
       } else if (tier_type_specified) {
-        if (tier_type == "cloud-s3") {
-          /* we support only cloud-s3 tier-type for now.
+        if (tier_type == "cloud-s3" || tier_type == "cloud-s3-glacier") {
+          /* we support only cloud-s3 & cloud-s3-glacier tier-type for now.
            * Once set cant be reset. */
           tier_class = true;
           pt->tier_type = tier_type;
diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
@@ -1517,7 +1517,7 @@ class LCOpAction_Transition : public LCOpAction {
 
     r = zonegroup.get_placement_tier(target_placement, &oc.tier);
 
-    if (!r && oc.tier->get_tier_type() == "cloud-s3") {
+    if (!r && oc.tier->is_tier_type_s3()) {
       ldpp_dout(oc.dpp, 30) << "Found cloud s3 tier: " << target_placement.storage_class << dendl;
       if (!oc.o.is_current() &&
           !pass_object_lock_check(oc.driver, oc.obj.get(), oc.dpp)) {
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
@@ -969,7 +969,7 @@ int handle_cloudtier_obj(req_state* s, const DoutPrefixProvider *dpp, rgw::sal::
   RGWObjManifest m;
   try { 
     decode(m, attr_iter->second);
-    if (m.get_tier_type() != "cloud-s3") {
+    if (!m.is_tier_type_s3()) {
       ldpp_dout(dpp, 20) << "not a cloud tier object " <<  s->object->get_key().name << dendl;
       if (restore_op) {
         op_ret = -ERR_INVALID_OBJECT_STATE;
@@ -983,7 +983,7 @@ int handle_cloudtier_obj(req_state* s, const DoutPrefixProvider *dpp, rgw::sal::
     m.get_tier_config(&tier_config);
     if (sync_cloudtiered) {
       bufferlist t, t_tier;
-      t.append("cloud-s3");
+      t.append(m.get_tier_type());
       attrs[RGW_ATTR_CLOUD_TIER_TYPE] = t;
       encode(tier_config, t_tier);
       attrs[RGW_ATTR_CLOUD_TIER_CONFIG] = t_tier;
@@ -1015,11 +1015,9 @@ int handle_cloudtier_obj(req_state* s, const DoutPrefixProvider *dpp, rgw::sal::
         s->err.message = "failed to restore object";
         return op_ret;
       }
-      rgw::sal::RadosPlacementTier* rtier = static_cast<rgw::sal::RadosPlacementTier*>(tier.get());
-      tier_config.tier_placement = rtier->get_rt();
       if (!restore_op) {
-        if (tier_config.tier_placement.allow_read_through) {
-          days = tier_config.tier_placement.read_through_restore_days;
+        if (tier->allow_read_through()) {
+          days = tier->get_read_through_restore_days();
         } else { //read-through is not enabled
           op_ret = -ERR_INVALID_OBJECT_STATE;
           s->err.message = "Read through is not enabled for this config";
@@ -4413,7 +4411,7 @@ void RGWPutObj::execute(optional_yield y)
       RGWObjManifest m;
       try{
         decode(m, bl);
-        if (m.get_tier_type() == "cloud-s3") {
+        if (m.is_tier_type_s3()) {
           op_ret = -ERR_INVALID_OBJECT_STATE;
           s->err.message = "This object was transitioned to cloud-s3";
           ldpp_dout(this, 4) << "Cannot copy cloud tiered object. Failing with "
@@ -5869,7 +5867,7 @@ void RGWCopyObj::execute(optional_yield y)
       RGWObjManifest m;
       try{
         decode(m, bl);
-        if (m.get_tier_type() == "cloud-s3") {
+        if (m.is_tier_type_s3()) {
           op_ret = -ERR_INVALID_OBJECT_STATE;
           s->err.message = "This object was transitioned to cloud-s3";
           ldpp_dout(this, 4) << "Cannot copy cloud tiered object. Failing with "
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
@@ -1719,10 +1719,16 @@ class PlacementTier {
 
   /** Get the type of this tier */
   virtual const std::string& get_tier_type() = 0;
+  /** Is the type of this tier cloud-s3/clous-s3-glacier */
+  virtual bool is_tier_type_s3() = 0;
   /** Get the storage class of this tier */
   virtual const std::string& get_storage_class() = 0;
   /** Should we retain the head object when transitioning */
   virtual bool retain_head_object() = 0;
+  /** Is read_through allowed */
+  virtual bool allow_read_through() = 0;
+  /** Get read_through restore_days */
+  virtual uint64_t get_read_through_restore_days() = 0;
   /** Get the placement rule associated with this tier */
 };
 
diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h
@@ -207,7 +207,10 @@ class DBNotification : public StoreNotification {
     virtual ~DBPlacementTier() = default;
 
     virtual const std::string& get_tier_type() { return tier.tier_type; }
+    virtual bool is_tier_type_s3() { return (tier.is_tier_type_s3()); }
     virtual const std::string& get_storage_class() { return tier.storage_class; }
+    virtual bool allow_read_through() { return tier.allow_read_through; }
+    virtual uint64_t get_read_through_restore_days() { return tier.read_through_restore_days; }
     virtual bool retain_head_object() { return tier.retain_head_object; }
     RGWZoneGroupPlacementTier& get_rt() { return tier; }
   };
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
@@ -29,8 +29,11 @@ class FilterPlacementTier : public PlacementTier {
   virtual ~FilterPlacementTier() = default;
 
   virtual const std::string& get_tier_type() override { return next->get_tier_type(); }
+  virtual bool is_tier_type_s3() { return next->is_tier_type_s3(); }
   virtual const std::string& get_storage_class() override { return next->get_storage_class(); }
   virtual bool retain_head_object() override { return next->retain_head_object(); }
+  virtual bool allow_read_through() { return next->allow_read_through(); }
+  virtual uint64_t get_read_through_restore_days() { return next->get_read_through_restore_days(); }
 
   /* Internal to Filters */
   PlacementTier* get_next() { return next.get(); }
diff --git a/src/rgw/rgw_zone.cc b/src/rgw/rgw_zone.cc
diff --git a/src/rgw/rgw_zone_types.h b/src/rgw/rgw_zone_types.h

Original file line number	Diff line number	Diff line change
`@@ -347,7 +347,7 @@ void RGWObjManifest::dump(Formatter *f) const`
`347`	`347`	`::encode_json("tail_placement", tail_placement, f);`
`348`	`348`	`::encode_json("tier_type", tier_type, f);`
`349`	`349`
`350`		`- if (tier_type == "cloud-s3") {`
	`350`	`+ if (tier_type == "cloud-s3" \|\| tier_type == "cloud-s3-glacier") {`
`351`	`351`	`::encode_json("tier_config", tier_config, f);`
`352`	`352`	`}`
`353`	`353`