rgw/cloudrestore: Add Restore support from Glacier/Tape cloud endpoints

Unlike regular S3 cloud services, restoring objects from S3/Tape or AWS Glacier services
would require special handling. We need to first restore the object using Glacier
RestoreObject API and then download it using GET.
https://docs.aws.amazon.com/cli/latest/reference/s3api/restore-object.html

This PR adds that support for "Expedited" tier retrieval type. That means the
restore would be quick and the object can be downloaded soon.

TODO: "Standard" tier-type support. Need to handle the case where in restore from
cloud endpoint could take a longer time and need to be monitored periodically
in the background.

Signed-off-by: Soumya Koduri <[email protected]>

Author: soumyakoduri

src/rgw/driver/rados/.rgw_lc_tier.cc.swm

@@ -252,6 +252,70 @@ static const struct generic_attr generic_attrs[] = {
   { "ETAG",                     RGW_ATTR_ETAG },
 };
 
+/* Restore object from remote endpoint.
+ */
+int rgw_cloud_tier_restore_object(RGWLCCloudTierCtx& tier_ctx,
+                         std::map<std::string, std::string>& headers,
+                         real_time* pset_mtime, std::string& etag,
+                         uint64_t& accounted_size, rgw::sal::Attrs& attrs,
+                         std::optional<uint64_t> days,
+                         RGWZoneGroupTierS3Glacier& glacier_params,
+                         void* cb) {
+  RGWRESTConn::get_obj_params req_params;
+  std::string target_obj_name;
+  int ret = 0;
+  rgw_lc_obj_properties obj_properties(tier_ctx.o.meta.mtime, tier_ctx.o.meta.etag,
+        tier_ctx.o.versioned_epoch, tier_ctx.acl_mappings,
+        tier_ctx.target_storage_class);
+
+  rgw_bucket dest_bucket;
+  dest_bucket.name = tier_ctx.target_bucket_name;
+  target_obj_name = tier_ctx.bucket_info.bucket.name + "/" +
+                    tier_ctx.obj->get_name();
+  if (!tier_ctx.o.is_current()) {
+    target_obj_name += get_key_instance(tier_ctx.obj->get_key());
+  }
+
+  if (glacier_params.glacier_restore_tier_type != GlacierRestoreTierType::Expedited) {
+    //XXX: Supporting STANDARD tier type is still in WIP
+    ldpp_dout(tier_ctx.dpp, -1) << __func__ << "ERROR: Only Expedited tier_type is supported " << dendl;
+    return -1;
+  }
+
+  rgw_obj dest_obj(dest_bucket, rgw_obj_key(target_obj_name));
+
+  ret = cloud_tier_restore(tier_ctx.dpp, tier_ctx.conn, dest_obj, days, glacier_params);
+  
+  ldpp_dout(tier_ctx.dpp, 20) << __func__ << "Restoring object=" << dest_obj << "returned ret = " << ret << dendl;
+ 
+  if (ret < 0 ) { 
+    ldpp_dout(tier_ctx.dpp, -1) << __func__ << "ERROR: failed to restore object=" << dest_obj << "; ret = " << ret << dendl;
+    return ret;
+  }
+
+  // now send HEAD request and verify if restore is complete on glacier/tape endpoint
+  bool restore_in_progress = false;
+  do {
+    ret = rgw_cloud_tier_get_object(tier_ctx, true, headers, nullptr, etag,
+                                    accounted_size, attrs, nullptr);
+
+    if (ret < 0) {
+      ldpp_dout(tier_ctx.dpp, 0) << __func__ << "ERROR: failed to fetch HEAD from cloud for obj=" << tier_ctx.obj << " , ret = " << ret << dendl;
+      return ret;
+    }
+
+    restore_in_progress = is_restore_in_progress(tier_ctx.dpp, headers);
+  } while(restore_in_progress);
+
+  // now do the actual GET
+  ret = rgw_cloud_tier_get_object(tier_ctx, false, headers, pset_mtime, etag,
+                                  accounted_size, attrs, cb);
+
+  ldpp_dout(tier_ctx.dpp, 20) << __func__ << "(): fetching object from cloud bucket:" << dest_bucket << ", object: " << target_obj_name << " returned ret:" << ret << dendl;
+
+  return ret;
+}
+
 /* Read object or just head from remote endpoint.
  */
 int rgw_cloud_tier_get_object(RGWLCCloudTierCtx& tier_ctx, bool head,
@@ -367,6 +431,30 @@ static bool is_already_tiered(const DoutPrefixProvider *dpp,
   return 0;
 }
 
+bool is_restore_in_progress(const DoutPrefixProvider *dpp,
+                            std::map<std::string, std::string>& headers) {
+  map<string, string> attrs = headers;
+
+  for (const auto& a : attrs) {
+    ldpp_dout(dpp, 20) << "GetCrf attr[" << a.first << "] = " << a.second <<dendl;
+  }
+  string s = attrs["X_AMZ_RESTORE"];
+
+  if (s.empty())
+    s = attrs["x_amz_restore"];
+
+  ldpp_dout(dpp, 0) << "is_already_tiered attrs[X_AMZ_RESTORE] = " << s <<dendl;
+
+  if (!s.empty()){
+    const char *r_str = "ongoing-request=\"true\"";
+    const char *found = std::strstr(s.c_str(), r_str);
+    if (found) {
+    	return true;
+    }
+  }
+  return false;;
+}
+
 /* Read object locally & also initialize dest rest obj based on read attrs */
 class RGWLCStreamRead
 {
@@ -914,6 +1002,95 @@ static int cloud_tier_send_multipart_part(RGWLCCloudTierCtx& tier_ctx,
   return 0;
 }
 
+int cloud_tier_restore(const DoutPrefixProvider *dpp, RGWRESTConn& dest_conn,
+                       const rgw_obj& dest_obj, std::optional<uint64_t> days,
+                       RGWZoneGroupTierS3Glacier& glacier_params) {
+  rgw_http_param_pair params[] = {{"restore", nullptr}, {nullptr, nullptr}};
+  // XXX: include versionId=VersionId in the params above
+
+  stringstream ss;
+  XMLFormatter formatter;
+  int ret;
+
+  bufferlist bl, out_bl;
+  string resource = obj_to_aws_path(dest_obj);
+
+  const std::string tier_v = (glacier_params.glacier_restore_tier_type == GlacierRestoreTierType::Expedited) ? "Expedited" : "Standard";
+
+  struct RestoreRequest {
+	  std::optional<uint64_t> days;
+	  std::optional<std::string> tier;
+
+    explicit RestoreRequest(std::optional<uint64_t> _days, std::optional<std::string> _tier) : days(_days), tier(_tier) {}
+
+    void dump_xml(Formatter *f) const {
+      encode_xml("Days", days, f);
+      if (tier) {
+      	f->open_object_section("GlacierJobParameters");
+	      encode_xml("Tier", tier, f);
+      	f->close_section();
+      };
+    }
+  } req_enc(days, tier_v);
+
+  struct RestoreResult {
+    std::string code;
+
+    void decode_xml(XMLObj *obj) {
+      RGWXMLDecoder::decode_xml("Code", code, obj);
+    }
+  } result;
+
+  req_enc.days = glacier_params.glacier_restore_days;
+  req_enc.tier = tier_v;
+  encode_xml("RestoreRequest", req_enc, &formatter);
+
+  formatter.flush(ss);
+  bl.append(ss.str());
+
+  ret = dest_conn.send_resource(dpp, "POST", resource, params, nullptr,
+                                out_bl, &bl, nullptr, null_yield);
+
+  if (ret < 0) {
+    ldpp_dout(dpp, 0) << __func__ << "ERROR: failed to send Restore request to cloud for obj=" << dest_obj << " , ret = " << ret << dendl;
+  } else {
+    ldpp_dout(dpp, 0) << __func__ << "Sent Restore request to cloud for obj=" << dest_obj << " , ret = " << ret << dendl;
+  }
+
+  if (out_bl.length() > 0) {
+    RGWXMLDecoder::XMLParser parser;
+    if (!parser.init()) {
+      ldpp_dout(dpp, 0) << "ERROR: failed to initialize xml parser for parsing restore request response from server" << dendl;
+      return -EIO;
+    }
+
+    if (!parser.parse(out_bl.c_str(), out_bl.length(), 1)) {
+      string str(out_bl.c_str(), out_bl.length());
+      ldpp_dout(dpp, 5) << "ERROR: failed to parse xml restore: " << str << dendl;
+      return -EIO;
+    }
+
+    try {
+      RGWXMLDecoder::decode_xml("Error", result, &parser, true);
+    } catch (RGWXMLDecoder::err& err) {
+      string str(out_bl.c_str(), out_bl.length());
+      ldpp_dout(dpp, 5) << "ERROR: unexpected xml: " << str << dendl;
+      return -EIO;
+    }
+
+    ldpp_dout(dpp, 0) << "ERROR: Restore request received result : " << result.code << dendl;
+    if (result.code != "RestoreAlreadyInProgress") {
+      return -EIO;
+    } else { // treat as success
+      return 0;
+    }
+
+    ldpp_dout(dpp, 0) << "ERROR: restore req failed with error: " << result.code << dendl;
+  }
+
+  return ret;
+}
+
 static int cloud_tier_abort_multipart(const DoutPrefixProvider *dpp,
       RGWRESTConn& dest_conn, const rgw_obj& dest_obj,
       const std::string& upload_id) {

src/rgw/driver/rados/rgw_lc_tier.cc

@@ -57,3 +57,18 @@ int rgw_cloud_tier_get_object(RGWLCCloudTierCtx& tier_ctx, bool head,
                          real_time* pset_mtime, std::string& etag,
                          uint64_t& accounted_size, rgw::sal::Attrs& attrs,
                          void* cb);
+int rgw_cloud_tier_restore_object(RGWLCCloudTierCtx& tier_ctx,
+                         std::map<std::string, std::string>& headers,
+                         real_time* pset_mtime, std::string& etag,
+                         uint64_t& accounted_size, rgw::sal::Attrs& attrs,
+                  			 std::optional<uint64_t> days,
+                         RGWZoneGroupTierS3Glacier& glacier_params,
+                         void* cb);
+
+int cloud_tier_restore(const DoutPrefixProvider *dpp,
+                       RGWRESTConn& dest_conn, const rgw_obj& dest_obj,
+                       std::optional<uint64_t> days,
+                       RGWZoneGroupTierS3Glacier& glacier_params);
+
+bool is_restore_in_progress(const DoutPrefixProvider *dpp,
+                            std::map<std::string, std::string>& headers);

src/rgw/driver/rados/rgw_lc_tier.h

@@ -5290,14 +5290,8 @@ int RGWRados::restore_obj_from_cloud(RGWLCCloudTierCtx& tier_ctx,
   dest_placement.storage_class = tier_ctx.restore_storage_class;
   RGWRadosPutObj cb(dpp, cct, plugin, compressor, &processor, progress_cb, progress_data,
                     [&](map<string, bufferlist> obj_attrs) {
-                      // XXX: do we need filter() lke in fetch_remote_obj() cb
+                      // XXX: do we need filter() like in fetch_remote_obj() cb
                       dest_placement.inherit_from(dest_bucket_info.placement_rule);
-                      /* For now we always restore to STANDARD storage-class.
-                       * Later we will add support to take restore-target-storage-class
-                       * for permanent restore
-                       */
-  //                    dest_placement.storage_class = RGW_STORAGE_CLASS_STANDARD;
-
                       processor.set_tail_placement(dest_placement);
 
                       ret = processor.prepare(rctx.y);
@@ -5326,9 +5320,18 @@ int RGWRados::restore_obj_from_cloud(RGWLCCloudTierCtx& tier_ctx,
   real_time set_mtime;
   std::map<std::string, std::string> headers;
   ldpp_dout(dpp, 20) << "Fetching from cloud, object:" << dest_obj << dendl;
-  ret = rgw_cloud_tier_get_object(tier_ctx, false,  headers,
+  if (tier_config.tier_placement.tier_type == "cloud-s3-glacier") {
+    ldpp_dout(dpp, 20) << "Restoring  object:" << dest_obj << " from the cloud" << dendl;
+    RGWZoneGroupTierS3Glacier& glacier_params = tier_config.tier_placement.s3_glacier;
+    ret = rgw_cloud_tier_restore_object(tier_ctx, headers,
+                                &set_mtime, etag, accounted_size,
+                                attrs, days, glacier_params, &cb);
+  } else {
+    ldpp_dout(dpp, 20) << "Fetching  object:" << dest_obj << "from the cloud" <<  dendl;
+    ret = rgw_cloud_tier_get_object(tier_ctx, false,  headers,
                                 &set_mtime, etag, accounted_size,
                                 attrs, &cb);
+  }
 
   if (ret < 0) { 
     ldpp_dout(dpp, 20) << "Fetching from cloud failed, object:" << dest_obj << dendl;

src/rgw/driver/rados/rgw_rados.cc

@@ -139,6 +139,7 @@ rgw_http_errors rgw_http_s3_errors({
     { ERR_NO_SUCH_BUCKET_ENCRYPTION_CONFIGURATION, {404, "ServerSideEncryptionConfigurationNotFoundError"}},
     { ERR_NO_SUCH_PUBLIC_ACCESS_BLOCK_CONFIGURATION, {404, "NoSuchPublicAccessBlockConfiguration"}},
     { ERR_ACCOUNT_EXISTS, {409, "AccountAlreadyExists"}},
+    { ERR_RESTORE_ALREADY_IN_PROGRESS, {409, "RestoreAlreadyInProgress"}},
     { ECANCELED, {409, "ConcurrentModification"}},
 });
 

src/rgw/rgw_common.cc

@@ -355,6 +355,8 @@ inline constexpr const char* RGW_REST_STS_XMLNS =
 #define ERR_NO_SUCH_TAG_SET 2402
 #define ERR_ACCOUNT_EXISTS 2403
 
+#define ERR_RESTORE_ALREADY_IN_PROGRESS 2500
+    
 #ifndef UINT32_MAX
 #define UINT32_MAX (0xffffffffu)
 #endif

src/rgw/rgw_common.h

@@ -3614,8 +3614,9 @@ void RGWRestoreObj_ObjStore_S3::send_response()
   if (restore_ret == 0) {
     s->err.http_ret = 202; // OK
   } else if (restore_ret == 1) {
-    s->err.http_ret = 409; // Conflict
-    dump_header(s, "x-amz-restore", "on-going-request=\"true\"");
+    restore_ret = -ERR_RESTORE_ALREADY_IN_PROGRESS;
+    set_req_state_err(s, restore_ret);
+    dump_header(s, "x-amz-restore", "ongoing-request=\"true\"");
   } else if (restore_ret == 2) {
     rgw::sal::Attrs attrs;
     ceph::real_time expiration_date;

src/rgw/rgw_rest_s3.cc

@@ -1719,7 +1719,7 @@ class PlacementTier {
 
   /** Get the type of this tier */
   virtual const std::string& get_tier_type() = 0;
-  /** Is the type of this tier cloud-s3/clous-s3-glacier */
+  /** Is the type of this tier cloud-s3/cloud-s3-glacier */
   virtual bool is_tier_type_s3() = 0;
   /** Get the storage class of this tier */
   virtual const std::string& get_storage_class() = 0;