rgw/cloud-restore: Handle versioned objects

While restoring non-current object versions, ensure they remain non-current.

Read `olh_epoch` from the restored object's metadata into a new attr
"RGW_ATTR_RESTORE_VERSIONED_EPOCH". This attr/olh_epoch is used while
updating bi entry and also to reset HEAD object post expiry of temporary
copies.

Signed-off-by: Soumya Koduri <[email protected]>

Author: soumyakoduri

src/rgw/driver/daos/rgw_sal_daos.cc

@@ -1032,7 +1032,6 @@ int DaosObject::restore_obj_from_cloud(Bucket* bucket,
           rgw_bucket_dir_entry& o,
 	  CephContext* cct,
           RGWObjTier& tier_config,
-          real_time& mtime,
           uint64_t olh_epoch,
           std::optional<uint64_t> days,
           const DoutPrefixProvider* dpp, 

src/rgw/driver/daos/rgw_sal_daos.h

@@ -655,7 +655,6 @@ class DaosObject : public StoreObject {
 			   rgw_bucket_dir_entry& o,
 			   CephContext* cct,
 			   RGWObjTier& tier_config,
-			   real_time& mtime,
 			   uint64_t olh_epoch,
 			   std::optional<uint64_t> days,
 			   const DoutPrefixProvider* dpp,

src/rgw/driver/posix/rgw_sal_posix.cc

@@ -3059,7 +3059,6 @@ int POSIXObject::restore_obj_from_cloud(Bucket* bucket,
           rgw_bucket_dir_entry& o,
 	  CephContext* cct,
           RGWObjTier& tier_config,
-          real_time& mtime,
           uint64_t olh_epoch,
           std::optional<uint64_t> days,
           const DoutPrefixProvider* dpp, 

src/rgw/driver/posix/rgw_sal_posix.h

@@ -697,7 +697,6 @@ class POSIXObject : public StoreObject {
 			   rgw_bucket_dir_entry& o,
 			   CephContext* cct,
 		           RGWObjTier& tier_config,
-			   real_time& mtime,
 			   uint64_t olh_epoch,
 			   std::optional<uint64_t> days,
 			   const DoutPrefixProvider* dpp,

src/rgw/driver/rados/rgw_lc_tier.cc

@@ -319,8 +319,15 @@ int rgw_cloud_tier_get_object(RGWLCCloudTierCtx& tier_ctx, bool head,
 
     const auto aiter = generic_attrs_map.find(name);
     if (aiter != std::end(generic_attrs_map)) {
-      ldpp_dout(tier_ctx.dpp, 20) << __func__ << " Received attrs aiter->first = " << aiter->first << ", aiter->second = " << aiter->second << ret << dendl;
-     attrs[aiter->second] = bl;
+      attrs[aiter->second] = bl;
+    } else {
+      std::string s1 = boost::algorithm::to_lower_copy(header.first);
+      std::replace(s1.begin(), s1.end(), '_', '-');
+
+      // copy versioned epoch
+      if (s1 == "x-amz-meta-rgwx-versioned-epoch") {
+        attrs[s1] = bl;
+      }
     }
 
     if (header.first == "CONTENT_LENGTH") {

src/rgw/driver/rados/rgw_putobj_processor.cc

@@ -35,9 +35,11 @@ namespace rgw::putobj {
  * cloudtier config info read from the attrs.
  * Since these attrs are used internally for only replication, do not store them
  * in the head object.
+ * 
+ * Update versioned epoch incase the object is being restored.
  */
-void read_cloudtier_info_from_attrs(rgw::sal::Attrs& attrs, RGWObjCategory& category,
-                          RGWObjManifest& manifest) {
+int read_cloudtier_info_from_attrs(rgw::sal::Attrs& attrs, RGWObjCategory& category,
+                          std::optional<uint64_t>& olh_epoch, RGWObjManifest& manifest) {
   auto attr_iter = attrs.find(RGW_ATTR_CLOUD_TIER_TYPE);
   if (attr_iter != attrs.end()) {
     auto i = attr_iter->second;
@@ -58,11 +60,37 @@ void read_cloudtier_info_from_attrs(rgw::sal::Attrs& attrs, RGWObjCategory& cate
           manifest.set_tier_config(tier_config);
           attrs.erase(config_iter);
         } catch (buffer::error& err) {
+          return -EIO;
         }
       }
     }
     attrs.erase(attr_iter);
   }
+  attr_iter = attrs.find(RGW_ATTR_RESTORE_VERSIONED_EPOCH);
+  if (attr_iter != attrs.end()) {
+    try {
+      using ceph::decode;
+      uint64_t v_epoch = 0;
+      decode(v_epoch, attr_iter->second);
+      olh_epoch = v_epoch;
+      /*
+       * Keep this attr only for Temp restored copies as its needed while
+       * resetting head object post expiry.
+       */
+      auto r_iter = attrs.find(RGW_ATTR_RESTORE_TYPE);
+      if (r_iter != attrs.end()) {
+        rgw::sal::RGWRestoreType restore_type;
+        using ceph::decode;
+        decode(restore_type, r_iter->second);
+        if (restore_type != rgw::sal::RGWRestoreType::Temporary) {
+	        attrs.erase(attr_iter);
+        }
+      }
+    } catch (buffer::error& err) {
+      return -EIO;
+    }
+  }
+  return 0;
 }
 
 int HeadObjectProcessor::process(bufferlist&& data, uint64_t logical_offset)
@@ -390,7 +418,11 @@ int AtomicObjectProcessor::complete(
   obj_op.meta.zones_trace = zones_trace;
   obj_op.meta.modify_tail = true;
 
-  read_cloudtier_info_from_attrs(attrs, obj_op.meta.category, manifest);
+  r = read_cloudtier_info_from_attrs(attrs, obj_op.meta.category, obj_op.meta.olh_epoch, manifest);
+
+  if (r < 0) { // incase of any errors while decoding tier_config/restore attrs
+    return r;
+  }
 
   r = obj_op.write_meta(actual_size, accounted_size, attrs, rctx,
                         writer.get_trace(), flags & rgw::sal::FLAG_LOG_OP);

src/rgw/driver/rados/rgw_rados.cc

@@ -5248,7 +5248,6 @@ int RGWRados::restore_obj_from_cloud(RGWLCCloudTierCtx& tier_ctx,
                              const rgw_obj& dest_obj,
                              rgw_placement_rule& dest_placement,
                              RGWObjTier& tier_config,
-                             real_time& mtime,
                              uint64_t olh_epoch,
                              std::optional<uint64_t> days,
                              const DoutPrefixProvider *dpp,
@@ -5258,6 +5257,7 @@ int RGWRados::restore_obj_from_cloud(RGWLCCloudTierCtx& tier_ctx,
   //XXX: read below from attrs .. check transition_obj()
   ACLOwner owner;
   rgw::sal::Attrs attrs;
+  real_time mtime;
   const req_context rctx{dpp, y, nullptr};
   int ret = 0;
   bufferlist t, t_tier;
@@ -5299,6 +5299,19 @@ int RGWRados::restore_obj_from_cloud(RGWLCCloudTierCtx& tier_ctx,
                       return 0;
                     });
 
+  // fetch mtime of the object and other attrs of the object
+  // to check for restore_status
+  RGWRados::Object op_target(this, dest_bucket_info, obj_ctx, dest_obj);
+  RGWRados::Object::Read read_op(&op_target);
+  read_op.params.lastmod = &mtime;
+  read_op.params.attrs = &attrs;
+
+  ret = read_op.prepare(y, dpp);
+  if (ret < 0) {
+    ldpp_dout(dpp, 0) << "Restoring object(" << dest_obj << ") , read_op failed ret=" << ret << dendl;
+    return ret;
+  }
+
   uint64_t accounted_size = 0;
   string etag;
   real_time set_mtime;
@@ -5331,6 +5344,24 @@ int RGWRados::restore_obj_from_cloud(RGWLCCloudTierCtx& tier_ctx,
     return ret;
   }
 
+  {
+    if (!olh_epoch) {
+      const auto aiter = attrs.find("x-amz-meta-rgwx-versioned-epoch");
+      if (aiter != std::end(attrs)) {
+        std::optional<uint64_t> olh_ep = ceph::parse<uint64_t>(rgw_bl_str(aiter->second));
+        if (olh_ep) {
+          olh_epoch = *olh_ep;
+        }
+        attrs.erase(aiter);
+      }
+    }
+    if (olh_epoch) { // needed for only versioned objects
+      bufferlist bl;
+      encode(olh_epoch, bl);
+      attrs[RGW_ATTR_RESTORE_VERSIONED_EPOCH] = std::move(bl);
+    }
+  }
+
   {
     bufferlist bl;
     encode(rgw::sal::RGWRestoreStatus::CloudRestored, bl);
@@ -5409,6 +5440,9 @@ int RGWRados::restore_obj_from_cloud(RGWLCCloudTierCtx& tier_ctx,
     attrs[RGW_ATTR_STORAGE_CLASS] = std::move(bl);
   }
 
+  for (auto& iter: attrs) {
+    ldpp_dout(dpp, 30) << "Restore attrs set: " << iter.first << dendl;
+  }
   // XXX: handle COMPLETE_RETRY like in fetch_remote_obj
   bool canceled = false;
   rgw_zone_set zone_set{};
@@ -5419,7 +5453,6 @@ int RGWRados::restore_obj_from_cloud(RGWLCCloudTierCtx& tier_ctx,
     return ret;
   }
 
-  // XXX: handle olh_epoch for versioned objects like in fetch_remote_obj
   return ret; 
 }
 

src/rgw/driver/rados/rgw_rados.h

@@ -1255,7 +1255,6 @@ int restore_obj_from_cloud(RGWLCCloudTierCtx& tier_ctx,
                              const rgw_obj& dest_obj,
                              rgw_placement_rule& dest_placement,
                              RGWObjTier& tier_config,
-                             real_time& mtime,
                              uint64_t olh_epoch,
                              std::optional<uint64_t> days,
                              const DoutPrefixProvider *dpp,

src/rgw/driver/rados/rgw_sal_rados.cc

@@ -2899,7 +2899,6 @@ int RadosObject::restore_obj_from_cloud(Bucket* bucket,
                             	  rgw_bucket_dir_entry& o,
                           	  CephContext* cct,
                                   RGWObjTier& tier_config,
-                                  real_time& mtime,
                                   uint64_t olh_epoch,
                                   std::optional<uint64_t> days,
                                   const DoutPrefixProvider* dpp, 
@@ -2918,16 +2917,6 @@ int RadosObject::restore_obj_from_cloud(Bucket* bucket,
   int ret = 0;
   string src_storage_class = o.meta.storage_class; // or take src_placement also as input
 
-  // fetch mtime of the object
-  std::unique_ptr<rgw::sal::Object::ReadOp> read_op(get_read_op());
-  read_op->params.lastmod = &mtime;
-
-  ret = read_op->prepare(y, dpp);
-  if (ret < 0) {
-    ldpp_dout(dpp, 0) << "Restoring object(" << o.key << "): read_op failed ret=" << ret << dendl;
-    return ret;
-  }
-
   if (bucket_name.empty()) {
     bucket_name = "rgwx-" + zonegroup.get_name() + "-" + tier->get_storage_class() +
                     "-cloud-bucket";
@@ -2968,7 +2957,7 @@ int RadosObject::restore_obj_from_cloud(Bucket* bucket,
   ret = store->getRados()->restore_obj_from_cloud(tier_ctx, *rados_ctx,
                                 bucket->get_info(), get_obj(), placement_rule,
                                 tier_config,
-                                mtime, olh_epoch, days, dpp, y, flags & FLAG_LOG_OP);
+                                olh_epoch, days, dpp, y, flags & FLAG_LOG_OP);
 
   if (ret < 0) { //failed to restore
     ldpp_dout(dpp, 0) << "Restoring object(" << o.key << ") from the cloud endpoint(" << endpoint << ") failed, ret=" << ret << dendl;
@@ -3140,7 +3129,7 @@ int RadosObject::handle_obj_expiry(const DoutPrefixProvider* dpp, optional_yield
           RGWObjManifest *pmanifest;
           pmanifest = &m;
 
-	        Object* head_obj = (Object*)this;
+          Object* head_obj = (Object*)this;
           RGWObjTier tier_config;
           m.get_tier_config(&tier_config);
 
@@ -3151,12 +3140,21 @@ int RadosObject::handle_obj_expiry(const DoutPrefixProvider* dpp, optional_yield
           pmanifest->set_obj_size(0);
           obj_op.meta.manifest = pmanifest;
 
+          auto v_iter = attrs.find(RGW_ATTR_RESTORE_VERSIONED_EPOCH);
+          if (v_iter != attrs.end()) {
+            uint64_t versioned_epoch;
+            using ceph::decode;
+            decode(versioned_epoch, v_iter->second);
+            obj_op.meta.olh_epoch = versioned_epoch;
+          }
+
           // erase restore attrs
           attrs.erase(RGW_ATTR_RESTORE_STATUS);
           attrs.erase(RGW_ATTR_RESTORE_TYPE);
           attrs.erase(RGW_ATTR_RESTORE_TIME);
           attrs.erase(RGW_ATTR_RESTORE_EXPIRY_DATE);
           attrs.erase(RGW_ATTR_CLOUDTIER_STORAGE_CLASS);
+      	  attrs.erase(RGW_ATTR_RESTORE_VERSIONED_EPOCH);
 
           bufferlist bl;
           bl.append(tier_config.name);

src/rgw/driver/rados/rgw_sal_rados.h

@@ -639,7 +639,6 @@ class RadosObject : public StoreObject {
 			   rgw_bucket_dir_entry& o,
 			   CephContext* cct,
          		   RGWObjTier& tier_config,
-			   real_time& mtime,
 			   uint64_t olh_epoch,
   		           std::optional<uint64_t> days,
 			   const DoutPrefixProvider* dpp,