Merge pull request ceph#53799 from smanjara/wip-shilpa-cleanup-bi

rgw/multisite: bucket instance cleanup after deletion

Author: smanjara

PendingReleaseNotes

@@ -136,6 +136,9 @@
   `ceph fs subvolume earmark rm` have been added to set, get and remove earmark from a given subvolume.
 * RADOS: Add ``messenger dump`` command to retrieve runtime information
   on connections, sockets, and kernel TCP stats from the messenger.
+* RGW: deleted buckets are automatically cleaned up as part of trimming process.
+  DeleteBucket will start returning 409 BucketNotEmpty errors until empty
+  on all zones when sync policy is enabled.
 
 * RADOS: A performance botteneck in the balancer mgr module has been fixed.
   Related Tracker: https://tracker.ceph.com/issues/68657

src/rgw/driver/rados/rgw_bucket.cc

@@ -2697,6 +2697,7 @@ class RGWBucketInstanceMetadataHandler : public RGWMetadataHandler {
   RGWSI_Zone* svc_zone{nullptr};
   RGWSI_Bucket* svc_bucket{nullptr};
   RGWSI_BucketIndex* svc_bi{nullptr};
+  RGWDataChangesLog *svc_datalog{nullptr};
 
   int put_prepare(const DoutPrefixProvider* dpp, optional_yield y,
                   const std::string& entry, RGWBucketCompleteInfo& bci,
@@ -2711,9 +2712,10 @@ class RGWBucketInstanceMetadataHandler : public RGWMetadataHandler {
   RGWBucketInstanceMetadataHandler(rgw::sal::Driver* driver,
                                    RGWSI_Zone* svc_zone,
                                    RGWSI_Bucket* svc_bucket,
-                                   RGWSI_BucketIndex* svc_bi)
+                                   RGWSI_BucketIndex* svc_bi,
+                                   RGWDataChangesLog *svc_datalog)
     : driver(driver), svc_zone(svc_zone),
-      svc_bucket(svc_bucket), svc_bi(svc_bi) {}
+      svc_bucket(svc_bucket), svc_bi(svc_bi), svc_datalog(svc_datalog) {}
 
   string get_type() override { return "bucket.instance"; }
 
@@ -2874,6 +2876,25 @@ int RGWBucketInstanceMetadataHandler::put_prepare(
     /* existing bucket, keep its placement */
     bci.info.bucket.explicit_placement = old_bci->info.bucket.explicit_placement;
     bci.info.placement_rule = old_bci->info.placement_rule;
+
+    //if the bucket is being deleted, create and store a special log type for
+    //bucket instance cleanup in multisite setup
+    const auto& log = bci.info.layout.logs.back();
+    if (bci.info.bucket_deleted() && log.layout.type != rgw::BucketLogType::Deleted) {
+      const auto index_log = bci.info.layout.logs.back();
+      const int shards_num = rgw::num_shards(index_log.layout.in_index);
+      bci.info.layout.logs.push_back({log.gen+1, {rgw::BucketLogType::Deleted}});
+      ldpp_dout(dpp, 10) << "store log layout type: " <<  bci.info.layout.logs.back().layout.type << dendl;
+      for (int i = 0; i < shards_num; ++i) {
+        ldpp_dout(dpp, 10) << "adding to data_log shard_id: " << i << " of gen:" << index_log.gen << dendl;
+        int ret = svc_datalog->add_entry(dpp, bci.info, index_log, i, y);
+        if (ret < 0) {
+          ldpp_dout(dpp, 1) << "WARNING: failed writing data log for bucket="
+          << bci.info.bucket << ", shard_id=" << i << "of generation="
+          << index_log.gen << dendl;
+          } // datalog error is not fatal
+      }
+    }
   }
 
   //always keep bucket versioning enabled on archive zone
@@ -2884,7 +2905,7 @@ int RGWBucketInstanceMetadataHandler::put_prepare(
   /* record the read version (if any), store the new version */
   bci.info.objv_tracker.read_version = objv_tracker.read_version;
   bci.info.objv_tracker.write_version = objv_tracker.write_version;
-
+  
   return 0;
 }
 
@@ -2963,10 +2984,8 @@ int RGWBucketInstanceMetadataHandler::remove(std::string& entry, RGWObjVersionTr
     return ret;
   }
 
-  ret = svc_bucket->remove_bucket_instance_info(
-      entry, bci.info, &bci.info.objv_tracker, y, dpp);
-  if (ret < 0)
-    return ret;
+  // skip bucket instance removal. each zone will handle it independently during trimming
+
   std::ignore = update_bucket_topic_mappings(dpp, &bci, /*current_bci=*/nullptr,
                                              driver);
   return 0;
@@ -3032,14 +3051,16 @@ RGWBucketCtl::RGWBucketCtl(RGWSI_Zone *zone_svc,
                            RGWSI_Bucket *bucket_svc,
                            RGWSI_Bucket_Sync *bucket_sync_svc,
                            RGWSI_BucketIndex *bi_svc,
-                           RGWSI_User* user_svc)
+                           RGWSI_User* user_svc,
+                           RGWDataChangesLog *datalog_svc)
   : cct(zone_svc->ctx())
 {
   svc.zone = zone_svc;
   svc.bucket = bucket_svc;
   svc.bucket_sync = bucket_sync_svc;
   svc.bi = bi_svc;
   svc.user = user_svc;
+  svc.datalog_rados = datalog_svc;
 }
 
 void RGWBucketCtl::init(RGWUserCtl *user_ctl,
@@ -3528,11 +3549,13 @@ auto create_bucket_metadata_handler(librados::Rados& rados,
 auto create_bucket_instance_metadata_handler(rgw::sal::Driver* driver,
                                              RGWSI_Zone* svc_zone,
                                              RGWSI_Bucket* svc_bucket,
-                                             RGWSI_BucketIndex* svc_bi)
+                                             RGWSI_BucketIndex* svc_bi,
+                                             RGWDataChangesLog *svc_datalog)
     -> std::unique_ptr<RGWMetadataHandler>
 {
   return std::make_unique<RGWBucketInstanceMetadataHandler>(driver, svc_zone,
-                                                            svc_bucket, svc_bi);
+                                                            svc_bucket, svc_bi,
+                                                            svc_datalog);
 }
 
 auto create_archive_bucket_metadata_handler(librados::Rados& rados,
@@ -3547,11 +3570,13 @@ auto create_archive_bucket_metadata_handler(librados::Rados& rados,
 auto create_archive_bucket_instance_metadata_handler(rgw::sal::Driver* driver,
                                                      RGWSI_Zone* svc_zone,
                                                      RGWSI_Bucket* svc_bucket,
-                                                     RGWSI_BucketIndex* svc_bi)
+                                                     RGWSI_BucketIndex* svc_bi,
+                                                     RGWDataChangesLog *svc_datalog)
     -> std::unique_ptr<RGWMetadataHandler>
 {
   return std::make_unique<RGWArchiveBucketInstanceMetadataHandler>(driver, svc_zone,
-                                                                   svc_bucket, svc_bi);
+                                                                   svc_bucket, svc_bi,
+                                                                   svc_datalog);
 }
 
 void RGWBucketEntryPoint::generate_test_instances(list<RGWBucketEntryPoint*>& o)

src/rgw/driver/rados/rgw_bucket.h

@@ -180,7 +180,8 @@ auto create_bucket_metadata_handler(librados::Rados& rados,
 auto create_bucket_instance_metadata_handler(rgw::sal::Driver* driver,
                                              RGWSI_Zone* svc_zone,
                                              RGWSI_Bucket* svc_bucket,
-                                             RGWSI_BucketIndex* svc_bi)
+                                             RGWSI_BucketIndex* svc_bi,
+                                             RGWDataChangesLog *svc_datalog)
     -> std::unique_ptr<RGWMetadataHandler>;
 
 // archive bucket entrypoint metadata handler factory
@@ -193,7 +194,8 @@ auto create_archive_bucket_metadata_handler(librados::Rados& rados,
 auto create_archive_bucket_instance_metadata_handler(rgw::sal::Driver* driver,
                                                      RGWSI_Zone* svc_zone,
                                                      RGWSI_Bucket* svc_bucket,
-                                                     RGWSI_BucketIndex* svc_bi)
+                                                     RGWSI_BucketIndex* svc_bi,
+                                                     RGWDataChangesLog *svc_datalog)
     -> std::unique_ptr<RGWMetadataHandler>;
 
 
@@ -426,6 +428,7 @@ class RGWBucketCtl {
     RGWSI_Bucket_Sync *bucket_sync{nullptr};
     RGWSI_BucketIndex *bi{nullptr};
     RGWSI_User* user = nullptr;
+    RGWDataChangesLog *datalog_rados{nullptr};
   } svc;
 
   struct Ctl {
@@ -437,7 +440,8 @@ class RGWBucketCtl {
                RGWSI_Bucket *bucket_svc,
                RGWSI_Bucket_Sync *bucket_sync_svc,
                RGWSI_BucketIndex *bi_svc,
-               RGWSI_User* user_svc);
+               RGWSI_User* user_svc,
+               RGWDataChangesLog *datalog_svc);
 
   void init(RGWUserCtl *user_ctl,
             RGWDataChangesLog *datalog,

src/rgw/driver/rados/rgw_cr_rados.cc

@@ -674,6 +674,18 @@ int RGWAsyncPutBucketInstanceInfo::_send_request(const DoutPrefixProvider *dpp)
   return 0;
 }
 
+int RGWAsyncRemoveBucketInstanceInfo::_send_request(const DoutPrefixProvider *dpp)
+{
+  auto r = store->ctl()->bucket->remove_bucket_instance_info(bucket, bucket_info,
+						       null_yield, dpp);
+  if (r < 0) {
+    ldpp_dout(dpp, 0) << "ERROR: failed to remove bucket instance info for "
+		      << bucket_info.bucket << dendl;
+    return r;
+  }
+  return 0;
+}
+
 RGWRadosBILogTrimCR::RGWRadosBILogTrimCR(
   const DoutPrefixProvider *dpp,
   rgw::sal::RadosStore* store,
@@ -1192,3 +1204,56 @@ int RGWDataPostNotifyCR::operate(const DoutPrefixProvider* dpp)
   }
   return 0;
 }
+  
+RGWStatRemoteBucketCR::RGWStatRemoteBucketCR(const DoutPrefixProvider *dpp,
+				    rgw::sal::RadosStore* const store,
+            const rgw_zone_id source_zone,
+            const rgw_bucket& bucket,
+            RGWHTTPManager* http,
+            std::vector<rgw_zone_id> zids,
+            std::vector<bucket_unordered_list_result>& peer_result)
+      : RGWCoroutine(store->ctx()), dpp(dpp), store(store),
+      source_zone(source_zone), bucket(bucket), http(http),
+      zids(std::move(zids)), peer_result(peer_result) {}
+  
+int RGWStatRemoteBucketCR::operate(const DoutPrefixProvider *dpp) {
+  reenter(this) {
+    yield {
+      auto result = peer_result.begin();
+      for (auto& zid : zids) {
+        auto& zone_conn_map = store->getRados()->svc.zone->get_zone_conn_map();
+        auto ziter = zone_conn_map.find(zid);
+        if (ziter == zone_conn_map.end()) {
+          ldpp_dout(dpp, 0) << "WARNING: no connection to zone " << ziter->first << dendl;
+          continue;
+        }
+        ldpp_dout(dpp, 20) << "query bucket from: " << ziter->first << dendl;
+        RGWRESTConn *conn = ziter->second;
+
+        rgw_http_param_pair pairs[] = { { "versions" , NULL },
+					{ "format" , "json" },
+					{ "objs-container" , "true" },
+          { "max-keys", "1" },
+          { "allow-unordered", "true"},
+          { "key-marker" , NULL },
+					{ "version-id-marker" , NULL },
+	                                { NULL, NULL } };
+        string p = string("/") + bucket.get_key(':', 0);
+        spawn(new RGWReadRESTResourceCR<bucket_unordered_list_result>(store->ctx(), &*conn, &*http, p, pairs, &*result), false);
+        ++result;
+      }
+    }
+
+    while (num_spawned()) {
+      yield wait_for_child();
+      collect(&child_ret, nullptr);
+      if (child_ret < 0) {
+        drain_all();
+        return set_cr_error(child_ret);
+      }
+    }
+
+    return set_cr_done();
+  }
+  return 0;
+}