Merge pull request ceph#59864 from cbodley/wip-68134

rgw/rados: RGWRados::delete_objs_inline() uses AioThrottle

Reviewed-by: Pritha Srivastava <[email protected]>

Author: cbodley

src/cls/rgw/cls_rgw_types.h

@@ -5,6 +5,7 @@
 
 #include <string>
 #include <list>
+#include <vector>
 #include <boost/container/flat_map.hpp>
 #include "common/ceph_time.h"
 #include "common/Formatter.h"
@@ -1197,16 +1198,14 @@ struct cls_rgw_obj {
 WRITE_CLASS_ENCODER(cls_rgw_obj)
 
 struct cls_rgw_obj_chain {
-  std::list<cls_rgw_obj> objs;
-
-  cls_rgw_obj_chain() {}
+  std::vector<cls_rgw_obj> objs;
 
   void push_obj(const std::string& pool, const cls_rgw_obj_key& key, const std::string& loc) {
     cls_rgw_obj obj;
     obj.pool = pool;
     obj.key = key;
     obj.loc = loc;
-    objs.push_back(obj);
+    objs.push_back(std::move(obj));
   }
 
   void encode(ceph::buffer::list& bl) const {
@@ -1223,9 +1222,9 @@ struct cls_rgw_obj_chain {
 
   void dump(ceph::Formatter *f) const {
     f->open_array_section("objs");
-    for (std::list<cls_rgw_obj>::const_iterator p = objs.begin(); p != objs.end(); ++p) {
+    for (const auto& o : objs) {
       f->open_object_section("obj");
-      p->dump(f);
+      o.dump(f);
       f->close_section();
     }
     f->close_section();

src/rgw/driver/rados/rgw_gc.cc

@@ -653,7 +653,6 @@ int RGWGC::process(int index, int max_secs, bool expired_only,
 	info.tag << "', time=" << info.time << ", chain.objs.size()=" <<
 	info.chain.objs.size() << dendl;
 
-      std::list<cls_rgw_obj>::iterator liter;
       cls_rgw_obj_chain& chain = info.chain;
 
       utime_t now = ceph_clock_now();
@@ -668,9 +667,7 @@ int RGWGC::process(int index, int max_secs, bool expired_only,
         }
       }
       if (! chain.objs.empty()) {
-	for (liter = chain.objs.begin(); liter != chain.objs.end(); ++liter) {
-	  cls_rgw_obj& obj = *liter;
-
+	for (const auto& obj : chain.objs) {
 	  if (obj.pool != last_pool) {
 	    delete ctx;
 	    ctx = new IoCtx;

src/rgw/driver/rados/rgw_rados.cc

@@ -5369,33 +5369,41 @@ std::tuple<int, std::optional<cls_rgw_obj_chain>> RGWRados::send_chain_to_gc(cls
 void RGWRados::delete_objs_inline(const DoutPrefixProvider *dpp, cls_rgw_obj_chain& chain,
                                   const string& tag, optional_yield y)
 {
-  string last_pool;
-  std::unique_ptr<IoCtx> ctx(new IoCtx);
-  int ret = 0;
-  for (auto liter = chain.objs.begin(); liter != chain.objs.end(); ++liter) {
-    cls_rgw_obj& obj = *liter;
-    if (obj.pool != last_pool) {
-      ctx.reset(new IoCtx);
-      ret = rgw_init_ioctx(dpp, get_rados_handle(), obj.pool, *ctx);
-      if (ret < 0) {
-        last_pool = "";
-        ldpp_dout(dpp, 0) << "ERROR: failed to create ioctx pool=" <<
-        obj.pool << dendl;
-        continue;
-      }
-      last_pool = obj.pool;
-    }
-    ctx->locator_set_key(obj.loc);
-    const string& oid = obj.key.name; /* just stored raw oid there */
-    ldpp_dout(dpp, 5) << "delete_objs_inline: removing " << obj.pool <<
-    ":" << obj.key.name << dendl;
+  if (chain.objs.empty()) {
+    return;
+  }
+
+  // initialize an IoCtx for the first object's pool. RGWObjManifest uses the
+  // same pool for all tail objects
+  auto obj = chain.objs.begin();
+
+  librados::IoCtx ioctx;
+  int ret = rgw_init_ioctx(dpp, get_rados_handle(), obj->pool, ioctx);
+  if (ret < 0) {
+    return;
+  }
+
+  // issue deletions in parallel, up to max_aio at a time
+  auto aio = rgw::make_throttle(cct->_conf->rgw_multi_obj_del_max_aio, y);
+  static constexpr uint64_t cost = 1; // 1 throttle unit per request
+  static constexpr uint64_t id = 0; // ids unused
+
+  for (; obj != chain.objs.end(); ++obj) {
     ObjectWriteOperation op;
     cls_refcount_put(op, tag, true);
-    ret = rgw_rados_operate(dpp, *ctx, oid, &op, y);
-    if (ret < 0) {
-      ldpp_dout(dpp, 5) << "delete_objs_inline: refcount put returned error " << ret << dendl;
-    }
+
+    rgw_raw_obj raw;
+    raw.pool = std::move(obj->pool);
+    raw.oid = std::move(obj->key.name);
+    raw.loc = std::move(obj->loc);
+
+    auto completed = aio->get(std::move(raw), rgw::Aio::librados_op(
+            ioctx, std::move(op), y), cost, id);
+    std::ignore = rgw::check_for_errors(completed);
   }
+
+  auto completed = aio->drain();
+  std::ignore = rgw::check_for_errors(completed);
 }
 
 static void accumulate_raw_stats(const rgw_bucket_dir_header& header,

src/rgw/rgw_admin.cc

@@ -8806,10 +8806,7 @@ int main(int argc, const char **argv)
 	formatter->dump_string("tag", info.tag);
 	formatter->dump_stream("time") << info.time;
 	formatter->open_array_section("objs");
-        list<cls_rgw_obj>::iterator liter;
-	cls_rgw_obj_chain& chain = info.chain;
-	for (liter = chain.objs.begin(); liter != chain.objs.end(); ++liter) {
-	  cls_rgw_obj& obj = *liter;
+	for (const auto& obj : info.chain.objs) {
           encode_json("obj", obj, formatter.get());
 	}
 	formatter->close_section(); // objs

src/test/cls_rgw/test_cls_rgw.cc

@@ -854,7 +854,7 @@ TEST_F(cls_rgw, gc_set)
     /* verify expected num of objects in chain */
     ASSERT_EQ(2, (int)entry.chain.objs.size());
 
-    list<cls_rgw_obj>::iterator oiter = entry.chain.objs.begin();
+    auto oiter = entry.chain.objs.begin();
     cls_rgw_obj obj1, obj2;
 
     /* create expected objects */
@@ -932,7 +932,7 @@ TEST_F(cls_rgw, gc_list)
     /* verify expected num of objects in chain */
     ASSERT_EQ(2, (int)entry.chain.objs.size());
 
-    list<cls_rgw_obj>::iterator oiter = entry.chain.objs.begin();
+    auto oiter = entry.chain.objs.begin();
     cls_rgw_obj obj1, obj2;
 
     /* create expected objects */