cls/rgw: define lc ops in terms of ObjectOperation instead of IoCtx

cbodley · cbodley · commit 8476a02bed81 · 2024-07-23T08:46:10.000-04:00
clean up the lc functions that were supposed to be hidden by
CLS_CLIENT_HIDE_IOCTX

this allows rgw to use them asynchonously with rgw_rados_operate() and
optional_yield, and warn about blocking calls that should be async

Signed-off-by: Casey Bodley &lt;cbodley@redhat.com&gt;
diff --git a/src/cls/rgw/cls_rgw_client.cc b/src/cls/rgw/cls_rgw_client.cc
@@ -940,92 +940,90 @@ void cls_rgw_gc_remove(librados::ObjectWriteOperation& op, const vector<string>&
   op.exec(RGW_CLASS, RGW_GC_REMOVE, in);
 }
 
-int cls_rgw_lc_get_head(IoCtx& io_ctx, const string& oid, cls_rgw_lc_obj_head& head)
+void cls_rgw_lc_get_head(ObjectReadOperation& op, bufferlist& out)
 {
-  bufferlist in, out;
-  int r = io_ctx.exec(oid, RGW_CLASS, RGW_LC_GET_HEAD, in, out);
-  if (r < 0)
-    return r;
+  bufferlist in;
+  op.exec(RGW_CLASS, RGW_LC_GET_HEAD, in, &out, nullptr);
+}
 
+int cls_rgw_lc_get_head_decode(const bufferlist& out, cls_rgw_lc_obj_head& head)
+{
   cls_rgw_lc_get_head_ret ret;
   try {
     auto iter = out.cbegin();
     decode(ret, iter);
   } catch (ceph::buffer::error& err) {
     return -EIO;
   }
-  head = ret.head;
+  head = std::move(ret.head);
 
- return r;
+  return 0;
 }
 
-int cls_rgw_lc_put_head(IoCtx& io_ctx, const string& oid, cls_rgw_lc_obj_head& head)
+void cls_rgw_lc_put_head(ObjectWriteOperation& op, const cls_rgw_lc_obj_head& head)
 {
-  bufferlist in, out;
+  bufferlist in;
   cls_rgw_lc_put_head_op call;
   call.head = head;
   encode(call, in);
-  int r = io_ctx.exec(oid, RGW_CLASS, RGW_LC_PUT_HEAD, in, out);
-  return r;
+  op.exec(RGW_CLASS, RGW_LC_PUT_HEAD, in);
 }
 
-int cls_rgw_lc_get_next_entry(IoCtx& io_ctx, const string& oid, const string& marker,
-			      cls_rgw_lc_entry& entry)
+void cls_rgw_lc_get_next_entry(ObjectReadOperation& op, const string& marker,
+                               bufferlist& out)
 {
-  bufferlist in, out;
+  bufferlist in;
   cls_rgw_lc_get_next_entry_op call;
   call.marker = marker;
   encode(call, in);
-  int r = io_ctx.exec(oid, RGW_CLASS, RGW_LC_GET_NEXT_ENTRY, in, out);
-  if (r < 0)
-    return r;
+  op.exec(RGW_CLASS, RGW_LC_GET_NEXT_ENTRY, in, &out, nullptr);
+}
 
+int cls_rgw_lc_get_next_entry_decode(const bufferlist& out, cls_rgw_lc_entry& entry)
+{
   cls_rgw_lc_get_next_entry_ret ret;
   try {
     auto iter = out.cbegin();
     decode(ret, iter);
   } catch (ceph::buffer::error& err) {
     return -EIO;
   }
-  entry = ret.entry;
+  entry = std::move(ret.entry);
 
- return r;
+  return 0;
 }
 
-int cls_rgw_lc_rm_entry(IoCtx& io_ctx, const string& oid,
-			const cls_rgw_lc_entry& entry)
+void cls_rgw_lc_rm_entry(ObjectWriteOperation& op,
+                         const cls_rgw_lc_entry& entry)
 {
-  bufferlist in, out;
+  bufferlist in;
   cls_rgw_lc_rm_entry_op call;
   call.entry = entry;
   encode(call, in);
-  int r = io_ctx.exec(oid, RGW_CLASS, RGW_LC_RM_ENTRY, in, out);
- return r;
+  op.exec(RGW_CLASS, RGW_LC_RM_ENTRY, in);
 }
 
-int cls_rgw_lc_set_entry(IoCtx& io_ctx, const string& oid,
-			 const cls_rgw_lc_entry& entry)
+void cls_rgw_lc_set_entry(ObjectWriteOperation& op,
+                          const cls_rgw_lc_entry& entry)
 {
   bufferlist in, out;
   cls_rgw_lc_set_entry_op call;
   call.entry = entry;
   encode(call, in);
-  int r = io_ctx.exec(oid, RGW_CLASS, RGW_LC_SET_ENTRY, in, out);
-  return r;
+  op.exec(RGW_CLASS, RGW_LC_SET_ENTRY, in);
 }
 
-int cls_rgw_lc_get_entry(IoCtx& io_ctx, const string& oid,
-			 const std::string& marker, cls_rgw_lc_entry& entry)
+void cls_rgw_lc_get_entry(ObjectReadOperation& op, const std::string& marker,
+                          bufferlist& out)
 {
-  bufferlist in, out;
-  cls_rgw_lc_get_entry_op call{marker};;
+  bufferlist in;
+  cls_rgw_lc_get_entry_op call{marker};
   encode(call, in);
-  int r = io_ctx.exec(oid, RGW_CLASS, RGW_LC_GET_ENTRY, in, out);
-
-  if (r < 0) {
-    return r;
-  }
+  op.exec(RGW_CLASS, RGW_LC_GET_ENTRY, in, &out, nullptr);
+}
 
+int cls_rgw_lc_get_entry_decode(const bufferlist& out, cls_rgw_lc_entry& entry)
+{
   cls_rgw_lc_get_entry_ret ret;
   try {
     auto iter = out.cbegin();
@@ -1035,28 +1033,24 @@ int cls_rgw_lc_get_entry(IoCtx& io_ctx, const string& oid,
   }
 
   entry = std::move(ret.entry);
-  return r;
+  return 0;
 }
 
-int cls_rgw_lc_list(IoCtx& io_ctx, const string& oid,
-                    const string& marker,
-                    uint32_t max_entries,
-                    vector<cls_rgw_lc_entry>& entries)
+void cls_rgw_lc_list(ObjectReadOperation& op, const string& marker,
+                     uint32_t max_entries, bufferlist& out)
 {
-  bufferlist in, out;
-  cls_rgw_lc_list_entries_op op;
-
-  entries.clear();
-
-  op.marker = marker;
-  op.max_entries = max_entries;
+  bufferlist in;
+  cls_rgw_lc_list_entries_op call;
+  call.marker = marker;
+  call.max_entries = max_entries;
 
-  encode(op, in);
+  encode(call, in);
 
-  int r = io_ctx.exec(oid, RGW_CLASS, RGW_LC_LIST_ENTRIES, in, out);
-  if (r < 0)
-    return r;
+  op.exec(RGW_CLASS, RGW_LC_LIST_ENTRIES, in, &out, nullptr);
+}
 
+int cls_rgw_lc_list_decode(const bufferlist& out, std::vector<cls_rgw_lc_entry>& entries)
+{
   cls_rgw_lc_list_entries_ret ret;
   try {
     auto iter = out.cbegin();
@@ -1069,7 +1063,7 @@ int cls_rgw_lc_list(IoCtx& io_ctx, const string& oid,
 	    [](const cls_rgw_lc_entry& a, const cls_rgw_lc_entry& b)
 	      { return a.bucket < b.bucket; });
   entries = std::move(ret.entries);
-  return r;
+  return 0;
 }
 
 void cls_rgw_mp_upload_part_info_update(librados::ObjectWriteOperation& op,
diff --git a/src/cls/rgw/cls_rgw_client.h b/src/cls/rgw/cls_rgw_client.h
@@ -609,19 +609,19 @@ int cls_rgw_gc_list(librados::IoCtx& io_ctx, std::string& oid, std::string& mark
 #endif
 
 /* lifecycle */
-// these overloads which call io_ctx.operate() should not be called in the rgw.
-// rgw_rados_operate() should be called after the overloads w/o calls to io_ctx.operate()
-#ifndef CLS_CLIENT_HIDE_IOCTX
-int cls_rgw_lc_get_head(librados::IoCtx& io_ctx, const std::string& oid, cls_rgw_lc_obj_head& head);
-int cls_rgw_lc_put_head(librados::IoCtx& io_ctx, const std::string& oid, cls_rgw_lc_obj_head& head);
-int cls_rgw_lc_get_next_entry(librados::IoCtx& io_ctx, const std::string& oid, const std::string& marker, cls_rgw_lc_entry& entry);
-int cls_rgw_lc_rm_entry(librados::IoCtx& io_ctx, const std::string& oid, const cls_rgw_lc_entry& entry);
-int cls_rgw_lc_set_entry(librados::IoCtx& io_ctx, const std::string& oid, const cls_rgw_lc_entry& entry);
-int cls_rgw_lc_get_entry(librados::IoCtx& io_ctx, const std::string& oid, const std::string& marker, cls_rgw_lc_entry& entry);
-int cls_rgw_lc_list(librados::IoCtx& io_ctx, const std::string& oid,
-		    const std::string& marker, uint32_t max_entries,
-                    std::vector<cls_rgw_lc_entry>& entries);
-#endif
+void cls_rgw_lc_get_head(librados::ObjectReadOperation& op, bufferlist& bl);
+int cls_rgw_lc_get_head_decode(const bufferlist& bl, cls_rgw_lc_obj_head& head);
+void cls_rgw_lc_put_head(librados::ObjectWriteOperation& op, const cls_rgw_lc_obj_head& head);
+void cls_rgw_lc_get_next_entry(librados::ObjectReadOperation& op, const std::string& marker, bufferlist& bl);
+int cls_rgw_lc_get_next_entry_decode(const bufferlist& bl, cls_rgw_lc_entry& entry);
+void cls_rgw_lc_rm_entry(librados::ObjectWriteOperation& op, const cls_rgw_lc_entry& entry);
+void cls_rgw_lc_set_entry(librados::ObjectWriteOperation& op, const cls_rgw_lc_entry& entry);
+void cls_rgw_lc_get_entry(librados::ObjectReadOperation& op, const std::string& marker, bufferlist& bl);
+int cls_rgw_lc_get_entry_decode(const bufferlist& bl, cls_rgw_lc_entry& entry);
+void cls_rgw_lc_list(librados::ObjectReadOperation& op,
+                     const std::string& marker, uint32_t max_entries,
+                     bufferlist& bl);
+int cls_rgw_lc_list_decode(const bufferlist& bl, std::vector<cls_rgw_lc_entry>& entries);
 
 /* multipart */
 void cls_rgw_mp_upload_part_info_update(librados::ObjectWriteOperation& op, const std::string& part_key, const RGWUploadPartInfo& info);
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -3608,10 +3608,21 @@ int RadosLifecycle::get_entry(const DoutPrefixProvider* dpp, optional_yield y,
                               const std::string& oid, const std::string& marker,
 			      std::unique_ptr<LCEntry>* entry)
 {
+  librados::ObjectReadOperation op;
+  bufferlist bl;
+  cls_rgw_lc_get_entry(op, marker, bl);
+
+  auto& ioctx = *store->getRados()->get_lc_pool_ctx();
+  int ret = rgw_rados_operate(dpp, ioctx, oid, &op, nullptr, y);
+  if (ret < 0) {
+    return ret;
+  }
+
   cls_rgw_lc_entry cls_entry;
-  int ret = cls_rgw_lc_get_entry(*store->getRados()->get_lc_pool_ctx(), oid, marker, cls_entry);
-  if (ret)
+  ret = cls_rgw_lc_get_entry_decode(bl, cls_entry);
+  if (ret < 0) {
     return ret;
+  }
 
   *entry = std::make_unique<StoreLCEntry>(cls_entry.bucket, cls_entry.start_time, cls_entry.status);
   return 0;
@@ -3621,12 +3632,21 @@ int RadosLifecycle::get_next_entry(const DoutPrefixProvider* dpp, optional_yield
                                    const std::string& oid, const std::string& marker,
 				   std::unique_ptr<LCEntry>* entry)
 {
-  cls_rgw_lc_entry cls_entry;
-  int ret = cls_rgw_lc_get_next_entry(*store->getRados()->get_lc_pool_ctx(), oid, marker,
-				      cls_entry);
+  librados::ObjectReadOperation op;
+  bufferlist bl;
+  cls_rgw_lc_get_next_entry(op, marker, bl);
 
-  if (ret)
+  auto& ioctx = *store->getRados()->get_lc_pool_ctx();
+  int ret = rgw_rados_operate(dpp, ioctx, oid, &op, nullptr, y);
+  if (ret < 0) {
     return ret;
+  }
+
+  cls_rgw_lc_entry cls_entry;
+  ret = cls_rgw_lc_get_next_entry_decode(bl, cls_entry);
+  if (ret < 0) {
+    return ret;
+  }
 
   *entry = std::make_unique<StoreLCEntry>(cls_entry.bucket, cls_entry.start_time, cls_entry.status);
   return 0;
@@ -3641,7 +3661,11 @@ int RadosLifecycle::set_entry(const DoutPrefixProvider* dpp, optional_yield y,
   cls_entry.start_time = entry.get_start_time();
   cls_entry.status = entry.get_status();
 
-  return cls_rgw_lc_set_entry(*store->getRados()->get_lc_pool_ctx(), oid, cls_entry);
+  librados::ObjectWriteOperation op;
+  cls_rgw_lc_set_entry(op, cls_entry);
+
+  auto& ioctx = *store->getRados()->get_lc_pool_ctx();
+  return rgw_rados_operate(dpp, ioctx, oid, &op, y);
 }
 
 int RadosLifecycle::list_entries(const DoutPrefixProvider* dpp, optional_yield y,
@@ -3650,11 +3674,21 @@ int RadosLifecycle::list_entries(const DoutPrefixProvider* dpp, optional_yield y
 {
   entries.clear();
 
-  vector<cls_rgw_lc_entry> cls_entries;
-  int ret = cls_rgw_lc_list(*store->getRados()->get_lc_pool_ctx(), oid, marker, max_entries, cls_entries);
+  librados::ObjectReadOperation op;
+  bufferlist bl;
+  cls_rgw_lc_list(op, marker, max_entries, bl);
 
-  if (ret < 0)
+  auto& ioctx = *store->getRados()->get_lc_pool_ctx();
+  int ret = rgw_rados_operate(dpp, ioctx, oid, &op, nullptr, y);
+  if (ret < 0) {
+    return ret;
+  }
+
+  vector<cls_rgw_lc_entry> cls_entries;
+  ret = cls_rgw_lc_list_decode(bl, cls_entries);
+  if (ret < 0) {
     return ret;
+  }
 
   for (auto& entry : cls_entries) {
     entries.push_back(std::make_unique<StoreLCEntry>(entry.bucket, oid,
@@ -3668,21 +3702,35 @@ int RadosLifecycle::rm_entry(const DoutPrefixProvider* dpp, optional_yield y,
                              const std::string& oid, LCEntry& entry)
 {
   cls_rgw_lc_entry cls_entry;
-
   cls_entry.bucket = entry.get_bucket();
   cls_entry.start_time = entry.get_start_time();
   cls_entry.status = entry.get_status();
 
-  return cls_rgw_lc_rm_entry(*store->getRados()->get_lc_pool_ctx(), oid, cls_entry);
+  librados::ObjectWriteOperation op;
+  cls_rgw_lc_rm_entry(op, cls_entry);
+
+  auto& ioctx = *store->getRados()->get_lc_pool_ctx();
+  return rgw_rados_operate(dpp, ioctx, oid, &op, y);
 }
 
 int RadosLifecycle::get_head(const DoutPrefixProvider* dpp, optional_yield y,
                              const std::string& oid, std::unique_ptr<LCHead>* head)
 {
+  librados::ObjectReadOperation op;
+  bufferlist bl;
+  cls_rgw_lc_get_head(op, bl);
+
+  auto& ioctx = *store->getRados()->get_lc_pool_ctx();
+  int ret = rgw_rados_operate(dpp, ioctx, oid, &op, nullptr, y);
+  if (ret < 0) {
+    return ret;
+  }
+
   cls_rgw_lc_obj_head cls_head;
-  int ret = cls_rgw_lc_get_head(*store->getRados()->get_lc_pool_ctx(), oid, cls_head);
-  if (ret)
+  ret = cls_rgw_lc_get_head_decode(bl, cls_head);
+  if (ret < 0) {
     return ret;
+  }
 
   *head = std::make_unique<StoreLCHead>(cls_head.start_date, cls_head.shard_rollover_date, cls_head.marker);
   return 0;
@@ -3697,7 +3745,11 @@ int RadosLifecycle::put_head(const DoutPrefixProvider* dpp, optional_yield y,
   cls_head.start_date = head.get_start_date();
   cls_head.shard_rollover_date = head.get_shard_rollover_date();
 
-  return cls_rgw_lc_put_head(*store->getRados()->get_lc_pool_ctx(), oid, cls_head);
+  librados::ObjectWriteOperation op;
+  cls_rgw_lc_put_head(op, cls_head);
+
+  auto& ioctx = *store->getRados()->get_lc_pool_ctx();
+  return rgw_rados_operate(dpp, ioctx, oid, &op, y);
 }
 
 std::unique_ptr<LCSerializer> RadosLifecycle::get_serializer(const std::string& lock_name,
