Merge pull request ceph#59611 from cbodley/wip-cls-rgw-busy-resharding

cls/rgw: duplicate reshard checks in all cls_rgw write operations

Reviewed-by: J. Eric Ivancich <[email protected]>

Author: cbodley

src/cls/rgw/cls_rgw.cc

@@ -186,7 +186,6 @@ bool BucketIndexAioManager::wait_for_completions(int valid_ret_code,
   return true;
 }
 
-// note: currently only called by testing code
 void cls_rgw_bucket_init_index(ObjectWriteOperation& o)
 {
   bufferlist in;
@@ -200,18 +199,24 @@ static bool issue_bucket_index_init_op(librados::IoCtx& io_ctx,
   bufferlist in;
   librados::ObjectWriteOperation op;
   op.create(true);
-  op.exec(RGW_CLASS, RGW_BUCKET_INIT_INDEX, in);
+  cls_rgw_bucket_init_index(op);
   return manager->aio_operate(io_ctx, shard_id, oid, &op);
 }
 
+void cls_rgw_bucket_init_index2(ObjectWriteOperation& o)
+{
+  bufferlist in;
+  o.exec(RGW_CLASS, RGW_BUCKET_INIT_INDEX2, in);
+}
+
 static bool issue_bucket_index_init_op2(librados::IoCtx& io_ctx,
 				       const int shard_id,
 				       const string& oid,
 				       BucketIndexAioManager *manager) {
   bufferlist in;
   librados::ObjectWriteOperation op;
   op.create(true);
-  op.exec(RGW_CLASS, RGW_BUCKET_INIT_INDEX2, in);
+  cls_rgw_bucket_init_index2(op);
   return manager->aio_operate(io_ctx, shard_id, oid, &op);
 }
 
@@ -470,32 +475,6 @@ int cls_rgw_bi_get(librados::IoCtx& io_ctx, const string oid,
   return 0;
 }
 
-int cls_rgw_bi_get_vals(librados::IoCtx& io_ctx, const std::string oid,
-                        std::set<std::string>& log_entries_wanted,
-                        std::list<rgw_cls_bi_entry> *entries)
-{
-  bufferlist in, out;
-  struct rgw_cls_bi_get_vals_op call;
-  call.log_entries_wanted = std::move(log_entries_wanted);
-  encode(call, in);
-  int r = io_ctx.exec(oid, RGW_CLASS, RGW_BI_GET_VALS, in, out);
-  if (r < 0)
-    return r;
-
-  struct rgw_cls_bi_list_ret op_ret;
-  auto iter = out.cbegin();
-  try {
-    decode(op_ret, iter);
-  } catch (ceph::buffer::error& err) {
-    return -EIO;
-  }
-
-  if (entries)
-    entries->swap(op_ret.entries);
-
-  return 0;
-}
-
 int cls_rgw_bi_put(librados::IoCtx& io_ctx, const string oid, const rgw_cls_bi_entry& entry)
 {
   bufferlist in, out;
@@ -518,6 +497,21 @@ void cls_rgw_bi_put(ObjectWriteOperation& op, const string oid, const rgw_cls_bi
   op.exec(RGW_CLASS, RGW_BI_PUT, in);
 }
 
+void cls_rgw_bi_put_entries(librados::ObjectWriteOperation& op,
+                            std::vector<rgw_cls_bi_entry> entries,
+                            bool check_existing)
+{
+  const auto call = rgw_cls_bi_put_entries_op{
+    .entries = std::move(entries),
+    .check_existing = check_existing
+  };
+
+  bufferlist in;
+  encode(call, in);
+
+  op.exec(RGW_CLASS, RGW_BI_PUT_ENTRIES, in);
+}
+
 /* nb: any entries passed in are replaced with the results of the cls
  * call, so caller does not need to clear entries between calls
  */

src/cls/rgw/cls_rgw_client.cc

@@ -264,6 +264,7 @@ class BucketIndexShardsManager {
 
 /* bucket index */
 void cls_rgw_bucket_init_index(librados::ObjectWriteOperation& o);
+void cls_rgw_bucket_init_index2(librados::ObjectWriteOperation& o);
 
 class CLSRGWConcurrentIO {
 protected:
@@ -383,11 +384,14 @@ void cls_rgw_obj_check_mtime(librados::ObjectOperation& o, const ceph::real_time
 int cls_rgw_bi_get(librados::IoCtx& io_ctx, const std::string oid,
                    BIIndexType index_type, const cls_rgw_obj_key& key,
                    rgw_cls_bi_entry *entry);
-int cls_rgw_bi_get_vals(librados::IoCtx& io_ctx, const std::string oid,
-                        std::set<std::string>& log_entries_wanted,
-                        std::list<rgw_cls_bi_entry> *entries);
 int cls_rgw_bi_put(librados::IoCtx& io_ctx, const std::string oid, const rgw_cls_bi_entry& entry);
 void cls_rgw_bi_put(librados::ObjectWriteOperation& op, const std::string oid, const rgw_cls_bi_entry& entry);
+// Write the given array of index entries and update bucket stats accordingly.
+// If existing entries may be overwritten, pass check_existing=true to decrement
+// their stats first.
+void cls_rgw_bi_put_entries(librados::ObjectWriteOperation& op,
+                            std::vector<rgw_cls_bi_entry> entries,
+                            bool check_existing);
 int cls_rgw_bi_list(librados::IoCtx& io_ctx, const std::string& oid,
                    const std::string& name, const std::string& marker, uint32_t max,
                    std::list<rgw_cls_bi_entry> *entries, bool *is_truncated, bool reshardlog = false);
@@ -671,8 +675,16 @@ int cls_rgw_reshard_list(librados::IoCtx& io_ctx, const std::string& oid, std::s
 int cls_rgw_reshard_get(librados::IoCtx& io_ctx, const std::string& oid, cls_rgw_reshard_entry& entry);
 #endif
 
-/* resharding attribute on bucket index shard headers */
+// If writes to the bucket index should be blocked during resharding, fail with
+// the given error code. RGWRados::guard_reshard() calls this in a loop to retry
+// the write until the reshard completes.
+//
+// As of the T release, all index write ops in cls_rgw perform this check
+// themselves. RGW can stop issuing this call in the T+2 (V) release once it
+// knows that OSDs are running T at least. The call can be safely removed from
+// cls_rgw in the T+4 (X) release.
 void cls_rgw_guard_bucket_resharding(librados::ObjectOperation& op, int ret_err);
+
 // these overloads which call io_ctx.operate() should not be called in the rgw.
 // rgw_rados_operate() should be called after the overloads w/o calls to io_ctx.operate()
 #ifndef CLS_CLIENT_HIDE_IOCTX

src/cls/rgw/cls_rgw_client.h

@@ -33,8 +33,8 @@ constexpr int RGWBIAdvanceAndRetryError = -EFBIG;
 #define RGW_OBJ_CHECK_MTIME "obj_check_mtime"
 
 #define RGW_BI_GET "bi_get"
-#define RGW_BI_GET_VALS "bi_get_vals"
 #define RGW_BI_PUT "bi_put"
+#define RGW_BI_PUT_ENTRIES "bi_put_entries"
 #define RGW_BI_LIST "bi_list"
 
 #define RGW_RESHARD_LOG_TRIM "reshard_log_trim"

src/cls/rgw/cls_rgw_const.h

@@ -580,3 +580,9 @@ void cls_rgw_get_bucket_resharding_op::generate_test_instances(
 void cls_rgw_get_bucket_resharding_op::dump(Formatter *f) const
 {
 }
+
+void rgw_cls_bi_put_entries_op::dump(Formatter *f) const
+{
+  encode_json("entries", entries, f);
+  encode_json("check_existing", check_existing, f);
+}

src/cls/rgw/cls_rgw_ops.cc

@@ -731,23 +731,6 @@ struct rgw_cls_bi_get_ret {
 };
 WRITE_CLASS_ENCODER(rgw_cls_bi_get_ret)
 
-struct rgw_cls_bi_get_vals_op {
-  std::set<std::string> log_entries_wanted;
-
-  void encode(ceph::buffer::list& bl) const {
-    ENCODE_START(1, 1, bl);
-    encode(log_entries_wanted, bl);
-    ENCODE_FINISH(bl);
-  }
-
-  void decode(ceph::buffer::list::const_iterator& bl) {
-    DECODE_START(1, bl);
-    decode(log_entries_wanted, bl);
-    DECODE_FINISH(bl);
-  }
-};
-WRITE_CLASS_ENCODER(rgw_cls_bi_get_vals_op)
-
 struct rgw_cls_bi_put_op {
   rgw_cls_bi_entry entry;
 
@@ -777,6 +760,35 @@ struct rgw_cls_bi_put_op {
 };
 WRITE_CLASS_ENCODER(rgw_cls_bi_put_op)
 
+struct rgw_cls_bi_put_entries_op {
+  std::vector<rgw_cls_bi_entry> entries;
+  bool check_existing = false;
+
+  void encode(ceph::buffer::list& bl) const {
+    ENCODE_START(1, 1, bl);
+    encode(entries, bl);
+    encode(check_existing, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(ceph::buffer::list::const_iterator& bl) {
+    DECODE_START(1, bl);
+    decode(entries, bl);
+    decode(check_existing, bl);
+    DECODE_FINISH(bl);
+  }
+
+  void dump(ceph::Formatter *f) const;
+
+  static void generate_test_instances(std::list<rgw_cls_bi_put_entries_op*>& o) {
+    o.push_back(new rgw_cls_bi_put_entries_op);
+    o.push_back(new rgw_cls_bi_put_entries_op);
+    o.back()->entries.push_back({.idx = "entry"});
+    o.back()->check_existing = true;
+  }
+};
+WRITE_CLASS_ENCODER(rgw_cls_bi_put_entries_op)
+
 struct rgw_cls_bi_list_op {
   uint32_t max;
   std::string name_filter; // limit result to one object and its instances

src/cls/rgw/cls_rgw_ops.h

@@ -406,7 +406,7 @@ void rgw_cls_bi_entry::dump(Formatter *f) const
 
 bool rgw_cls_bi_entry::get_info(cls_rgw_obj_key *key,
                                 RGWObjCategory *category,
-                                rgw_bucket_category_stats *accounted_stats)
+                                rgw_bucket_category_stats *accounted_stats) const
 {
   using ceph::decode;
   auto iter = data.cbegin();

src/cls/rgw/cls_rgw_types.cc

@@ -18,6 +18,8 @@
 #define CEPH_RGW_DIR_SUGGEST_LOG_OP  0x80
 #define CEPH_RGW_DIR_SUGGEST_OP_MASK 0x7f
 
+#define CLS_RGW_ERR_BUSY_RESHARDING 2300 // also in rgw_common.h, don't change!
+
 constexpr uint64_t CEPH_RGW_DEFAULT_TAG_TIMEOUT = 120; // in seconds
 
 class JSONObj;
@@ -197,20 +199,17 @@ inline std::ostream& operator<<(std::ostream& out, RGWObjCategory c) {
 }
 
 struct rgw_bucket_dir_entry_meta {
-  RGWObjCategory category;
-  uint64_t size;
+  RGWObjCategory category = RGWObjCategory::None;
+  uint64_t size = 0;
   ceph::real_time mtime;
   std::string etag;
   std::string owner;
   std::string owner_display_name;
   std::string content_type;
-  uint64_t accounted_size;
+  uint64_t accounted_size = 0;
   std::string user_data;
   std::string storage_class;
-  bool appendable;
-
-  rgw_bucket_dir_entry_meta() :
-    category(RGWObjCategory::None), size(0), accounted_size(0), appendable(false) { }
+  bool appendable = false;
 
   void encode(ceph::buffer::list &bl) const {
     ENCODE_START(7, 3, bl);
@@ -479,12 +478,10 @@ enum class BIIndexType : uint8_t {
 struct rgw_bucket_category_stats;
 
 struct rgw_cls_bi_entry {
-  BIIndexType type;
+  BIIndexType type = BIIndexType::Invalid;
   std::string idx;
   ceph::buffer::list data;
 
-  rgw_cls_bi_entry() : type(BIIndexType::Invalid) {}
-
   void encode(ceph::buffer::list& bl) const {
     ENCODE_START(1, 1, bl);
     encode(type, bl);
@@ -507,7 +504,7 @@ struct rgw_cls_bi_entry {
   void decode_json(JSONObj *obj, cls_rgw_obj_key *effective_key = NULL);
   static void generate_test_instances(std::list<rgw_cls_bi_entry*>& o);
   bool get_info(cls_rgw_obj_key *key, RGWObjCategory *category,
-		rgw_bucket_category_stats *accounted_stats);
+		rgw_bucket_category_stats *accounted_stats) const;
 };
 WRITE_CLASS_ENCODER(rgw_cls_bi_entry)
 

src/cls/rgw/cls_rgw_types.h

@@ -9247,17 +9247,6 @@ int RGWRados::bi_get(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_
   return cls_rgw_bi_get(ref.ioctx, ref.obj.oid, index_type, key, entry);
 }
 
-int RGWRados::bi_get_vals(BucketShard& bs, set<string>& log_entries_wanted,
-                          list<rgw_cls_bi_entry> *entries, optional_yield y)
-{
-  auto& ref = bs.bucket_obj;
-  int ret = cls_rgw_bi_get_vals(ref.ioctx, ref.obj.oid, log_entries_wanted, entries);
-  if (ret < 0)
-    return ret;
-
-  return 0;
-}
-
 void RGWRados::bi_put(ObjectWriteOperation& op, BucketShard& bs, rgw_cls_bi_entry& entry, optional_yield y)
 {
   auto& ref = bs.bucket_obj;

src/rgw/driver/rados/rgw_rados.cc

@@ -1525,7 +1525,6 @@ class RGWRados
   int bi_get_instance(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_bucket_dir_entry *dirent, optional_yield y);
   int bi_get_olh(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_bucket_olh_entry *olh, optional_yield y);
   int bi_get(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, BIIndexType index_type, rgw_cls_bi_entry *entry, optional_yield y);
-  int bi_get_vals(BucketShard& bs, std::set<std::string>& log_entries_wanted, std::list<rgw_cls_bi_entry> *entries, optional_yield y);
   void bi_put(librados::ObjectWriteOperation& op, BucketShard& bs, rgw_cls_bi_entry& entry, optional_yield y);
   int bi_put(BucketShard& bs, rgw_cls_bi_entry& entry, optional_yield y);
   int bi_put(const DoutPrefixProvider *dpp, rgw_bucket& bucket, rgw_obj& obj, rgw_cls_bi_entry& entry, optional_yield y);