Merge pull request ceph#56180 from yehudasa/wip-objecter-local-read

objecter: use configurable to set local and balanced reads

Reviewed-by: Mark Kogan <[email protected]>
Reviewed-by: Casey Bodley <[email protected]>
Reviewed-by: Radoslaw Zarzynski <[email protected]>
Reviewed-by: Ilya Dryomov <[email protected]>

Author: yuriw

qa/suites/rados/verify/read-affinity/balance.yaml

@@ -0,0 +1,8 @@
+overrides:
+  ceph:
+    conf:
+      osd:
+        crush location: 'root=default host=host-$id'
+      client:
+        rados replica read policy: balance
+

qa/suites/rados/verify/read-affinity/default.yaml

@@ -0,0 +1,9 @@
+overrides:
+  ceph:
+    conf:
+      osd:
+        crush location: 'root=default host=host-$id'
+      client:
+        crush location: host=host-0
+        rados replica read policy: localize
+

qa/suites/rados/verify/read-affinity/local.yaml

@@ -29,6 +29,9 @@ namespace cls_lua_client {
     bufferlist inbl;
     encode(op, inbl);
 
-    return ioctx.exec(oid, "lua", "eval_bufferlist", inbl, output);
+    librados::ObjectWriteOperation wop;
+    int rval;
+    wop.exec("lua", "eval_bufferlist", inbl, &output, &rval);
+    return ioctx.operate(oid, &wop);
   }
 }

src/cls/lua/cls_lua_client.cc

@@ -28,15 +28,18 @@ namespace rados {
               const std::string& key,
               double value_to_add)
       {
-        bufferlist in, out;
+        bufferlist in;
         encode(key, in);
 
         std::stringstream stream;
         stream << value_to_add;
 
         encode(stream.str(), in);
 
-        return ioctx->exec(oid, "numops", "add", in, out);
+        librados::ObjectWriteOperation op;
+        op.exec("numops", "add", in);
+
+        return ioctx->operate(oid, &op);
       }
 
       int sub(librados::IoCtx *ioctx,
@@ -60,7 +63,10 @@ namespace rados {
 
         encode(stream.str(), in);
 
-        return ioctx->exec(oid, "numops", "mul", in, out);
+        librados::ObjectWriteOperation op;
+        op.exec("numops", "mul", in);
+
+        return ioctx->operate(oid, &op);
       }
 
       int div(librados::IoCtx *ioctx,

src/cls/numops/cls_numops_client.cc

@@ -68,7 +68,9 @@ namespace rados {
         bufferlist in;
         bufferlist out;
         encode(op, in);
-        int r = ioctx.exec(oid, "otp", "otp_check", in, out);
+        librados::ObjectWriteOperation wop;
+        wop.exec("otp", "otp_check", in);
+        int r = ioctx.operate(oid, &wop);
         if (r < 0) {
           return r;
         }

src/cls/otp/cls_otp_client.cc

@@ -1873,9 +1873,9 @@ int mirror_mode_set(librados::IoCtx *ioctx,
   bufferlist in_bl;
   encode(static_cast<uint32_t>(mirror_mode), in_bl);
 
-  bufferlist out_bl;
-  int r = ioctx->exec(RBD_MIRRORING, "rbd", "mirror_mode_set", in_bl,
-                      out_bl);
+  librados::ObjectWriteOperation op;
+  op.exec("rbd", "mirror_mode_set", in_bl);
+  int r = ioctx->operate(RBD_MIRRORING, &op);
   if (r < 0) {
     return r;
   }
@@ -2002,9 +2002,9 @@ int mirror_peer_remove(librados::IoCtx *ioctx,
   bufferlist in_bl;
   encode(uuid, in_bl);
 
-  bufferlist out_bl;
-  int r = ioctx->exec(RBD_MIRRORING, "rbd", "mirror_peer_remove", in_bl,
-                      out_bl);
+  librados::ObjectWriteOperation op;
+  op.exec("rbd", "mirror_peer_remove", in_bl);
+  int r = ioctx->operate(RBD_MIRRORING, &op);
   if (r < 0) {
     return r;
   }
@@ -2018,9 +2018,9 @@ int mirror_peer_set_client(librados::IoCtx *ioctx,
   encode(uuid, in_bl);
   encode(client_name, in_bl);
 
-  bufferlist out_bl;
-  int r = ioctx->exec(RBD_MIRRORING, "rbd", "mirror_peer_set_client",
-                      in_bl, out_bl);
+  librados::ObjectWriteOperation op;
+  op.exec("rbd", "mirror_peer_set_client", in_bl);
+  int r = ioctx->operate(RBD_MIRRORING, &op);
   if (r < 0) {
     return r;
   }
@@ -2034,9 +2034,9 @@ int mirror_peer_set_cluster(librados::IoCtx *ioctx,
   encode(uuid, in_bl);
   encode(cluster_name, in_bl);
 
-  bufferlist out_bl;
-  int r = ioctx->exec(RBD_MIRRORING, "rbd", "mirror_peer_set_cluster",
-                      in_bl, out_bl);
+  librados::ObjectWriteOperation op;
+  op.exec("rbd", "mirror_peer_set_cluster", in_bl);
+  int r = ioctx->operate(RBD_MIRRORING, &op);
   if (r < 0) {
     return r;
   }
@@ -2050,9 +2050,9 @@ int mirror_peer_set_direction(
   encode(uuid, in_bl);
   encode(static_cast<uint8_t>(mirror_peer_direction), in_bl);
 
-  bufferlist out_bl;
-  int r = ioctx->exec(RBD_MIRRORING, "rbd", "mirror_peer_set_direction",
-                      in_bl, out_bl);
+  librados::ObjectWriteOperation op;
+  op.exec("rbd", "mirror_peer_set_direction", in_bl);
+  int r = ioctx->operate(RBD_MIRRORING, &op);
   if (r < 0) {
     return r;
   }
@@ -2630,39 +2630,47 @@ int group_dir_list(librados::IoCtx *ioctx, const std::string &oid,
 int group_dir_add(librados::IoCtx *ioctx, const std::string &oid,
                   const std::string &name, const std::string &id)
 {
-  bufferlist in, out;
+  bufferlist in;
   encode(name, in);
   encode(id, in);
-  return ioctx->exec(oid, "rbd", "group_dir_add", in, out);
+  librados::ObjectWriteOperation op;
+  op.exec("rbd", "group_dir_add", in);
+  return ioctx->operate(oid, &op);
 }
 
 int group_dir_rename(librados::IoCtx *ioctx, const std::string &oid,
                      const std::string &src, const std::string &dest,
                      const std::string &id)
 {
-  bufferlist in, out;
+  bufferlist in;
   encode(src, in);
   encode(dest, in);
   encode(id, in);
-  return ioctx->exec(oid, "rbd", "group_dir_rename", in, out);
+  librados::ObjectWriteOperation op;
+  op.exec("rbd", "group_dir_rename", in);
+  return ioctx->operate(oid, &op);
 }
 
 int group_dir_remove(librados::IoCtx *ioctx, const std::string &oid,
                      const std::string &name, const std::string &id)
 {
-  bufferlist in, out;
+  bufferlist in;
   encode(name, in);
   encode(id, in);
-  return ioctx->exec(oid, "rbd", "group_dir_remove", in, out);
+  librados::ObjectWriteOperation op;
+  op.exec("rbd", "group_dir_remove", in);
+  return ioctx->operate(oid, &op);
 }
 
 int group_image_remove(librados::IoCtx *ioctx, const std::string &oid,
                        const cls::rbd::GroupImageSpec &spec)
 {
-  bufferlist bl, bl2;
+  bufferlist bl;
   encode(spec, bl);
 
-  return ioctx->exec(oid, "rbd", "group_image_remove", bl, bl2);
+  librados::ObjectWriteOperation op;
+  op.exec("rbd", "group_image_remove", bl);
+  return ioctx->operate(oid, &op);
 }
 
 int group_image_list(librados::IoCtx *ioctx,
@@ -2692,28 +2700,34 @@ int group_image_list(librados::IoCtx *ioctx,
 int group_image_set(librados::IoCtx *ioctx, const std::string &oid,
                     const cls::rbd::GroupImageStatus &st)
 {
-  bufferlist bl, bl2;
+  bufferlist bl;
   encode(st, bl);
 
-  return ioctx->exec(oid, "rbd", "group_image_set", bl, bl2);
+  librados::ObjectWriteOperation op;
+  op.exec("rbd", "group_image_set", bl);
+  return ioctx->operate(oid, &op);
 }
 
 int image_group_add(librados::IoCtx *ioctx, const std::string &oid,
                     const cls::rbd::GroupSpec &group_spec)
 {
-  bufferlist bl, bl2;
+  bufferlist bl;
   encode(group_spec, bl);
 
-  return ioctx->exec(oid, "rbd", "image_group_add", bl, bl2);
+  librados::ObjectWriteOperation op;
+  op.exec("rbd", "image_group_add", bl);
+  return ioctx->operate(oid, &op);
 }
 
 int image_group_remove(librados::IoCtx *ioctx, const std::string &oid,
                        const cls::rbd::GroupSpec &group_spec)
 {
-  bufferlist bl, bl2;
+  bufferlist bl;
   encode(group_spec, bl);
 
-  return ioctx->exec(oid, "rbd", "image_group_remove", bl, bl2);
+  librados::ObjectWriteOperation op;
+  op.exec("rbd", "image_group_remove", bl);
+  return ioctx->operate(oid, &op);
 }
 
 void image_group_get_start(librados::ObjectReadOperation *op)
@@ -2753,19 +2767,23 @@ int group_snap_set(librados::IoCtx *ioctx, const std::string &oid,
                    const cls::rbd::GroupSnapshot &snapshot)
 {
   using ceph::encode;
-  bufferlist inbl, outbl;
+  bufferlist inbl;
   encode(snapshot, inbl);
-  int r = ioctx->exec(oid, "rbd", "group_snap_set", inbl, outbl);
+  librados::ObjectWriteOperation op;
+  op.exec("rbd", "group_snap_set", inbl);
+  int r = ioctx->operate(oid, &op);
   return r;
 }
 
 int group_snap_remove(librados::IoCtx *ioctx, const std::string &oid,
                       const std::string &snap_id)
 {
   using ceph::encode;
-  bufferlist inbl, outbl;
+  bufferlist inbl;
   encode(snap_id, inbl);
-  return ioctx->exec(oid, "rbd", "group_snap_remove", inbl, outbl);
+  librados::ObjectWriteOperation op;
+  op.exec("rbd", "group_snap_remove", inbl);
+  return ioctx->operate(oid, &op);
 }
 
 int group_snap_get_by_id(librados::IoCtx *ioctx, const std::string &oid,

src/cls/rbd/cls_rbd_client.cc

@@ -481,7 +481,9 @@ int cls_rgw_bi_put(librados::IoCtx& io_ctx, const string oid, const rgw_cls_bi_e
   rgw_cls_bi_put_op call;
   call.entry = entry;
   encode(call, in);
-  int r = io_ctx.exec(oid, RGW_CLASS, RGW_BI_PUT, in, out);
+  librados::ObjectWriteOperation op;
+  op.exec(RGW_CLASS, RGW_BI_PUT, in);
+  int r = io_ctx.operate(oid, &op);
   if (r < 0)
     return r;
 
@@ -1223,15 +1225,19 @@ int cls_rgw_set_bucket_resharding(librados::IoCtx& io_ctx, const string& oid,
   cls_rgw_set_bucket_resharding_op call;
   call.entry = entry;
   encode(call, in);
-  return io_ctx.exec(oid, RGW_CLASS, RGW_SET_BUCKET_RESHARDING, in, out);
+  librados::ObjectWriteOperation op;
+  op.exec(RGW_CLASS, RGW_SET_BUCKET_RESHARDING, in);
+  return io_ctx.operate(oid, &op);
 }
 
 int cls_rgw_clear_bucket_resharding(librados::IoCtx& io_ctx, const string& oid)
 {
   bufferlist in, out;
   cls_rgw_clear_bucket_resharding_op call;
   encode(call, in);
-  return io_ctx.exec(oid, RGW_CLASS, RGW_CLEAR_BUCKET_RESHARDING, in, out);
+  librados::ObjectWriteOperation op;
+  op.exec(RGW_CLASS, RGW_CLEAR_BUCKET_RESHARDING, in);
+  return io_ctx.operate(oid, &op);
 }
 
 int cls_rgw_get_bucket_resharding(librados::IoCtx& io_ctx, const string& oid,

src/cls/rgw/cls_rgw_client.cc

@@ -6377,6 +6377,36 @@ options:
   min: 0
   flags:
   - runtime
+- name: rados_replica_read_policy
+  type: str
+  level: advanced
+  desc: read policy for sending read requests to OSD
+  fmt_desc : |
+    Policy for determining which OSD will receive read operations.
+    If set to ``default``, each PG's primary OSD will always be used
+    for read operations. If set to ``balance``, read operations will
+    be sent to a randomly selected OSD within the replica set. If set
+    to ``localize``, read operations will be sent to the closest OSD
+    as determined by the CRUSH map.
+  default: default
+  enum_values:
+  - default
+  - balance
+  - localize
+  flags:
+  - runtime
+- name: rados_replica_read_policy_on_objclass
+  type: bool
+  level: advanced
+  desc: enable read policy for sending read requests to OSD on objclass ops
+  fmt_desc : |
+    This would enable objclass ops to leverage read policy that can
+    determine which OSD will receive read operation. The reason
+    we might want to disable this is because objclass operations may
+    not be flagged correctly as read or write ops and we don't want
+    write ops to be sent to the wrong OSD (and system won't function
+    correctly).
+  default: false
 # true if LTTng-UST tracepoints should be enabled
 - name: rados_tracing
   type: bool