librados: disable local affinity for IoCtx::exec() and aio_exec() calls

These calls are read calls but can and have been used as write calls,
which doesn't work with local (and balanced) affinity.

Signed-off-by: Yehuda Sadeh <[email protected]>

Author: yehudasa

src/librados/IoCtxImpl.cc

@@ -682,7 +682,8 @@ int librados::IoCtxImpl::operate(const object_t& oid, ::ObjectOperation *o,
 int librados::IoCtxImpl::operate_read(const object_t& oid,
 				      ::ObjectOperation *o,
 				      bufferlist *pbl,
-				      int flags)
+				      int flags,
+                                      int flags_mask)
 {
   if (!o->size())
     return 0;
@@ -701,6 +702,7 @@ int librados::IoCtxImpl::operate_read(const object_t& oid,
     oid, oloc,
     *o, snap_seq, pbl,
     flags | extra_op_flags,
+    flags_mask,
     onack, &ver);
   objecter->op_submit(objecter_op);
 
@@ -741,7 +743,7 @@ int librados::IoCtxImpl::aio_operate_read(const object_t &oid,
   trace.event("init root span");
   Objecter::Op *objecter_op = objecter->prepare_read_op(
     oid, oloc,
-    *o, snap_seq, pbl, flags | extra_op_flags,
+    *o, snap_seq, pbl, flags | extra_op_flags, -1,
     oncomplete, &c->objver, nullptr, 0, &trace);
   objecter->op_submit(objecter_op, &c->tid);
   trace.event("rados operate read submitted");
@@ -883,7 +885,7 @@ int librados::IoCtxImpl::aio_sparse_read(const object_t oid,
 
   Objecter::Op *o = objecter->prepare_read_op(
     oid, oloc,
-    onack->m_ops, snapid, NULL, extra_op_flags,
+    onack->m_ops, snapid, NULL, extra_op_flags, -1,
     onack, &c->objver);
   objecter->op_submit(o, &c->tid);
   return 0;
@@ -932,7 +934,7 @@ int librados::IoCtxImpl::aio_cmpext(const object_t& oid,
   onack->m_ops.cmpext(off, cmp_len, cmp_buf, NULL);
 
   Objecter::Op *o = objecter->prepare_read_op(
-    oid, oloc, onack->m_ops, snap_seq, NULL, extra_op_flags, onack, &c->objver);
+    oid, oloc, onack->m_ops, snap_seq, NULL, extra_op_flags, -1, onack, &c->objver);
   objecter->op_submit(o, &c->tid);
   return 0;
 }
@@ -1311,7 +1313,7 @@ int librados::IoCtxImpl::exec(const object_t& oid,
   ::ObjectOperation rd;
   prepare_assert_ops(&rd);
   rd.call(cls, method, inbl);
-  return operate_read(oid, &rd, &outbl);
+  return operate_read(oid, &rd, &outbl, 0, ~(CEPH_OSD_FLAG_LOCALIZE_READS | CEPH_OSD_FLAG_BALANCE_READS));
 }
 
 int librados::IoCtxImpl::aio_exec(const object_t& oid, AioCompletionImpl *c,
@@ -1331,7 +1333,7 @@ int librados::IoCtxImpl::aio_exec(const object_t& oid, AioCompletionImpl *c,
   prepare_assert_ops(&rd);
   rd.call(cls, method, inbl);
   Objecter::Op *o = objecter->prepare_read_op(
-    oid, oloc, rd, snap_seq, outbl, extra_op_flags, oncomplete, &c->objver);
+    oid, oloc, rd, snap_seq, outbl, extra_op_flags, ~(CEPH_OSD_FLAG_LOCALIZE_READS | CEPH_OSD_FLAG_BALANCE_READS), oncomplete, &c->objver);
   objecter->op_submit(o, &c->tid);
   return 0;
 }
@@ -1357,7 +1359,7 @@ int librados::IoCtxImpl::aio_exec(const object_t& oid, AioCompletionImpl *c,
   prepare_assert_ops(&rd);
   rd.call(cls, method, inbl);
   Objecter::Op *o = objecter->prepare_read_op(
-    oid, oloc, rd, snap_seq, &c->bl, extra_op_flags, oncomplete, &c->objver);
+    oid, oloc, rd, snap_seq, &c->bl, extra_op_flags, ~(CEPH_OSD_FLAG_LOCALIZE_READS | CEPH_OSD_FLAG_BALANCE_READS), oncomplete, &c->objver);
   objecter->op_submit(o, &c->tid);
   return 0;
 }

src/librados/IoCtxImpl.h

@@ -155,7 +155,7 @@ struct librados::IoCtxImpl {
   int rmxattr(const object_t& oid, const char *name);
 
   int operate(const object_t& oid, ::ObjectOperation *o, ceph::real_time *pmtime, int flags=0, const jspan_context *otel_trace = nullptr);
-  int operate_read(const object_t& oid, ::ObjectOperation *o, bufferlist *pbl, int flags=0);
+  int operate_read(const object_t& oid, ::ObjectOperation *o, bufferlist *pbl, int flags=0, int flags_mask=-1);
   int aio_operate(const object_t& oid, ::ObjectOperation *o,
 		  AioCompletionImpl *c, const SnapContext& snap_context,
 		  const ceph::real_time *pmtime, int flags,

src/osdc/Objecter.h

@@ -3092,12 +3092,13 @@ class Objecter : public md_config_obs_t, public Dispatcher {
   Op *prepare_read_op(
     const object_t& oid, const object_locator_t& oloc,
     ObjectOperation& op,
-    snapid_t snapid, ceph::buffer::list *pbl, int flags,
+    snapid_t snapid, ceph::buffer::list *pbl,
+    int flags, int flags_mask,
     Context *onack, version_t *objver = NULL,
     int *data_offset = NULL,
     uint64_t features = 0,
     ZTracer::Trace *parent_trace = nullptr) {
-    Op *o = new Op(oid, oloc, std::move(op.ops), get_read_flags(flags), onack, objver,
+    Op *o = new Op(oid, oloc, std::move(op.ops), get_read_flags(flags) & flags_mask, onack, objver,
 		   data_offset, parent_trace);
     o->priority = op.priority;
     o->snapid = snapid;
@@ -3118,7 +3119,7 @@ class Objecter : public md_config_obs_t, public Dispatcher {
     Context *onack, version_t *objver = NULL,
     int *data_offset = NULL,
     uint64_t features = 0) {
-    Op *o = prepare_read_op(oid, oloc, op, snapid, pbl, flags, onack, objver,
+    Op *o = prepare_read_op(oid, oloc, op, snapid, pbl, flags, -1, onack, objver,
 			    data_offset);
     if (features)
       o->features = features;