Merge pull request ceph#61263 from aainscow/legacy_ec

OSD: Refactor EC to have two versions: Legacy and New EC

Author: SrinivasaBharath

src/crimson/osd/CMakeLists.txt

@@ -46,6 +46,7 @@ add_executable(crimson-osd
   ${PROJECT_SOURCE_DIR}/src/objclass/class_api.cc
   ${PROJECT_SOURCE_DIR}/src/osd/ClassHandler.cc
   ${PROJECT_SOURCE_DIR}/src/osd/ECUtil.cc
+  ${PROJECT_SOURCE_DIR}/src/osd/ECUtilL.cc
   ${PROJECT_SOURCE_DIR}/src/osd/osd_op_util.cc
   ${PROJECT_SOURCE_DIR}/src/osd/OSDCap.cc
   ${PROJECT_SOURCE_DIR}/src/osd/PeeringState.cc

src/osd/CMakeLists.txt

@@ -17,9 +17,6 @@ set(osd_srcs
   PGLog.cc
   PrimaryLogPG.cc
   ReplicatedBackend.cc
-  ECCommon.cc
-  ECBackend.cc
-  ECTransaction.cc
   PGBackend.cc
   OSDCap.cc
   scrubber/pg_scrubber.cc
@@ -36,8 +33,6 @@ set(osd_srcs
   Session.cc
   SnapMapper.cc
   osd_types.cc
-  ECUtil.cc
-  ExtentCache.cc
   scheduler/OpScheduler.cc
   scheduler/OpSchedulerItem.cc
   scheduler/mClockScheduler.cc
@@ -46,6 +41,18 @@ set(osd_srcs
   recovery_types.cc
   MissingLoc.cc
   osd_perf_counters.cc
+  ECCommonL.cc
+  ECBackendL.cc
+  ECExtentCacheL.cc
+  ECTransactionL.cc
+  ECUtilL.cc
+  ECCommon.cc
+  ECBackend.cc
+  ExtentCache.cc
+  ECTransaction.cc
+  ECUtil.cc
+  ECInject.cc
+  ECInject.h
   ${CMAKE_SOURCE_DIR}/src/common/TrackedOp.cc
   ${CMAKE_SOURCE_DIR}/src/mgr/OSDPerfMetricTypes.cc
   ${osd_cyg_functions_src}

src/osd/ECBackend.cc

@@ -16,13 +16,16 @@
 #include <sstream>
 
 #include "ECBackend.h"
+#include "ECInject.h"
 #include "messages/MOSDPGPush.h"
 #include "messages/MOSDPGPushReply.h"
 #include "messages/MOSDECSubOpWrite.h"
 #include "messages/MOSDECSubOpWriteReply.h"
 #include "messages/MOSDECSubOpRead.h"
 #include "messages/MOSDECSubOpReadReply.h"
 #include "ECMsgTypes.h"
+#include "ECTypes.h"
+#include "ECSwitch.h"
 
 #include "PrimaryLogPG.h"
 #include "osd_tracer.h"
@@ -121,16 +124,14 @@ void ECBackend::RecoveryBackend::RecoveryOp::dump(Formatter *f) const
 
 ECBackend::ECBackend(
   PGBackend::Listener *pg,
-  const coll_t &coll,
-  ObjectStore::CollectionHandle &ch,
-  ObjectStore *store,
   CephContext *cct,
   ErasureCodeInterfaceRef ec_impl,
-  uint64_t stripe_width)
-  : PGBackend(cct, pg, store, coll, ch),
+  uint64_t stripe_width,
+  ECSwitch *s)
+  : parent(pg), cct(cct), switcher(s),
     read_pipeline(cct, ec_impl, this->sinfo, get_parent()->get_eclistener()),
     rmw_pipeline(cct, ec_impl, this->sinfo, get_parent()->get_eclistener(), *this),
-    recovery_backend(cct, this->coll, ec_impl, this->sinfo, read_pipeline, unstable_hashinfo_registry, get_parent(), this),
+    recovery_backend(cct, switcher->coll, ec_impl, this->sinfo, read_pipeline, unstable_hashinfo_registry, get_parent(), this),
     ec_impl(ec_impl),
     sinfo(ec_impl, stripe_width),
     unstable_hashinfo_registry(cct, ec_impl) {
@@ -195,8 +196,8 @@ struct RecoveryMessages {
     const map<pg_shard_t, vector<pair<int, int>>> &need,
     bool attrs)
   {
-    list<ECCommon::ec_align_t> to_read;
-    to_read.emplace_back(ECCommon::ec_align_t{off, len, 0});
+    list<ec_align_t> to_read;
+    to_read.emplace_back(ec_align_t{off, len, 0});
     ceph_assert(!recovery_reads.count(hoid));
     want_to_read.insert(make_pair(hoid, std::move(_want_to_read)));
     recovery_reads.insert(
@@ -233,7 +234,7 @@ void ECBackend::handle_recovery_push(
      !(get_parent()->pgb_is_primary()) &&
      get_parent()->pg_is_remote_backfilling()) {
     struct stat st;
-    int r = store->stat(ch, ghobject_t(op.soid, ghobject_t::NO_GEN,
+    int r = switcher->store->stat(switcher->ch, ghobject_t(op.soid, ghobject_t::NO_GEN,
                         get_parent()->whoami_shard().shard), &st);
     if (r == 0) {
       get_parent()->pg_sub_local_num_bytes(st.st_size);
@@ -463,7 +464,7 @@ struct RecoveryReadCompleter : ECCommon::ReadCompleter {
   void finish_single_request(
     const hobject_t &hoid,
     ECCommon::read_result_t &res,
-    list<ECCommon::ec_align_t>,
+    list<ec_align_t>,
     set<int> wanted_to_read) override
   {
     if (!(res.r == 0 && res.errors.empty())) {
@@ -720,13 +721,13 @@ void ECBackend::RecoveryBackend::continue_recovery_op(
 }
 
 void ECBackend::run_recovery_op(
-  RecoveryHandle *_h,
+  PGBackend::RecoveryHandle *_h,
   int priority)
 {
   ceph_assert(_h);
   ECRecoveryHandle &h = static_cast<ECRecoveryHandle&>(*_h);
   recovery_backend.run_recovery_op(h, priority);
-  send_recovery_deletes(priority, h.deletes);
+  switcher->send_recovery_deletes(priority, h.deletes);
   delete _h;
 }
 
@@ -751,7 +752,7 @@ int ECBackend::recover_object(
   eversion_t v,
   ObjectContextRef head,
   ObjectContextRef obc,
-  RecoveryHandle *_h)
+  PGBackend::RecoveryHandle *_h)
 {
   return recovery_backend.recover_object(hoid, v, head, obc, _h);
 }
@@ -761,7 +762,7 @@ int ECBackend::RecoveryBackend::recover_object(
   eversion_t v,
   ObjectContextRef head,
   ObjectContextRef obc,
-  RecoveryHandle *_h)
+  PGBackend::RecoveryHandle *_h)
 {
   ECRecoveryHandle *h = static_cast<ECRecoveryHandle*>(_h);
   h->ops.push_back(RecoveryOp());
@@ -832,7 +833,7 @@ bool ECBackend::_handle_message(
     auto op = _op->get_req<MOSDECSubOpRead>();
     MOSDECSubOpReadReply *reply = new MOSDECSubOpReadReply;
     reply->pgid = get_parent()->primary_spg_t();
-    reply->map_epoch = get_osdmap_epoch();
+    reply->map_epoch = switcher->get_osdmap_epoch();
     reply->min_epoch = get_parent()->get_interval_start_epoch();
     handle_sub_read(op->op.from, op->op, &(reply->op), _op->pg_trace);
     reply->trace = _op->pg_trace;
@@ -918,7 +919,7 @@ void ECBackend::sub_write_committed(
     get_parent()->update_last_complete_ondisk(last_complete);
     MOSDECSubOpWriteReply *r = new MOSDECSubOpWriteReply;
     r->pgid = get_parent()->primary_spg_t();
-    r->map_epoch = get_osdmap_epoch();
+    r->map_epoch = switcher->get_osdmap_epoch();
     r->min_epoch = get_parent()->get_interval_start_epoch();
     r->op.tid = tid;
     r->op.last_complete = last_complete;
@@ -929,7 +930,7 @@ void ECBackend::sub_write_committed(
     r->trace = trace;
     r->trace.event("sending sub op commit");
     get_parent()->send_message_osd_cluster(
-      get_parent()->primary_shard().osd, r, get_osdmap_epoch());
+      get_parent()->primary_shard().osd, r, switcher->get_osdmap_epoch());
   }
 }
 
@@ -946,14 +947,14 @@ void ECBackend::handle_sub_write(
   trace.event("handle_sub_write");
 
   if (cct->_conf->bluestore_debug_inject_read_err &&
-      ec_inject_test_write_error3(op.soid)) {
+      ECInject::test_write_error3(op.soid)) {
     ceph_abort_msg("Error inject - OSD down");
   }
   if (!get_parent()->pgb_is_primary())
     get_parent()->update_stats(op.stats);
   ObjectStore::Transaction localt;
   if (!op.temp_added.empty()) {
-    add_temp_objs(op.temp_added);
+    switcher->add_temp_objs(op.temp_added);
   }
   if (op.backfill_or_async_recovery) {
     for (set<hobject_t>::iterator i = op.temp_removed.begin();
@@ -962,14 +963,14 @@ void ECBackend::handle_sub_write(
       dout(10) << __func__ << ": removing object " << *i
 	       << " since we won't get the transaction" << dendl;
       localt.remove(
-	coll,
+	switcher->coll,
 	ghobject_t(
 	  *i,
 	  ghobject_t::NO_GEN,
 	  get_parent()->whoami_shard().shard));
     }
   }
-  clear_temp_objs(op.temp_removed);
+  switcher->clear_temp_objs(op.temp_removed);
   dout(30) << __func__ << " missing before " << get_parent()->get_log().get_missing().get_items() << dendl;
   // flag set to true during async recovery
   bool async = false;
@@ -1033,8 +1034,8 @@ void ECBackend::handle_sub_read(
           (op.subchunks.find(i->first)->second.front().second == 
                                             ec_impl->get_sub_chunk_count())) {
         dout(20) << __func__ << " case1: reading the complete chunk/shard." << dendl;
-        r = store->read(
-	  ch,
+        r = switcher->store->read(
+	  switcher->ch,
 	  ghobject_t(i->first, ghobject_t::NO_GEN, shard),
 	  j->get<0>(),
 	  j->get<1>(),
@@ -1050,8 +1051,8 @@ void ECBackend::handle_sub_read(
              m += sinfo.get_chunk_size()) {
           for (auto &&k:op.subchunks.find(i->first)->second) {
             bufferlist bl0;
-            r = store->read(
-                ch,
+            r = switcher->store->read(
+                switcher->ch,
                 ghobject_t(i->first, ghobject_t::NO_GEN, shard),
                 j->get<0>() + m + (k.first)*subchunk_size,
                 (k.second)*subchunk_size,
@@ -1101,7 +1102,7 @@ void ECBackend::handle_sub_read(
 	int r = object_stat(i->first, &st);
         if (r >= 0) {
 	  dout(10) << __func__ << ": found on disk, size " << st.st_size << dendl;
-	  r = PGBackend::objects_get_attrs(i->first, &attrs);
+	  r = switcher->objects_get_attrs_with_hinfo(i->first, &attrs);
 	}
 	if (r >= 0) {
 	  hinfo = unstable_hashinfo_registry.get_hash_info(i->first, false, attrs, st.st_size);
@@ -1148,8 +1149,8 @@ void ECBackend::handle_sub_read(
 	     << *i << dendl;
     if (reply->errors.count(*i))
       continue;
-    int r = store->getattrs(
-      ch,
+    int r = switcher->store->getattrs(
+      switcher->ch,
       ghobject_t(
 	*i, ghobject_t::NO_GEN, shard),
       reply->attrs_read[*i]);
@@ -1196,7 +1197,7 @@ void ECBackend::handle_sub_write_reply(
   }
   if (cct->_conf->bluestore_debug_inject_read_err &&
       (i->second->pending_commit.size() == 1) &&
-      ec_inject_test_write_error2(i->second->hoid)) {
+      ECInject::test_write_error2(i->second->hoid)) {
     std::string cmd =
       "{ \"prefix\": \"osd down\", \"ids\": [\"" + std::to_string( get_parent()->whoami() ) + "\"] }";
     vector<std::string> vcmd{cmd};
@@ -1224,7 +1225,7 @@ void ECBackend::handle_sub_read_reply(
     for (auto i = op.buffers_read.begin();
 	 i != op.buffers_read.end();
 	 ++i) {
-      if (ec_inject_test_read_error0(ghobject_t(i->first, ghobject_t::NO_GEN, op.from.shard))) {
+      if (ECInject::test_read_error0(ghobject_t(i->first, ghobject_t::NO_GEN, op.from.shard))) {
 	dout(0) << __func__ << " Error inject - EIO error for shard " << op.from.shard << dendl;
 	op.buffers_read.erase(i->first);
 	op.attrs_read.erase(i->first);
@@ -1482,7 +1483,7 @@ std::tuple<
     return { r, {}, 0 };
   }
   map<string, bufferlist, less<>> real_attrs;
-  if (int r = PGBackend::objects_get_attrs(hoid, &real_attrs); r < 0) {
+  if (int r = switcher->objects_get_attrs_with_hinfo(hoid, &real_attrs); r < 0) {
     dout(10) << __func__ << ": get attr error " << r << " on" << hoid << dendl;
     return { r, {}, 0 };
   }
@@ -1567,7 +1568,8 @@ int ECBackend::objects_read_sync(
 
 void ECBackend::objects_read_async(
   const hobject_t &hoid,
-  const list<pair<ECCommon::ec_align_t,
+  uint64_t object_size,
+  const list<pair<ec_align_t,
                   pair<bufferlist*, Context*>>> &to_read,
   Context *on_complete,
   bool fast_read)
@@ -1599,14 +1601,14 @@ void ECBackend::objects_read_async(
   struct cb {
     ECBackend *ec;
     hobject_t hoid;
-    list<pair<ECCommon::ec_align_t,
+    list<pair<ec_align_t,
 	      pair<bufferlist*, Context*> > > to_read;
     unique_ptr<Context> on_complete;
     cb(const cb&) = delete;
     cb(cb &&) = default;
     cb(ECBackend *ec,
        const hobject_t &hoid,
-       const list<pair<ECCommon::ec_align_t,
+       const list<pair<ec_align_t,
                   pair<bufferlist*, Context*> > > &to_read,
        Context *on_complete)
       : ec(ec),
@@ -1680,7 +1682,7 @@ void ECBackend::objects_read_async(
 
 void ECBackend::objects_read_and_reconstruct(
   const map<hobject_t,
-    std::list<ECBackend::ec_align_t>
+    std::list<ec_align_t>
   > &reads,
   bool fast_read,
   GenContextURef<ECCommon::ec_extents_t &&> &&func)
@@ -1697,8 +1699,8 @@ int ECBackend::object_stat(
   const hobject_t &hoid,
   struct stat* st)
 {
-  int r = store->stat(
-    ch,
+  int r = switcher->store->stat(
+    switcher->ch,
     ghobject_t{hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard},
     st);
   return r;
@@ -1708,11 +1710,6 @@ int ECBackend::objects_get_attrs(
   const hobject_t &hoid,
   map<string, bufferlist, less<>> *out)
 {
-  // call from parents -- get raw attrs, without any filtering for hinfo
-  int r = PGBackend::objects_get_attrs(hoid, out);
-  if (r < 0)
-    return r;
-
   for (map<string, bufferlist>::iterator i = out->begin();
        i != out->end();
        ) {
@@ -1721,20 +1718,7 @@ int ECBackend::objects_get_attrs(
     else
       ++i;
   }
-  return r;
-}
-
-void ECBackend::rollback_append(
-  const hobject_t &hoid,
-  uint64_t old_size,
-  ObjectStore::Transaction *t)
-{
-  ceph_assert(old_size % sinfo.get_stripe_width() == 0);
-  t->truncate(
-    coll,
-    ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
-    sinfo.aligned_logical_offset_to_chunk_offset(
-      old_size));
+  return 0;
 }
 
 int ECBackend::be_deep_scrub(
@@ -1766,8 +1750,8 @@ int ECBackend::be_deep_scrub(
     stride += sinfo.get_chunk_size() - (stride % sinfo.get_chunk_size());
 
   bufferlist bl;
-  r = store->read(
-    ch,
+  r = switcher->store->read(
+    switcher->ch,
     ghobject_t(
       poid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard),
     pos.data_pos,