osd: Move ec_inject interface in ECCommon into ECInject.cc

This is going to be common with the new EC code, so it makes sense to have it in a seperate file.  It also does not fit in with the class structure of either new or old code.

Signed-off-by: Alex Ainscow <[email protected]>

Author: aainscow

src/osd/CMakeLists.txt

@@ -47,6 +47,8 @@ set(osd_srcs
   ECTransactionL.cc
   ECUtilL.cc
   ECUtil.cc
+  ECInject.cc
+  ECInject.h
   ${CMAKE_SOURCE_DIR}/src/common/TrackedOp.cc
   ${CMAKE_SOURCE_DIR}/src/mgr/OSDPerfMetricTypes.cc
   ${osd_cyg_functions_src}

src/osd/ECBackendL.cc

@@ -16,6 +16,7 @@
 #include <sstream>
 
 #include "ECBackendL.h"
+#include "ECInject.h"
 #include "messages/MOSDPGPush.h"
 #include "messages/MOSDPGPushReply.h"
 #include "messages/MOSDECSubOpWrite.h"
@@ -947,7 +948,7 @@ void ECBackendL::handle_sub_write(
   trace.event("handle_sub_write");
 
   if (cct->_conf->bluestore_debug_inject_read_err &&
-      ec_inject_test_write_error3(op.soid)) {
+      ECInject::test_write_error3(op.soid)) {
     ceph_abort_msg("Error inject - OSD down");
   }
   if (!get_parent()->pgb_is_primary())
@@ -1197,7 +1198,7 @@ void ECBackendL::handle_sub_write_reply(
   }
   if (cct->_conf->bluestore_debug_inject_read_err &&
       (i->second->pending_commit.size() == 1) &&
-      ec_inject_test_write_error2(i->second->hoid)) {
+      ECInject::test_write_error2(i->second->hoid)) {
     std::string cmd =
       "{ \"prefix\": \"osd down\", \"ids\": [\"" + std::to_string( get_parent()->whoami() ) + "\"] }";
     vector<std::string> vcmd{cmd};
@@ -1225,7 +1226,7 @@ void ECBackendL::handle_sub_read_reply(
     for (auto i = op.buffers_read.begin();
 	 i != op.buffers_read.end();
 	 ++i) {
-      if (ec_inject_test_read_error0(ghobject_t(i->first, ghobject_t::NO_GEN, op.from.shard))) {
+      if (ECInject::test_read_error0(ghobject_t(i->first, ghobject_t::NO_GEN, op.from.shard))) {
 	dout(0) << __func__ << " Error inject - EIO error for shard " << op.from.shard << dendl;
 	op.buffers_read.erase(i->first);
 	op.attrs_read.erase(i->first);

src/osd/ECCommonL.cc

@@ -16,6 +16,7 @@
 #include <sstream>
 
 #include "ECCommonL.h"
+#include "ECInject.h"
 #include "messages/MOSDPGPush.h"
 #include "messages/MOSDPGPushReply.h"
 #include "messages/MOSDECSubOpWrite.h"
@@ -223,7 +224,7 @@ void ECCommonL::ReadPipeline::get_all_avail_shards(
       continue;
     }
     if (cct->_conf->bluestore_debug_inject_read_err &&
-        ec_inject_test_read_error1(ghobject_t(hoid, ghobject_t::NO_GEN, i->shard))) {
+        ECInject::test_read_error1(ghobject_t(hoid, ghobject_t::NO_GEN, i->shard))) {
       dout(0) << __func__ << " Error inject - Missing shard " << i->shard << dendl;
       continue;
     }
@@ -918,7 +919,7 @@ bool ECCommonL::RMWPipeline::try_reads_to_commit()
       should_write_local = true;
       local_write_op.claim(sop);
     } else if (cct->_conf->bluestore_debug_inject_read_err &&
-	       ec_inject_test_write_error1(ghobject_t(op->hoid,
+	       ECInject::test_write_error1(ghobject_t(op->hoid,
 		 ghobject_t::NO_GEN, i->shard))) {
       dout(0) << " Error inject - Dropping write message to shard " <<
 	i->shard << dendl;
@@ -1100,305 +1101,3 @@ ECUtilL::HashInfoRef ECCommonL::UnstableHashInfoRegistry::get_hash_info(
   }
   return ref;
 }
-
-// Error inject interfaces
-static ceph::recursive_mutex ec_inject_lock =
-  ceph::make_recursive_mutex("ECCommon::ec_inject_lock");
-static std::map<ghobject_t,std::pair<int64_t,int64_t>> ec_inject_read_failures0;
-static std::map<ghobject_t,std::pair<int64_t,int64_t>> ec_inject_read_failures1;
-static std::map<ghobject_t,std::pair<int64_t,int64_t>> ec_inject_write_failures0;
-static std::map<ghobject_t,std::pair<int64_t,int64_t>> ec_inject_write_failures1;
-static std::map<ghobject_t,std::pair<int64_t,int64_t>> ec_inject_write_failures2;
-static std::map<ghobject_t,std::pair<int64_t,int64_t>> ec_inject_write_failures3;
-static std::map<ghobject_t,shard_id_t> ec_inject_write_failures0_shard;
-static std::set<osd_reqid_t> ec_inject_write_failures0_reqid;
-
-/**
- * Configure a read error inject that typically forces additional reads of
- * shards in an EC pool to recover data using the redundancy. With multiple
- * errors it is possible to force client reads to fail.
- *
- * Type 0 - Simulate a medium error. Fail a read with -EIO to force
- * additional reads and a decode
- *
- * Type 1 - Simulate a missing OSD. Dont even try to read a shard
- *
- * @brief Set up a read error inject for an object in an EC pool.
- * @param o Target object for the error inject.
- * @param when Error inject starts after this many object store reads.
- * @param duration Error inject affects this many object store reads.
- * @param type Type of error inject 0 = EIO, 1 = missing shard.
- * @return string Result of configuring the error inject.
- */
-std::string ec_inject_read_error(const ghobject_t& o,
-				 const int64_t type,
-				 const int64_t when,
-				 const int64_t duration) {
-  std::lock_guard<ceph::recursive_mutex> l(ec_inject_lock);
-  ghobject_t os = o;
-  if (os.hobj.oid.name == "*") {
-    os.hobj.set_hash(0);
-  }
-  switch (type) {
-  case 0:
-    ec_inject_read_failures0[os] = std::pair(when, duration);
-    return "ok - read returns EIO";
-  case 1:
-    ec_inject_read_failures1[os] = std::pair(when, duration);
-    return "ok - read pretends shard is missing";
-  default:
-    break;
-  }
-  return "unrecognized error inject type";
-}
-
-/**
- * Configure a write error inject that either fails an OSD or causes a
- * client write operation to be rolled back.
- *
- * Type 0 - Tests rollback. Drop a write I/O to a shard, then simulate an OSD
- * down to force rollback to occur, lastly fail the retried write from the
- * client so the results of the rollback can be inspected.
- *
- * Type 1 - Drop a write I/O to a shard. Used on its own this will hang a
- * write I/O.
- *
- * Type 2 - Simulate an OSD down (ceph osd down) to force a new epoch. Usually
- * used together with type 1 to force a rollback
- *
- * Type 3 - Abort when an OSD processes a write I/O to a shard. Typically the
- * client write will be commited while the OSD is absent which will result in
- * recovery or backfill later when the OSD returns.
- *
- * @brief Set up a write error inject for an object in an EC pool.
- * @param o Target object for the error inject.
- * @param when Error inject starts after this many object store reads.
- * @param duration Error inject affects this many object store reads.
- * @param type Type of error inject 0 = EIO, 1 = missing shard.
- * @return string Result of configuring the error inect.
- */
-std::string ec_inject_write_error(const ghobject_t& o,
-				  const int64_t type,
-				  const int64_t when,
-				  const int64_t duration) {
-  std::lock_guard<ceph::recursive_mutex> l(ec_inject_lock);
-  std::map<ghobject_t,std::pair<int64_t,int64_t>> *failures;
-  ghobject_t os = o;
-  bool no_shard = true;
-  std::string result;
-  switch (type) {
-  case 0:
-    failures = &ec_inject_write_failures0;
-    result = "ok - drop write, sim OSD down and fail client retry with EINVAL";
-    break;
-  case 1:
-    failures = &ec_inject_write_failures1;
-    no_shard = false;
-    result = "ok - drop write to shard";
-    break;
-  case 2:
-    failures = &ec_inject_write_failures2;
-    result = "ok - inject OSD down";
-    break;
-  case 3:
-    if (duration != 1) {
-      return "duration must be 1";
-    }
-    failures = &ec_inject_write_failures3;
-    result = "ok - write abort OSDs";
-    break;
-  default:
-    return "unrecognized error inject type";
-  }
-  if (no_shard) {
-    os.set_shard(shard_id_t::NO_SHARD);
-  }
-  if (os.hobj.oid.name == "*") {
-    os.hobj.set_hash(0);
-  }
-  (*failures)[os] = std::pair(when, duration);
-  if (type == 0) {
-    ec_inject_write_failures0_shard[os] = o.shard_id;
-  }
-  return result;
-}
-
-/**
- * @brief Clear a previously configured read error inject.
- * @param o Target object for the error inject.
- * @param type Type of error inject 0 = EIO, 1 = missing shard.
- * @return string Indication of how many errors were cleared.
- */
-std::string ec_inject_clear_read_error(const ghobject_t& o,
-				       const int64_t type) {
-  std::lock_guard<ceph::recursive_mutex> l(ec_inject_lock);
-  std::map<ghobject_t,std::pair<int64_t,int64_t>> *failures;
-  ghobject_t os = o;
-  int64_t remaining = 0;
-  switch (type) {
-  case 0:
-    failures = &ec_inject_read_failures0;
-    break;
-  case 1:
-    failures = &ec_inject_read_failures1;
-    break;
-  default:
-    return "unrecognized error inject type";
-  }
-  if (os.hobj.oid.name == "*") {
-    os.hobj.set_hash(0);
-  }
-  auto it = failures->find(os);
-  if (it != failures->end()) {
-    remaining = it->second.second;
-    failures->erase(it);
-  }
-  if (remaining == 0) {
-    return "no outstanding error injects";
-  } else if (remaining == 1) {
-    return "ok - 1 inject cleared";
-  }
-  return "ok - " + std::to_string(remaining) + " injects cleared";
-}
-
-/**
- * @brief Clear a previously configured write error inject.
- * @param o Target object for the error inject.
- * @param type Type of error inject 0 = EIO, 1 = missing shard.
- * @return string Indication of how many errors were cleared.
- */
-std::string ec_inject_clear_write_error(const ghobject_t& o,
-					const int64_t type) {
-  std::lock_guard<ceph::recursive_mutex> l(ec_inject_lock);
-  std::map<ghobject_t,std::pair<int64_t,int64_t>> *failures;
-  ghobject_t os = o;
-  bool no_shard = true;
-  int64_t remaining = 0;
-  switch (type) {
-  case 0:
-    failures = &ec_inject_write_failures0;
-    break;
-  case 1:
-    failures = &ec_inject_write_failures1;
-    no_shard = false;
-    break;
-  case 2:
-    failures = &ec_inject_write_failures2;
-    break;
-  case 3:
-    failures = &ec_inject_write_failures3;
-    break;
-  default:
-    return "unrecognized error inject type";
-  }
-  if (no_shard) {
-    os.set_shard(shard_id_t::NO_SHARD);
-  }
-  if (os.hobj.oid.name == "*") {
-    os.hobj.set_hash(0);
-  }
-  auto it = failures->find(os);
-  if (it != failures->end()) {
-    remaining = it->second.second;
-    failures->erase(it);
-    if (type == 0) {
-      ec_inject_write_failures0_shard.erase(os);
-    }
-  }
-  if (remaining == 0) {
-    return "no outstanding error injects";
-  } else if (remaining == 1) {
-    return "ok - 1 inject cleared";
-  }
-  return "ok - " + std::to_string(remaining) + " injects cleared";
-}
-
-static bool ec_inject_test_error(const ghobject_t& o,
-  std::map<ghobject_t,std::pair<int64_t,int64_t>> *failures)
-{
-  std::lock_guard<ceph::recursive_mutex> l(ec_inject_lock);
-  auto it = failures->find(o);
-  if (it == failures->end()) {
-    ghobject_t os = o;
-    os.hobj.oid.name = "*";
-    os.hobj.set_hash(0);
-    it = failures->find(os);
-  }
-  if (it != failures->end()) {
-    auto && [when,duration] = it->second;
-    if (when > 0) {
-      when--;
-      return false;
-    }
-    if (--duration <= 0) {
-      failures->erase(it);
-    }
-    return true;
-  }
-  return false;
-}
-
-bool ec_inject_test_read_error0(const ghobject_t& o)
-{
-  return ec_inject_test_error(o, &ec_inject_read_failures0);
-}
-
-bool ec_inject_test_read_error1(const ghobject_t& o)
-{
-  return ec_inject_test_error(o, &ec_inject_read_failures1);
-}
-
-bool ec_inject_test_write_error0(const hobject_t& o,
-				 const osd_reqid_t& reqid) {
-  std::lock_guard<ceph::recursive_mutex> l(ec_inject_lock);
-  ghobject_t os = ghobject_t(o, ghobject_t::NO_GEN, shard_id_t::NO_SHARD);
-  if (ec_inject_write_failures0_reqid.count(reqid)) {
-    // Matched reqid of retried write - flag for failure
-    ec_inject_write_failures0_reqid.erase(reqid);
-    return true;
-  }
-  auto it = ec_inject_write_failures0.find(os);
-  if (it == ec_inject_write_failures0.end()) {
-    os.hobj.oid.name = "*";
-    os.hobj.set_hash(0);
-    it = ec_inject_write_failures0.find(os);
-  }
-  if (it != ec_inject_write_failures0.end()) {
-    auto && [when, duration] = it->second;
-    auto shard = ec_inject_write_failures0_shard.find(os)->second;
-    if (when > 0) {
-      when--;
-    } else {
-      if (--duration <= 0) {
-	ec_inject_write_failures0.erase(it);
-	ec_inject_write_failures0_shard.erase(os);
-      }
-      // Error inject triggered - save reqid
-      ec_inject_write_failures0_reqid.insert(reqid);
-      // Set up error inject to drop message to primary
-      ec_inject_write_error(ghobject_t(o, ghobject_t::NO_GEN, shard), 1, 0, 1);
-    }
-  }
-  return false;
-}
-
-bool ec_inject_test_write_error1(const ghobject_t& o) {
-  bool rc = ec_inject_test_error(o, &ec_inject_write_failures1);
-  if (rc) {
-    // Set up error inject to generate OSD down
-    ec_inject_write_error(o, 2, 0, 1);
-  }
-  return rc;
-}
-
-bool ec_inject_test_write_error2(const hobject_t& o) {
-  return ec_inject_test_error(
-    ghobject_t(o, ghobject_t::NO_GEN, shard_id_t::NO_SHARD),
-    &ec_inject_write_failures2);
-}
-
-bool ec_inject_test_write_error3(const hobject_t& o) {
-  return ec_inject_test_error(
-    ghobject_t(o, ghobject_t::NO_GEN, shard_id_t::NO_SHARD),
-    &ec_inject_write_failures3);
-}

src/osd/ECCommonL.h

@@ -683,15 +683,3 @@ void ECCommonL::ReadPipeline::filter_read_op(
     on_schedule_recovery(op);
   }
 }
-
-// Error inject interfaces
-std::string ec_inject_read_error(const ghobject_t& o, const int64_t type, const int64_t when, const int64_t duration);
-std::string ec_inject_write_error(const ghobject_t& o, const int64_t type, const int64_t when, const int64_t duration);
-std::string ec_inject_clear_read_error(const ghobject_t& o, const int64_t type);
-std::string ec_inject_clear_write_error(const ghobject_t& o, const int64_t type);
-bool ec_inject_test_read_error0(const ghobject_t& o);
-bool ec_inject_test_read_error1(const ghobject_t& o);
-bool ec_inject_test_write_error0(const hobject_t& o,const osd_reqid_t& reqid);
-bool ec_inject_test_write_error1(const ghobject_t& o);
-bool ec_inject_test_write_error2(const hobject_t& o);
-bool ec_inject_test_write_error3(const hobject_t& o);