test/common: rados io sequencer exerciser extensions

1. Add miscompare message containing object ID.
2. Fix compiler warning due to strangely placed assert.
3. Add barriers following error injects to insure they
   are in place before IO.
4. Do not, by default, test EC profiles that are not
   known to be good for EC optimisations.
5. Add "allow_unstable_pool_configs" to override above.

Signed-off-by: Alex Ainscow <[email protected]>

Author: aainscow

src/common/io_exerciser/DataGenerator.cc

@@ -226,6 +226,8 @@ bool HeaderedSeededRandomGenerator::validate(bufferlist& bufferlist,
   }
 
   if (!invalid_block_offsets.empty()) {
+    dout(0) << "Miscompare for read of " << m_model.get_oid() <<
+      " offset=" << offset << " length=" << length << dendl;
     printDebugInformationForOffsets(offset, invalid_block_offsets, bufferlist);
   }
 

src/common/io_exerciser/EcIoSequence.cc

@@ -207,8 +207,8 @@ std::unique_ptr<IoOp> ReadInjectSequence::next() {
   switch (child_op->getOpType()) {
     case OpType::Remove:
       next_op.swap(child_op);
+      ceph_assert(shard_to_inject.has_value());
       switch (inject_op_type) {
-        ceph_assert(shard_to_inject.has_value());
         case InjectOpType::ReadEIO:
           return ClearReadErrorInjectOp::generate(*shard_to_inject, 0);
         case InjectOpType::ReadMissingShard:
@@ -303,6 +303,7 @@ std::string ceph::io_exerciser::Seq10::get_name() const {
 std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq10::_next() {
   if (!inject_error_done) {
     inject_error_done = true;
+    barrier = true;
     return InjectWriteErrorOp::generate(*shard_to_inject, 0, 0,
                                         std::numeric_limits<uint64_t>::max());
   } else if (!failed_write_done) {
@@ -316,6 +317,7 @@ std::unique_ptr<ceph::io_exerciser::IoOp> ceph::io_exerciser::Seq10::_next() {
     return SingleReadOp::generate(offset, length);
   } else if (!clear_inject_done) {
     clear_inject_done = true;
+    barrier = true;
     return ClearWriteErrorInjectOp::generate(*shard_to_inject, 0);
   } else if (!successful_write_done) {
     successful_write_done = true;

src/test/osd/ceph_test_rados_io_sequence/ProgramOptionReader.h

@@ -115,6 +115,53 @@ class ProgramOptionSelector : public ProgramOptionReader<option_type> {
   std::optional<option_type> first_value;
 };
 
+template <typename option_type,
+          int num_selections,
+          const std::array< option_type,
+                            num_selections>& selections_array,
+          int num_selections_stable,
+          const std::array< option_type,
+                            num_selections_stable>& elections_array_stable>
+class StableOptionSelector : public ProgramOptionReader<option_type> {
+public:
+  StableOptionSelector(ceph::util::random_number_generator<int>& rng,
+                        po::variables_map& vm,
+                        const std::string& option_name,
+                        bool select_first)
+      : ProgramOptionReader<option_type>(vm, option_name), rng(rng),
+        stable(!vm.contains("allow_unstable_pool_configs") ||
+          vm.contains("disable_pool_ec_optimizations")) {
+    if (select_first) {
+      if (stable) {
+        ceph_assert(selections_array.size() > 0);
+        first_value = elections_array_stable[0];
+      } else {
+        ceph_assert(selections_array.size() > 0);
+        first_value = selections_array[0];
+      }
+    }
+  }
+
+  virtual ~StableOptionSelector() = default;
+
+  virtual const option_type select() override {
+    if (this->force_value.has_value()) {
+      return *this->force_value;
+    } else if (first_value.has_value()) {
+      return *std::exchange(first_value, std::nullopt);
+    } else if (stable) {
+      return elections_array_stable[rng(num_selections_stable - 1)];
+    } else {
+      return selections_array[rng(num_selections - 1)];
+    }
+  }
+
+protected:
+  ceph::util::random_number_generator<int>& rng;
+  std::optional<option_type> first_value;
+  bool stable;
+};
+
 template <typename option_type>
 class ProgramOptionGeneratedSelector
     : public OptionalProgramOptionReader<option_type> {

src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.cc

@@ -184,7 +184,12 @@ po::options_description get_options_description() {
       "allow_pool_balancer", "Enables pool balancing. Disabled by default.")(
       "allow_pool_deep_scrubbing",
       "Enables pool deep scrub. Disabled by default.")(
-      "allow_pool_scrubbing", "Enables pool scrubbing. Disabled by default.");
+      "allow_pool_scrubbing", "Enables pool scrubbing. Disabled by default.")(
+      "disable_pool_ec_optimizations",
+      "Disables EC optimizations. Enabled by default.")(
+      "allow_unstable_pool_configs",
+      "Permits pool configs that are known to be unstable. This option "
+      " may be removed. at a later date. Disabled by default if ec optimized");
 
   return desc;
 }
@@ -272,22 +277,28 @@ ceph::io_sequence::tester::SelectErasureTechnique::SelectErasureTechnique(
     : ProgramOptionGeneratedSelector<std::string>(rng, vm, "technique",
                                                   first_use),
       rng(rng),
-      plugin(plugin) {}
+      plugin(plugin),
+      stable(!vm.contains("allow_unstable_pool_configs") ||
+        vm.contains("disable_pool_ec_optimizations")) {}
 
 const std::vector<std::string>
 ceph::io_sequence::tester::SelectErasureTechnique::generate_selections() {
   std::vector<std::string> techniques = {};
   if (plugin == "jerasure") {
     techniques.push_back("reed_sol_van");
-    techniques.push_back("reed_sol_r6_op");
-    techniques.push_back("cauchy_orig");
-    techniques.push_back("cauchy_good");
-    techniques.push_back("liberation");
-    techniques.push_back("blaum_roth");
-    techniques.push_back("liber8tion");
+    if (!stable) {
+      techniques.push_back("reed_sol_r6_op");
+      techniques.push_back("cauchy_orig");
+      techniques.push_back("cauchy_good");
+      techniques.push_back("liberation");
+      techniques.push_back("blaum_roth");
+      techniques.push_back("liber8tion");
+    }
   } else if (plugin == "isa") {
     techniques.push_back("reed_sol_van");
-    techniques.push_back("cauchy");
+    if (!stable) {
+      techniques.push_back("cauchy");
+    }
   } else if (plugin == "shec") {
     techniques.push_back("single");
     techniques.push_back("multiple");
@@ -337,28 +348,24 @@ ceph::io_sequence::tester::SelectErasureKM::generate_selections() {
        (technique == "reed_sol_van" || technique == "cauchy_orig" ||
         technique == "cauchy_good" || technique == std::nullopt))) {
     for (int m = 1; m <= 3; m++)
-      for (int k = 2; k <= 6; k++) selection.push_back({k, m});
+      for (int k = 2; k <= 4; k++) selection.push_back({k, m});
   } else if (plugin == "shec" ||
              (plugin == "jerasure" &&
               (technique == "liberation" || technique == "blaum_roth"))) {
     for (int m = 1; m <= 2; m++)
-      for (int k = 2; k <= 6; k++) selection.push_back({k, m});
+      for (int k = 2; k <= 4; k++) selection.push_back({k, m});
   } else if (plugin == "jerasure" &&
              (technique == "reed_sol_r6_op" || technique == "liber8tion")) {
-    for (int k = 2; k <= 6; k++) selection.push_back({k, 2});
+    for (int k = 2; k <= 4; k++) selection.push_back({k, 2});
   }
 
   // We want increased chances of these as we will test with c=1 and c=2
   if (plugin == "shec")
     for (int i = 0; i < 2; i++)
-      for (int k = 3; k <= 6; k++) selection.push_back({k, 3});
+      for (int k = 3; k <= 4; k++) selection.push_back({k, 3});
 
   // Add extra miscelaneous interesting options for testing w values
   if (plugin == "jerasure") {
-    if (technique == "reed_sol_van")
-      // Double chance of chosing to test more w values
-      for (int i = 0; i < 2; i++) selection.push_back({6, 3});
-
     if (technique == "liberation" || technique == "blaum_roth")
       // Double chance of chosing to test more different w values
       for (int i = 0; i < 2; i++) selection.push_back({6, 2});

src/test/osd/ceph_test_rados_io_sequence/ceph_test_rados_io_sequence.h

@@ -108,10 +108,19 @@ inline static constexpr std::array<uint64_t, block_size_array_size> block_size_c
     {2048,  // Default - test boundaries for EC 4K chunk size
      512, 3767, 4096, 32768}};
 
+// Choices for block size
+inline static constexpr int block_size_array_size_stable = 2;
+inline static constexpr std::array<uint64_t, block_size_array_size_stable> block_size_choices_stable = {
+  {2048,  // Default - test boundaries for EC 4K chunk size
+   32768}};
+
+
 using SelectBlockSize =
-    ProgramOptionSelector<uint64_t,
-                          io_sequence::tester::block_size_array_size,
-                          io_sequence::tester::block_size_choices>;
+  StableOptionSelector<uint64_t,
+                        io_sequence::tester::block_size_array_size,
+                        io_sequence::tester::block_size_choices,
+                        io_sequence::tester::block_size_array_size_stable,
+                        io_sequence::tester::block_size_choices_stable>;
 
 // Choices for number of threads
 inline static constexpr int thread_array_size = 4;
@@ -138,10 +147,16 @@ inline static constexpr int plugin_array_size = 5;
 inline static constexpr std::array<std::string_view, plugin_array_size>
     plugin_choices = {{"jerasure", "isa", "clay", "shec", "lrc"}};
 
+inline static constexpr int plugin_array_size_stable = 2;
+inline static constexpr std::array<std::string_view, plugin_array_size_stable>
+    plugin_choices_stable = {{"jerasure", "isa"}};
+
 using SelectErasurePlugin =
-    ProgramOptionSelector<std::string_view,
+    StableOptionSelector<std::string_view,
                           io_sequence::tester::plugin_array_size,
-                          io_sequence::tester::plugin_choices>;
+                          io_sequence::tester::plugin_choices,
+                          io_sequence::tester::plugin_array_size_stable,
+                          io_sequence::tester::plugin_choices_stable>;
 
 class SelectErasureKM
     : public ProgramOptionGeneratedSelector<std::pair<int, int>> {
@@ -306,6 +321,7 @@ class SelectErasureTechnique
   ceph::util::random_number_generator<int>& rng;
 
   std::string_view plugin;
+  bool stable;
 };
 
 class SelectErasureChunkSize : public ProgramOptionGeneratedSelector<uint64_t> {