Adds a new command-line utility which can check the consistency of objects within an erasure coded pool.

A new test-only inject tells the EC backend to return both data and parity shards to the client so that they can
be checked for consistency by the new tool.

Supports both optimized and unoptimized EC pools.

Signed-off-by: Connor Fawcett <[email protected]>

Author: Connor Fawcett

src/common/json/OSDStructures.cc

@@ -61,14 +61,16 @@ void OSDPoolGetRequest::decode_json(JSONObj* obj) {
 
 void OSDPoolGetReply::dump(Formatter* f) const {
   encode_json("erasure_code_profile", erasure_code_profile, f);
+  encode_json("allow_ec_optimizations", allow_ec_optimizations, f);
 }
 
 void OSDPoolGetReply::decode_json(JSONObj* obj) {
   JSONDecoder::decode_json("erasure_code_profile", erasure_code_profile, obj);
+  JSONDecoder::decode_json("allow_ec_optimizations", allow_ec_optimizations, obj);
 }
 
 void OSDECProfileGetRequest::dump(Formatter* f) const {
-  encode_json("prefix", "osd pool get", f);
+  encode_json("prefix", "osd erasure-code-profile get", f);
   encode_json("name", name, f);
   encode_json("format", format, f);
 }
@@ -159,4 +161,26 @@ void OSDSetRequest::dump(Formatter* f) const {
 void OSDSetRequest::decode_json(JSONObj* obj) {
   JSONDecoder::decode_json("key", key, obj);
   JSONDecoder::decode_json("yes_i_really_mean_it", yes_i_really_mean_it, obj);
+}
+
+void InjectECParityRead::dump(Formatter* f) const {
+  encode_json("prefix", "injectparityread", f);
+  encode_json("pool", pool, f);
+  encode_json("objname", objname, f);
+}
+
+void InjectECParityRead::decode_json(JSONObj* obj) {
+  JSONDecoder::decode_json("pool", pool, obj);
+  JSONDecoder::decode_json("objname", objname, obj);
+}
+
+void InjectECClearParityRead::dump(Formatter* f) const {
+  encode_json("prefix", "injectclearparityread", f);
+  encode_json("pool", pool, f);
+  encode_json("objname", objname, f);
+}
+
+void InjectECClearParityRead::decode_json(JSONObj* obj) {
+  JSONDecoder::decode_json("pool", pool, obj);
+  JSONDecoder::decode_json("objname", objname, obj);
 }

src/common/json/OSDStructures.h

@@ -50,7 +50,7 @@ struct OSDPoolGetRequest {
 
 struct OSDPoolGetReply {
   std::string erasure_code_profile;
-
+  bool allow_ec_optimizations;
   void dump(Formatter* f) const;
   void decode_json(JSONObj* obj);
 };
@@ -190,6 +190,20 @@ struct InjectECClearErrorRequest {
     JSONDecoder::decode_json("type", type, obj);
   }
 };
+struct InjectECParityRead {
+  std::string pool;
+  std::string objname;
+
+  void dump(Formatter* f) const;
+  void decode_json(JSONObj* obj);
+};
+struct InjectECClearParityRead {
+  std::string pool;
+  std::string objname;
+
+  void dump(Formatter* f) const;
+  void decode_json(JSONObj* obj);
+};
 }  // namespace osd
 }  // namespace messaging
 }  // namespace ceph

src/erasure-code/CMakeLists.txt

@@ -21,6 +21,7 @@ add_subdirectory(jerasure)
 add_subdirectory(lrc)
 add_subdirectory(shec)
 add_subdirectory(clay)
+add_subdirectory(consistency)
 
 if(HAVE_NASM_X64_AVX2 OR HAVE_ARMV8_SIMD)
   set(WITH_EC_ISA_PLUGIN TRUE CACHE BOOL "")

src/erasure-code/consistency/CMakeLists.txt

@@ -0,0 +1,26 @@
+add_library(ec_consistency STATIC
+  ECReader.cc
+  ECEncoder.cc
+  ECEncoderSwitch.cc
+  Pool.cc
+  ConsistencyChecker.cc
+  RadosCommands.cc
+  ${PROJECT_SOURCE_DIR}/src/osd/ECUtilL.cc
+  ${PROJECT_SOURCE_DIR}/src/osd/ECUtil.cc
+)
+
+target_link_libraries(ec_consistency
+  librados
+  global
+  json_structures
+)
+
+add_executable(ceph_ec_consistency_checker
+  ${CMAKE_CURRENT_SOURCE_DIR}/ceph_ec_consistency_checker.cc
+  ${PROJECT_SOURCE_DIR}/src/osd/ECUtilL.cc
+  ${PROJECT_SOURCE_DIR}/src/osd/ECUtil.cc)
+target_link_libraries(ceph_ec_consistency_checker
+  librados global ec_consistency)
+install(TARGETS
+ceph_ec_consistency_checker
+  DESTINATION ${CMAKE_INSTALL_BINDIR})

src/erasure-code/consistency/ConsistencyChecker.cc

@@ -0,0 +1,169 @@
+#include "ConsistencyChecker.h"
+
+#include "RadosCommands.h"
+#include "Pool.h"
+#include "ECReader.h"
+#include "ECEncoder.h"
+#include "ECEncoderSwitch.h"
+
+using ConsistencyChecker = ceph::consistency::ConsistencyChecker;
+
+using Read = ceph::consistency::Read;
+using ReadResult = ceph::consistency::ReadResult;
+using bufferlist = ceph::bufferlist;
+
+ConsistencyChecker::ConsistencyChecker(librados::Rados &rados,
+                                       boost::asio::io_context& asio,
+                                       const std::string& pool_name,
+                                       int stripe_unit) :
+  rados(rados),
+  asio(asio),
+  reader(ceph::consistency::ECReader(rados, asio, pool_name)),
+  commands(ceph::consistency::RadosCommands(rados)),
+  pool(pool_name,
+       commands.get_ec_profile_for_pool(pool_name),
+       commands.get_pool_allow_ec_optimizations(pool_name)),
+  encoder(ceph::consistency::ECEncoderSwitch(pool.get_ec_profile(),
+                                             stripe_unit,
+                                             commands.get_pool_allow_ec_optimizations(pool_name)
+                                            )) {}
+
+/**
+ * Perform an end-to-end read and consistency check on a single object.
+ * Current implementation only supports reading the entire object, so length and
+ * offset should normally be 0.
+ *
+ * @param oid string Name of the pool to perform inject on
+ * @param block_size int Block size for the data being read
+ * @param offset int Which offset to read from
+ * @param length int How much data of each shard to read
+ * @return bool true if consistent, otherwise false
+ */
+bool ConsistencyChecker::single_read_and_check_consistency(const std::string& oid,
+                                                           int block_size,
+                                                           int offset,
+                                                           int length)
+{
+  clear_results();
+  std::string error_message = "";
+  bool success = true;
+
+  auto read = Read(oid, block_size, offset, length);
+  queue_ec_read(read);
+
+  auto read_results = reader.get_results();
+  int result_count = read_results->size();
+  if (result_count != 1) {
+    error_message = "Incorrect number of RADOS read results returned, count: "
+                    + std::to_string(result_count);
+    success = false;
+  }
+
+  ReadResult read_result = (*read_results)[0];
+  boost::system::error_code ec = read_result.get_ec();
+  if (success && ec != boost::system::errc::success) {
+    error_message = "RADOS Read failed, error message: " + ec.message();
+    success = false;
+  }
+
+  if (success && read_result.get_data().length() == 0) {
+    error_message = "Empty object returned from RADOS read.";
+    success = false;
+  }
+
+  if (success && !check_object_consistency(oid, read_result.get_data())) {
+    error_message = "Generated parity did not match read in parity shards.";
+    success = false;
+  }
+
+  results.push_back({oid, error_message, success});
+  commands.inject_clear_parity_read_on_primary_osd(pool.get_pool_name(),
+                                                   oid);
+  return success;
+}
+
+/**
+ * Queue up an EC read with the parity read inject set
+ *
+ * @param read Object containing information about the read
+ */
+void ConsistencyChecker::queue_ec_read(Read read)
+{
+  commands.inject_parity_read_on_primary_osd(pool.get_pool_name(),
+                                             read.get_oid());
+  reader.do_read(read);
+}
+
+/**
+ * Generate parities from the data and compare to the parity shards
+ *
+ * @param oid string The object ID of the object being checked
+ * @param inbl bufferlist The entire contents of the object, including parities
+ * @param stripe_unit int The chunk size for the object
+ */
+bool ConsistencyChecker::check_object_consistency(const std::string& oid,
+                                                  const bufferlist& inbl)
+{
+  bool is_optimized = pool.has_optimizations_enabled();
+  std::pair<bufferlist, bufferlist> data_and_parity;
+  data_and_parity = split_data_and_parity(oid, inbl, encoder.get_k(), 
+                                          encoder.get_m(), is_optimized);
+
+  std::optional<bufferlist> outbl;
+  outbl = encoder.do_encode(data_and_parity.first);
+
+  if (!outbl.has_value()) {
+    return false;
+  }
+
+  return buffers_match(outbl.value(), data_and_parity.second);
+}
+
+void ConsistencyChecker::print_results(std::ostream& out)
+{
+  out << "Results:" << std::endl;
+  for (const auto &r : results) {
+    std::string result_str = (r.get_result()) ? "Passed" : "Failed";
+    std::string error_str = r.get_error_message();
+    out << "Object ID " << r.get_oid() << ": " << result_str << std::endl;
+    if (!error_str.empty()) {
+      out << "Error Message: " << error_str << std::endl;
+    }
+  }
+
+  int count = results.size();
+  std::string obj_str = (count == 1) ? "object checked." : "objects checked.";
+  out << "Total: " << count << " " << obj_str << std::endl;
+}
+
+std::pair<bufferlist, bufferlist>
+  ConsistencyChecker::split_data_and_parity(const std::string& oid,
+                                            const bufferlist& read,
+                                            int k, int m,
+                                            bool is_optimized)
+{
+  uint64_t data_size, parity_size;
+
+  // Optimized EC parity read should return the exact object size + parity shards
+  // Legacy EC parity read will return the entire padded data shards + parity shards
+  data_size = is_optimized ? reader.get_object_size(oid) : (read.length() / (k + m)) * k;
+  parity_size = read.length() - data_size;
+
+  bufferlist data, parity;
+  auto it = read.begin();
+  it.copy(data_size, data);
+  it.copy(parity_size, parity);
+  return std::pair<bufferlist, bufferlist>(data, parity);
+}
+
+bool ConsistencyChecker::buffers_match(const bufferlist& b1,
+                                       const bufferlist& b2)
+{
+  return (b1.contents_equal(b2));
+}
+
+void ConsistencyChecker::clear_results()
+{
+  reader.clear_results();
+  results.clear();
+}

src/erasure-code/consistency/ConsistencyChecker.h

@@ -0,0 +1,68 @@
+#pragma once
+
+#include <boost/asio/io_context.hpp>
+#include <boost/program_options.hpp>
+#include "librados/librados_asio.h"
+#include "global/global_init.h"
+#include "global/global_context.h"
+
+#include "Pool.h"
+#include "ECReader.h"
+#include "RadosCommands.h"
+#include "ECEncoder.h"
+#include "ECEncoderSwitch.h"
+
+#define dout_context g_ceph_context
+
+namespace ceph {
+namespace consistency {
+class ConsistencyCheckResult {
+  private:
+    std::string oid;
+    std::string error_message;
+    bool result;
+
+  public:
+    std::string get_oid() const { return oid; }
+    std::string get_error_message() const { return error_message; }
+    bool get_result() const { return result; }
+    ConsistencyCheckResult(std::string oid,
+                           std::string error_message,
+                           bool result) :
+      oid(oid),
+      error_message(error_message),
+      result(result) {}
+};
+
+class ConsistencyChecker {
+  private:
+    librados::Rados& rados;
+    boost::asio::io_context& asio;
+    ceph::consistency::ECReader reader;
+    ceph::consistency::RadosCommands commands;
+    ceph::consistency::Pool pool;
+    ceph::consistency::ECEncoderSwitch encoder;
+    std::vector<ConsistencyCheckResult> results;
+    bool buffers_match(const bufferlist& b1, const bufferlist& b2);
+    std::pair<bufferlist, bufferlist> split_data_and_parity(const std::string& oid,
+                                                            const bufferlist& read,
+                                                            int k, int m,
+                                                            bool is_optimized);
+
+  public:
+    ConsistencyChecker(librados::Rados& rados,
+                        boost::asio::io_context& asio,
+                        const std::string& pool_name,
+                        int stripe_unit);
+    void queue_ec_read(Read read);
+    bool check_object_consistency(const std::string& oid,
+                                  const bufferlist& inbl);
+    void print_results(std::ostream& out);
+    void clear_results();
+    bool single_read_and_check_consistency(const std::string& oid,
+                                           int block_size,
+                                           int offset,
+                                           int length);
+};
+}
+}