osd/scrub: introduce ScrubStore::at_level_t

to hold the caching and backend details related to the representation
of scrub-detected errors as OMap entries of a uniquely-named object.

In a followup commit - the ScrubStore is modified to hold two of
these objects, one for the shallow errors and one for the deep errors.

Signed-off-by: Ronen Friedman <[email protected]>

Author: ronen-fr

src/osd/scrubber/ScrubStore.cc

@@ -109,19 +109,29 @@ Store::create(ObjectStore* store,
   ceph_assert(t);
   ghobject_t oid = make_scrub_object(pgid);
   t->touch(coll, oid);
-  return new Store{coll, oid, store};
+  return new Store{*store, t, pgid, coll};
+}
+
+
+Store::Store(
+    ObjectStore& osd_store,
+    ObjectStore::Transaction* t,
+    const spg_t& pgid,
+    const coll_t& coll)
+    : object_store{osd_store}
+    , coll{coll}
+{
+  ceph_assert(t);
+
+  const auto err_obj = pgid.make_temp_ghobject(fmt::format("scrub_{}", pgid));
+  t->touch(coll, err_obj);
+  errors_db.emplace(pgid, err_obj, OSDriver{&object_store, coll, err_obj});
 }
 
-Store::Store(const coll_t& coll, const ghobject_t& oid, ObjectStore* store)
-  : coll(coll),
-    hoid(oid),
-    driver(store, coll, hoid),
-    backend(&driver)
-{}
 
 Store::~Store()
 {
-  ceph_assert(results.empty());
+  ceph_assert(!errors_db || errors_db->results.empty());
 }
 
 void Store::add_error(int64_t pool, const inconsistent_obj_wrapper& e)
@@ -131,11 +141,13 @@ void Store::add_error(int64_t pool, const inconsistent_obj_wrapper& e)
 
 void Store::add_object_error(int64_t pool, const inconsistent_obj_wrapper& e)
 {
+  const auto key = to_object_key(pool, e.object);
   bufferlist bl;
   e.encode(bl);
-  results[to_object_key(pool, e.object)] = bl;
+  errors_db->results[key] = bl;
 }
 
+
 void Store::add_error(int64_t pool, const inconsistent_snapset_wrapper& e)
 {
   add_snap_error(pool, e);
@@ -145,26 +157,28 @@ void Store::add_snap_error(int64_t pool, const inconsistent_snapset_wrapper& e)
 {
   bufferlist bl;
   e.encode(bl);
-  results[to_snap_key(pool, e.object)] = bl;
+  errors_db->results[to_snap_key(pool, e.object)] = bl;
 }
 
 bool Store::empty() const
 {
-  return results.empty();
+  return errors_db->results.empty();
 }
 
 void Store::flush(ObjectStore::Transaction* t)
 {
   if (t) {
-    OSDriver::OSTransaction txn = driver.get_transaction(t);
-    backend.set_keys(results, &txn);
+    OSDriver::OSTransaction txn = errors_db->driver.get_transaction(t);
+    errors_db->backend.set_keys(errors_db->results, &txn);
   }
-  results.clear();
+  errors_db->results.clear();
 }
 
 void Store::cleanup(ObjectStore::Transaction* t)
 {
-  t->remove(coll, hoid);
+  ceph_assert(t);
+  if (errors_db)
+    t->remove(coll, errors_db->errors_hoid);
 }
 
 std::vector<bufferlist>
@@ -195,8 +209,11 @@ Store::get_errors(const string& begin,
 		  uint64_t max_return) const
 {
   vector<bufferlist> errors;
+  if (!errors_db)
+    return errors;
+
   auto next = std::make_pair(begin, bufferlist{});
-  while (max_return && !backend.get_next(next.first, &next)) {
+  while (max_return && !errors_db->backend.get_next(next.first, &next)) {
     if (next.first >= end)
       break;
     errors.push_back(next.second);

src/osd/scrubber/ScrubStore.h

@@ -5,6 +5,7 @@
 #define CEPH_SCRUB_RESULT_H
 
 #include "common/map_cacher.hpp"
+#include "osd/osd_types_fmt.h"
 #include "osd/SnapMapper.h"  // for OSDriver
 
 namespace librados {
@@ -45,18 +46,56 @@ class Store {
     uint64_t max_return) const;
 
  private:
-  Store(const coll_t& coll, const ghobject_t& oid, ObjectStore* store);
+  /**
+   * at_level_t
+   *
+   * The machinery for caching and storing errors at a specific scrub level.
+   */
+  struct at_level_t {
+    at_level_t(const spg_t& pgid, const ghobject_t& err_obj, OSDriver&& drvr)
+	: errors_hoid{err_obj}
+	, driver{std::move(drvr)}
+	, backend{&driver}
+    {}
+
+    /// the object in the PG store, where the errors are stored
+    ghobject_t errors_hoid;
+
+    /// abstracted key fetching
+    OSDriver driver;
+
+    /// a K,V cache for the errors that are detected during the scrub
+    /// session. The errors marked for a specific object are stored as
+    /// an OMap entry with the object's name as the key.
+    MapCacher::MapCacher<std::string, ceph::buffer::list> backend;
+
+    /// a temp object mapping seq-id to inconsistencies
+    std::map<std::string, ceph::buffer::list> results;
+  };
+
+  Store(ObjectStore& osd_store,
+    ObjectStore::Transaction* t,
+    const spg_t& pgid,
+    const coll_t& coll);
+
   std::vector<ceph::buffer::list> get_errors(const std::string& start,
 					     const std::string& end,
 					     uint64_t max_return) const;
  private:
+  /// the OSD's storage backend
+  ObjectStore& object_store;
+
+  /// the collection (i.e. - the PG store) in which the errors are stored
   const coll_t coll;
-  const ghobject_t hoid;
-  // a temp object holding mappings from seq-id to inconsistencies found in
-  // scrubbing
-  OSDriver driver;
-  mutable MapCacher::MapCacher<std::string, ceph::buffer::list> backend;
-  std::map<std::string, ceph::buffer::list> results;
+
+  /**
+   * the machinery (backend details, cache, etc.) for storing both levels
+   * of errors (note: 'optional' to allow delayed creation w/o dynamic
+   * allocations; and 'mutable', as the caching mechanism is used in const
+   * methods)
+   */
+  mutable std::optional<at_level_t> errors_db;
+  // not yet: mutable std::optional<at_level_t> deep_db;
 };
 }  // namespace Scrub