os/bluestore: introduce bluestore_debug_enforce_min_alloc_size config

parameter.

This allows to override persistent min_alloc_size if needed.
This might be helpful to troubleshoot and work around issues like
https://tracker.ceph.com/issues/63618

Signed-off-by: Igor Fedotov <[email protected]>

Author: ifed01

src/common/options/global.yaml.in

@@ -4406,6 +4406,21 @@ options:
   flags:
   - create
   with_legacy: true
+- name: bluestore_debug_enforce_min_alloc_size
+  type: uint
+  level: dev
+  desc: Enforces specific min_alloc size usages
+  long_desc: This overrides actual min_alloc_size value persisted on mkfs
+    (and originally obtained from bluestore_min_alloc_size) and permits to
+    use arbitrary value for this value. Intended primarily for dev/debug
+    purposes and should be used with care and deep understanding of potential
+    consequences, e.g. data corruption.
+  default: 0
+  see_also:
+  - bluestore_min_alloc_size
+  flags:
+  - startup
+  with_legacy: true
 - name: bluestore_use_optimal_io_size_for_min_alloc_size 
   type: bool
   level: advanced

src/os/bluestore/BitmapFreelistManager.cc

@@ -610,3 +610,19 @@ void BitmapFreelistManager::get_meta(
   res->emplace_back("bfm_bytes_per_block", stringify(bytes_per_block));
   res->emplace_back("bfm_blocks_per_key", stringify(blocks_per_key));
 }
+
+bool BitmapFreelistManager::validate(uint64_t min_alloc_size) const
+{
+  bool ret = true;
+  auto my_alloc_size = get_alloc_size();
+  ceph_assert(my_alloc_size);
+  ceph_assert(min_alloc_size);
+  if (!is_null_manager() &&
+       ((min_alloc_size < my_alloc_size) || (min_alloc_size % my_alloc_size))) {
+    derr << __func__ << " inconsistent alloc units:" << std::hex
+         << "0x" << get_alloc_size() << " vs. 0x" << min_alloc_size
+         << std::dec << dendl;
+    ret = false;
+  }
+  return ret;
+}

src/os/bluestore/BitmapFreelistManager.h

@@ -94,6 +94,8 @@ class BitmapFreelistManager : public FreelistManager {
   }
   void get_meta(uint64_t target_size,
     std::vector<std::pair<std::string, std::string>>*) const override;
+
+  bool validate(uint64_t min_alloc_size) const override;
 };
 
 #endif

src/os/bluestore/BlueStore.cc

@@ -6791,6 +6791,14 @@ int BlueStore::_open_fm(KeyValueDB::Transaction t,
       return r;
     }
   }
+  dout(1) << __func__ << " effective freelist_type = " << freelist_type << std::hex
+          << ", freelist_alloc_size = 0x" << fm->get_alloc_size()
+          << ", min_alloc_size = 0x" << min_alloc_size
+          << std::dec << dendl;
+  if (!fm->validate(min_alloc_size)) {
+    derr << __func__ << " freelist validation failed, unable to proceed." << dendl;
+    ceph_assert(false);
+  }
   // if space size tracked by free list manager is that higher than actual
   // dev size one can hit out-of-space allocation which will result
   // in data loss and/or assertions
@@ -13116,20 +13124,26 @@ int BlueStore::_open_super_meta()
   }
 
   {
-    bufferlist bl;
-    db->get(PREFIX_SUPER, "min_alloc_size", &bl);
-    auto p = bl.cbegin();
-    try {
-      uint64_t val;
-      decode(val, p);
-      min_alloc_size = val;
-      min_alloc_size_order = std::countr_zero(val);
-      min_alloc_size_mask  = min_alloc_size - 1;
+    if(cct->_conf->bluestore_debug_enforce_min_alloc_size == 0) {
+      bufferlist bl;
+      db->get(PREFIX_SUPER, "min_alloc_size", &bl);
+      auto p = bl.cbegin();
+      try {
+        uint64_t val;
+        decode(val, p);
+        min_alloc_size = val;
+        min_alloc_size_order = std::countr_zero(val);
+        min_alloc_size_mask  = min_alloc_size - 1;
 
-      ceph_assert(min_alloc_size == 1u << min_alloc_size_order);
-    } catch (ceph::buffer::error& e) {
-      derr << __func__ << " unable to read min_alloc_size" << dendl;
-      return -EIO;
+        ceph_assert(min_alloc_size == 1u << min_alloc_size_order);
+      } catch (ceph::buffer::error& e) {
+        derr << __func__ << " unable to read min_alloc_size" << dendl;
+        return -EIO;
+      }
+    } else {
+      min_alloc_size = cct->_conf->bluestore_debug_enforce_min_alloc_size;
+      min_alloc_size_order = std::countr_zero(min_alloc_size);
+      min_alloc_size_mask  = min_alloc_size - 1;
     }
     dout(1) << __func__ << " min_alloc_size 0x" << std::hex << min_alloc_size
 	     << std::dec << dendl;

src/os/bluestore/FreelistManager.h

@@ -52,6 +52,8 @@ class FreelistManager {
   virtual void get_meta(uint64_t target_size,
   std::vector<std::pair<std::string, std::string>>*) const = 0;
 
+  virtual bool validate(uint64_t min_alloc_size) const = 0;
+
   void set_null_manager() {
     null_manager = true;
   }