Merge pull request ceph#63358 from ifed01/wip-ifed-fix-vselector-math

os/bluestore:fix bluestore_volume_selection_reserved_factor usage

Author: aclamk

src/os/bluestore/BlueStore.cc

@@ -19677,13 +19677,13 @@ void RocksDBBlueFSVolumeSelector::dump(ostream& sout) {
     sout.width(width);
     switch (l + LEVEL_FIRST) {
     case LEVEL_LOG:
-      sout << "LOG"; break;
+      sout << "log"; break;
     case LEVEL_WAL:
-      sout << "WAL"; break;
+      sout << "db.wal"; break;
     case LEVEL_DB:
-      sout << "DB"; break;
+      sout << "db"; break;
     case LEVEL_SLOW:
-      sout << "SLOW"; break;
+      sout << "db.slow"; break;
     case LEVEL_MAX:
       sout << "TOTAL"; break;
     }
@@ -19704,13 +19704,13 @@ void RocksDBBlueFSVolumeSelector::dump(ostream& sout) {
     sout.width(width);
     switch (l + LEVEL_FIRST) {
     case LEVEL_LOG:
-      sout << "LOG"; break;
+      sout << "log"; break;
     case LEVEL_WAL:
-      sout << "WAL"; break;
+      sout << "db.wal"; break;
     case LEVEL_DB:
-      sout << "DB"; break;
+      sout << "db"; break;
     case LEVEL_SLOW:
-      sout << "SLOW"; break;
+      sout << "db.slow"; break;
     case LEVEL_MAX:
       sout << "TOTAL"; break;
     }

src/os/bluestore/BlueStore.h

@@ -4508,20 +4508,18 @@ class RocksDBBlueFSVolumeSelector : public BlueFSVolumeSelector
       level_multiplier = _level_multiplier;
       uint64_t prev_levels = _level0_size;
       uint64_t cur_level = _level_base;
-      uint64_t cur_threshold = prev_levels + cur_level;
       extra_level = 1;
       do {
 	uint64_t next_level = cur_level * _level_multiplier;
         uint64_t next_threshold = prev_levels + cur_level + next_level;
         ++extra_level;
         if (_db_total <= next_threshold) {
-	  cur_threshold *= reserved_factor;
+	  uint64_t cur_threshold = prev_levels + cur_level * reserved_factor;
           db_avail4slow = cur_threshold < _db_total ? _db_total - cur_threshold : 0;
           break;
         } else {
           prev_levels += cur_level;
           cur_level = next_level;
-          cur_threshold = next_threshold;
         }
       } while (true);
     } else {
@@ -4530,6 +4528,12 @@ class RocksDBBlueFSVolumeSelector : public BlueFSVolumeSelector
     }
   }
 
+  uint64_t get_available_extra() const {
+    return db_avail4slow;
+  }
+  uint64_t get_extra_level() const {
+    return extra_level;
+  }
   void* get_hint_for_log() const override {
     return  reinterpret_cast<void*>(LEVEL_LOG);
   }

src/test/objectstore/CMakeLists.txt

@@ -12,6 +12,7 @@ target_include_directories(store_test_fixture PRIVATE
 add_executable(ceph_test_objectstore
   store_test.cc
   $<TARGET_OBJECTS:store_test_fixture>)
+
 target_link_libraries(ceph_test_objectstore
   os
   ceph-common
@@ -24,6 +25,21 @@ target_link_libraries(ceph_test_objectstore
 install(TARGETS ceph_test_objectstore
   DESTINATION ${CMAKE_INSTALL_BINDIR})
 
+add_executable(ceph_test_bluestore_vselector
+  test_bluestore_vselector.cc)
+
+target_link_libraries(ceph_test_bluestore_vselector
+  os
+  ceph-common
+  ${UNITTEST_LIBS}
+  global
+  ${EXTRALIBS}
+  ${BLKID_LIBRARIES}
+  ${CMAKE_DL_LIBS}
+  )
+install(TARGETS ceph_test_bluestore_vselector
+  DESTINATION ${CMAKE_INSTALL_BINDIR})
+
 add_subdirectory(allocsim)
 
 add_executable(ceph_test_keyvaluedb

src/test/objectstore/test_bluestore_vselector.cc

@@ -0,0 +1,143 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "gtest/gtest.h"
+#include "os/bluestore/BlueStore.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include "global/global_context.h"
+
+using namespace std;
+
+TEST(rocksdb_bluefs_vselector, basic) {
+
+  uint64_t db_size = 168ull << 30;
+  uint64_t level_base = 1ull << 30;
+  size_t level_multi = 8;
+
+  RocksDBBlueFSVolumeSelector selector(
+    10ull << 30,
+    db_size,
+    1000ull << 30,
+    1ull << 30,
+    level_base,
+    level_multi,
+    g_ceph_context->_conf->bluestore_volume_selection_reserved_factor,
+    g_ceph_context->_conf->bluestore_volume_selection_reserved,
+    g_ceph_context->_conf->bluestore_volume_selection_policy.find("use_some_extra")
+      == 0);
+
+  // taken from RocksDBBlueFSVolumeSelector::
+  size_t log_bdev = 1; // LEVEL_LOG
+  size_t wal_bdev = 2; // LEVEL_WAL
+  size_t db_bdev = 3;  // LEVEL_DB
+  size_t slow_bdev = 4;// LEVEL_SLOW
+  bluefs_extent_t e;
+
+  ASSERT_EQ(4, selector.get_extra_level());
+  ASSERT_EQ(30ull << 30, selector.get_available_extra()); // 168GB - 1GB (L0) - 1GB (L1) - 8GB (L2) - 2*64GB (L3)
+
+  ASSERT_EQ(0, selector.select_prefer_bdev((void*)log_bdev));
+  ASSERT_EQ(0, selector.select_prefer_bdev((void*)wal_bdev));
+  ASSERT_EQ(1, selector.select_prefer_bdev((void*)db_bdev));
+  ASSERT_EQ(1, selector.select_prefer_bdev((void*)slow_bdev));
+  // 'Use' 138GB DB level data at DB vol
+  for (size_t i = 0; i < 138; i++) {
+    e.bdev = 1; // DB dev
+    e.length = 1ull * (1 << 30);
+    selector.add_usage((void*)db_bdev, e);
+  }
+  ASSERT_EQ(0, selector.select_prefer_bdev((void*)log_bdev));
+  ASSERT_EQ(0, selector.select_prefer_bdev((void*)wal_bdev));
+  ASSERT_EQ(1, selector.select_prefer_bdev((void*)db_bdev));
+  ASSERT_EQ(1, selector.select_prefer_bdev((void*)slow_bdev));
+
+  // 'Use' 30GB Slow level data at DB vol
+  for (size_t i = 0; i < 30; i++) {
+    e.bdev = 1; // DB dev
+    e.length = 1ull * (1 << 30);
+    ASSERT_EQ(1, selector.select_prefer_bdev((void*)slow_bdev));
+    selector.add_usage((void*)slow_bdev, e);
+  }
+  ASSERT_EQ(0, selector.select_prefer_bdev((void*)log_bdev));
+  ASSERT_EQ(0, selector.select_prefer_bdev((void*)wal_bdev));
+  ASSERT_EQ(1, selector.select_prefer_bdev((void*)db_bdev));
+  ASSERT_EQ(2, selector.select_prefer_bdev((void*)slow_bdev));
+
+  // 'Unuse' 10GB DB level data at DB vol, slow data still wouldn't fit
+  // as it's exceeds the threshold
+  for (size_t i = 0; i < 10; i++) {
+    e.bdev = 1; // DB dev
+    e.length = 1ull * (1 << 30);
+    selector.sub_usage((void*)db_bdev, e);
+  }
+  ASSERT_EQ(2, selector.select_prefer_bdev((void*)slow_bdev));
+
+  // 'Unuse' 10GB Slow level data at DB vol, slow data fits now
+  for (size_t i = 0; i < 10; i++) {
+    e.bdev = 1; // DB dev
+    e.length = 1ull * (1 << 30);
+    selector.sub_usage((void*)slow_bdev, e);
+  }
+  ASSERT_EQ(1, selector.select_prefer_bdev((void*)slow_bdev));
+
+  // 'Unuse' remaining 20GB Slow level data at DB vol, slow data fits now
+  for (size_t i = 0; i < 20; i++) {
+    e.bdev = 1; // DB dev
+    e.length = 1ull * (1 << 30);
+    selector.sub_usage((void*)slow_bdev, e);
+  }
+  // 'Use' 30GB DB level data at DB vol to raise historic maximum, 10GB slow data fits only since now
+  for (size_t i = 0; i < 30; i++) {
+    e.bdev = 1; // DB dev
+    e.length = 1ull * (1 << 30);
+    ASSERT_EQ(1, selector.select_prefer_bdev((void*)slow_bdev));
+    selector.add_usage((void*)db_bdev, e);
+  }
+  for (size_t i = 0; i < 10; i++) {
+    e.bdev = 1; // DB dev
+    e.length = 1ull * (1 << 30);
+    ASSERT_EQ(1, selector.select_prefer_bdev((void*)slow_bdev));
+    selector.add_usage((void*)slow_bdev, e);
+  }
+  ASSERT_EQ(2, selector.select_prefer_bdev((void*)slow_bdev));
+
+  // 'Unuse' remaining 10GB Slow level data at DB vol
+  for (size_t i = 0; i < 10; i++) {
+    e.bdev = 1; // DB dev
+    e.length = 1ull * (1 << 30);
+    selector.sub_usage((void*)slow_bdev, e);
+  }
+  // 'Use' additional 10GB DB level data at DB vol to raise historic maximum, 10GB slow data wouldn't fit since now
+  for (size_t i = 0; i < 10; i++) {
+    e.bdev = 1; // DB dev
+    e.length = 1ull * (1 << 30);
+    ASSERT_EQ(1, selector.select_prefer_bdev((void*)slow_bdev));
+    selector.add_usage((void*)db_bdev, e);
+  }
+  ASSERT_EQ(2, selector.select_prefer_bdev((void*)slow_bdev));
+
+  // 'Unuse' 50GB DB level data, thi s wouldn't let slow data use DB volume anyway
+  // due to updated historic maximum
+  for (size_t i = 0; i < 50; i++) {
+    e.bdev = 1; // DB dev
+    e.length = 1ull * (1 << 30);
+    selector.sub_usage((void*)db_bdev, e);
+  }
+  ASSERT_EQ(2, selector.select_prefer_bdev((void*)slow_bdev));
+
+
+  std::stringstream ss;
+  selector.dump(ss);
+  std::cout << ss.str() << std::endl;
+}
+
+int main(int argc, char **argv) {
+  auto args = argv_to_vec(argc, argv);
+  auto cct =
+      global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
+                  CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}