Merge pull request ceph#51820 from ifed01/wip-ifed-fragmentation-info

ifed01 · web-flow · commit d71eea965de0 · 2023-07-22T17:38:13.000+03:00
os/bluestore: introduce allocator state histogram

Reviewed-by: Adam Kupczyk &lt;akupczyk@ibm.com&gt;
diff --git a/src/os/bluestore/Allocator.cc b/src/os/bluestore/Allocator.cc
@@ -14,6 +14,7 @@
 #include "common/debug.h"
 #include "common/admin_socket.h"
 #define dout_subsys ceph_subsys_bluestore
+using TOPNSPC::common::cmd_getval;
 
 using std::string;
 using std::to_string;
@@ -52,6 +53,13 @@ class Allocator::SocketHook : public AdminSocketHook {
           this,
           "give allocator fragmentation (0-no fragmentation, 1-absolute fragmentation)");
         ceph_assert(r == 0);
+        r = admin_socket->register_command(
+	  ("bluestore allocator fragmentation histogram " + name +
+           " name=alloc_unit,type=CephInt,req=false" +
+           " name=num_buckets,type=CephInt,req=false").c_str(),
+	  this,
+	  "build allocator free regions state histogram");
+        ceph_assert(r == 0);
       }
     }
   }
@@ -100,6 +108,39 @@ class Allocator::SocketHook : public AdminSocketHook {
       f->open_object_section("fragmentation");
       f->dump_float("fragmentation_rating", alloc->get_fragmentation());
       f->close_section();
+    } else if (command == "bluestore allocator fragmentation histogram " + name) {
+      int64_t alloc_unit = 4096;
+      cmd_getval(cmdmap, "alloc_unit", alloc_unit);
+      if (alloc_unit == 0  ||
+          p2align(alloc_unit, alloc->get_block_size()) != alloc_unit) {
+        ss << "Invalid allocation unit: '" << alloc_unit
+           << ", to be aligned with: '" << alloc->get_block_size()
+           << std::endl;
+        return -EINVAL;
+      }
+      int64_t num_buckets = 8;
+      cmd_getval(cmdmap, "num_buckets", num_buckets);
+      if (num_buckets < 2) {
+        ss << "Invalid amount of buckets (min=2): '" << num_buckets
+           << std::endl;
+        return -EINVAL;
+      }
+
+      Allocator::FreeStateHistogram hist;
+      hist.resize(num_buckets);
+      alloc->build_free_state_histogram(alloc_unit, hist);
+      f->open_array_section("extent_counts");
+      for(int i = 0; i < num_buckets; i++) {
+        f->open_object_section("c");
+        f->dump_unsigned("max_len",
+          hist[i].get_max(i, num_buckets)
+        );
+        f->dump_unsigned("total", hist[i].total);
+        f->dump_unsigned("aligned", hist[i].aligned);
+        f->dump_unsigned("units", hist[i].alloc_units);
+        f->close_section();
+      }
+      f->close_section();
     } else {
       ss << "Invalid command" << std::endl;
       r = -ENOSYS;
@@ -234,3 +275,51 @@ double Allocator::get_fragmentation_score()
   double terrible = (sum / block_size) * get_score(block_size);
   return (ideal - score_sum) / (ideal - terrible);
 }
+
+void Allocator::build_free_state_histogram(
+  size_t alloc_unit, Allocator::FreeStateHistogram& hist)
+{
+  auto num_buckets = hist.size();
+  ceph_assert(num_buckets);
+
+  auto base = free_state_hist_bucket::base;
+  auto base_bits = free_state_hist_bucket::base_bits;
+  auto mux = free_state_hist_bucket::mux;
+  // maximum chunk size we track,
+  // provided by the bucket before the last one
+  size_t max =
+    free_state_hist_bucket::get_max(num_buckets - 2, num_buckets);
+
+  auto iterated_allocation = [&](size_t off, size_t len) {
+    size_t idx;
+    if (len <= base) {
+      idx = 0;
+    } else if (len > max) {
+      idx = num_buckets - 1;
+    } else {
+      size_t most_bit = cbits(uint64_t(len-1)) - 1;
+      idx = 1 + ((most_bit - base_bits) / mux);
+    }
+    ceph_assert(idx < num_buckets);
+    ++hist[idx].total;
+
+    // now calculate the bucket for the chunk after alignment,
+    // resulting chunks shorter than alloc_unit are discarded
+    auto delta = p2roundup(off, alloc_unit) - off;
+    if (len >= delta + alloc_unit) {
+      len -= delta;
+      if (len <= base) {
+        idx = 0;
+      } else if (len > max) {
+        idx = num_buckets - 1;
+      } else {
+        size_t most_bit = cbits(uint64_t(len-1)) - 1;
+        idx = 1 + ((most_bit - base_bits) / mux);
+      }
+      ++hist[idx].aligned;
+      hist[idx].alloc_units += len / alloc_unit;
+    }
+  };
+
+  foreach(iterated_allocation);
+}
diff --git a/src/os/bluestore/Allocator.h b/src/os/bluestore/Allocator.h
@@ -88,6 +88,37 @@ class Allocator {
     return block_size;
   }
 
+  // The following code build Allocator's free extents histogram.
+  // Which is a set of N buckets to track extents layout.
+  // Extent matches a bucket depending on its length using the following
+  // length spans:
+  // [0..4K] (4K..16K] (16K..64K] .. (4M..16M] (16M..]
+  // Each bucket tracks:
+  // - total amount of extents of specific lengths
+  // - amount of extents aligned with allocation boundary
+  // - amount of allocation units in aligned extents
+  //
+  struct free_state_hist_bucket {
+    static const size_t base_bits = 12;
+    static const size_t base = 1ull << base_bits;
+    static const size_t mux = 2;
+
+    size_t total = 0;
+    size_t aligned = 0;
+    size_t alloc_units = 0;
+
+    // returns upper bound of the bucket
+    static size_t get_max(size_t bucket, size_t num_buckets) {
+      return
+        bucket < num_buckets - 1 ?
+          base << (mux * bucket) :
+          std::numeric_limits<uint64_t>::max();
+    };
+  };
+
+  typedef std::vector<free_state_hist_bucket> FreeStateHistogram;
+  void build_free_state_histogram(size_t alloc_unit, FreeStateHistogram& hist);
+
 private:
   class SocketHook;
   SocketHook* asok_hook = nullptr;
diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
@@ -17763,9 +17763,11 @@ void BlueStore::_log_alerts(osd_alert_list_t& alerts)
 void BlueStore::_collect_allocation_stats(uint64_t need, uint32_t alloc_size,
                                           const PExtentVector& extents)
 {
-  alloc_stats_count++;
-  alloc_stats_fragments += extents.size();
-  alloc_stats_size += need;
+  if (alloc_size != min_alloc_size) {
+    alloc_stats_count++;
+    alloc_stats_fragments += extents.size();
+    alloc_stats_size += need;
+  }
 
   for (auto& e : extents) {
     logger->hinc(l_bluestore_allocate_hist, e.length, need);
diff --git a/src/test/objectstore/allocator_replay_test.cc b/src/test/objectstore/allocator_replay_test.cc
@@ -20,7 +20,15 @@
 using namespace std;
 
 void usage(const string &name) {
-  cerr << "Usage: " << name << " <log_to_replay> <raw_duplicates|duplicates|free_dump|try_alloc count want alloc_unit|replay_alloc alloc_list_file|export_binary out_file>" << std::endl;
+  cerr << "Usage: " << name << " <log_to_replay|free-dump> "
+       << " raw_duplicates|"
+          "duplicates|"
+          "free_dump|"
+          "try_alloc <count> <want> <alloc_unit>|"
+          "replay_alloc <alloc_list_file|"
+          "export_binary <out_file>|"
+          "free_histogram [<alloc_unit>] [<num_buckets>]"
+       << std::endl;
 }
 
 void usage_replay_alloc(const string &name) {
@@ -548,7 +556,7 @@ int main(int argc, char **argv)
                   << std::endl;
         std::cout << "Fragmentation score:" << a->get_fragmentation_score()
                   << std::endl;
-        std::cout << "Free:" << std::hex << a->get_free() << std::dec
+        std::cout << "Free: 0x" << std::hex << a->get_free() << std::dec
                   << std::endl;
         {
         // stub to implement various testing stuff on properly initialized allocator
@@ -574,7 +582,7 @@ int main(int argc, char **argv)
                   << std::endl;
         std::cout << "Fragmentation score:" << a->get_fragmentation_score()
                   << std::endl;
-        std::cout << "Free:" << std::hex << a->get_free() << std::dec
+        std::cout << "Free: 0x" << std::hex << a->get_free() << std::dec
                   << std::endl;
         {
           PExtentVector extents;
@@ -606,7 +614,7 @@ int main(int argc, char **argv)
                   << std::endl;
         std::cout << "Fragmentation score:" << a->get_fragmentation_score()
                   << std::endl;
-        std::cout << "Free:" << std::hex << a->get_free() << std::dec
+        std::cout << "Free: 0x" << std::hex << a->get_free() << std::dec
                   << std::endl;
         {
           /* replay a set of allocation requests */
@@ -655,7 +663,7 @@ int main(int argc, char **argv)
                           << std::endl;
                 std::cerr << "Fragmentation score:" << a->get_fragmentation_score()
                           << std::endl;
-                std::cerr << "Free:" << std::hex << a->get_free() << std::dec
+                std::cerr << "Free: 0x" << std::hex << a->get_free() << std::dec
                           << std::endl;
                 /* return 0 if the allocator ran out of space */
                 if (r == -ENOSPC) {
@@ -681,11 +689,48 @@ int main(int argc, char **argv)
                     << std::endl;
           std::cout << "Fragmentation score:" << a->get_fragmentation_score()
                     << std::endl;
-          std::cout << "Free:" << std::hex << a->get_free() << std::dec
+          std::cout << "Free: 0x" << std::hex << a->get_free() << std::dec
                     << std::endl;
         }
         return 0;
     });
+  } else if (strcmp(argv[2], "free_histogram") == 0) {
+    uint64_t alloc_unit = 4096;
+    auto num_buckets = 8;
+    if (argc >= 4) {
+      alloc_unit = strtoul(argv[3], nullptr, 10);
+    }
+    if (argc >= 5) {
+      num_buckets = strtoul(argv[4], nullptr, 10);
+    }
+    return replay_free_dump_and_apply(argv[1],
+      [&](Allocator *a, const string &aname) {
+        ceph_assert(a);
+        std::cout << "Fragmentation:" << a->get_fragmentation()
+                  << std::endl;
+        std::cout << "Fragmentation score:" << a->get_fragmentation_score()
+                  << std::endl;
+        std::cout << "Free: 0x" << std::hex << a->get_free() << std::dec
+                  << std::endl;
+        std::cout << "Allocation unit:" << alloc_unit
+                  << std::endl;
+
+        Allocator::FreeStateHistogram hist;
+        hist.resize(num_buckets);
+        a->build_free_state_histogram(alloc_unit, hist);
+
+        uint64_t s = 0;
+        for(int i = 0; i < num_buckets; i++) {
+          uint64_t e = hist[i].get_max(i, num_buckets);
+	  std::cout << "(" << s << ".." << e << "]"
+                    << " -> " << hist[i].total
+                    << " chunks, " << hist[i].aligned << " aligned with "
+                    << hist[i].alloc_units << " alloc_units."
+		    << std::endl;
+          s = e;
+        }
+	return 0;
+    });
   } else if (strcmp(argv[2], "export_binary") == 0) {
     return export_as_binary(argv[1], argv[3]);
   } else if (strcmp(argv[2], "duplicates") == 0) {
