Collect telemetry for how many spans objects are pulled from to fulfill a batch request.

This data may be useful to come up with and/or prioritize optimizations to `RemoveRange` that involve reducing the number of spans we pull from to fulfill a batch request.

PiperOrigin-RevId: 828154762
Change-Id: Ief8bd5622bd274ac2a76fba6764fa762b3b562dc

Author: shortstopb513

tcmalloc/BUILD

@@ -1430,6 +1430,7 @@ cc_test(
         "@com_google_absl//absl/numeric:bits",
         "@com_google_absl//absl/random",
         "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/synchronization",
         "@com_google_absl//absl/time",
         "@com_google_absl//absl/types:span",

tcmalloc/central_freelist.h

@@ -135,11 +135,14 @@ class CentralFreeList {
   size_t NumSpansInList(int n) ABSL_LOCKS_EXCLUDED(lock_);
   SpanStats GetSpanStats() const;
 
-  // Reports span utilization and lifetime histogram stats.
+  // Reports span utilization, lifetime histogram stats, and number of spans
+  // used to fill a batch.
   void PrintSpanUtilStats(Printer& out);
   void PrintSpanLifetimeStats(Printer& out);
+  void PrintNumSpansUsed(Printer& out);
   void PrintSpanUtilStatsInPbtxt(PbtxtRegion& region);
   void PrintSpanLifetimeStatsInPbtxt(PbtxtRegion& region);
+  void PrintNumSpansUsedInPbtxt(PbtxtRegion& region);
 
   // Get number of spans in the histogram bucket. We record spans in the
   // histogram indexed by absl::bit_width(allocated). So, instead of using the
@@ -250,6 +253,9 @@ class CentralFreeList {
 
   StatsCounter completed_spans_[kLifetimeBuckets];
 
+  // Tracks the number of spans used to fill a batch in RemoveRange
+  StatsCounters<kMaxObjectsToMove> span_allocations_tracker_;
+
   int LifetimeBucketNum(absl::Duration duration) {
     int64_t duration_ms = absl::ToInt64Milliseconds(duration);
     auto it = std::upper_bound(lifetime_bucket_bounds_,
@@ -532,9 +538,11 @@ inline int CentralFreeList<Forwarder>::RemoveRange(absl::Span<void*> batch) {
   // Use local copy of variable to ensure that it is not reloaded.
   size_t object_size = object_size_;
   int result = 0;
+  size_t num_spans = 0;
   absl::base_internal::SpinLockHolder h(lock_);
 
   do {
+    num_spans++;
     Span* span = FirstNonEmptySpan();
     if (ABSL_PREDICT_FALSE(!span)) {
       result += Populate(batch.subspan(result));
@@ -572,6 +580,10 @@ inline int CentralFreeList<Forwarder>::RemoveRange(absl::Span<void*> batch) {
     }
     result += here;
   } while (result < batch.size());
+  TC_ASSERT_GT(num_spans, 0);
+  TC_ASSERT_LE(batch.size(), kMaxObjectsToMove);
+  TC_ASSERT_LE(num_spans, kMaxObjectsToMove);
+  span_allocations_tracker_[num_spans - 1].LossyAdd(1);
   UpdateObjectCounts(-result);
   return result;
 }
@@ -713,6 +725,18 @@ inline void CentralFreeList<Forwarder>::PrintSpanLifetimeStats(Printer& out) {
   out.printf("\n");
 }
 
+template <class Forwarder>
+inline void CentralFreeList<Forwarder>::PrintNumSpansUsed(Printer& out) {
+  out.printf("class %3d [ %8zu bytes ] : ", size_class_, object_size_);
+  for (size_t i = 1; i <= kMaxObjectsToMove; ++i) {
+    out.printf("%zu : %8zu", i, span_allocations_tracker_[i - 1].value());
+    if (i < kMaxObjectsToMove) {
+      out.printf(",");
+    }
+  }
+  out.printf("\n");
+}
+
 template <class Forwarder>
 inline void CentralFreeList<Forwarder>::PrintSpanUtilStatsInPbtxt(
     PbtxtRegion& region) {
@@ -731,6 +755,18 @@ inline void CentralFreeList<Forwarder>::PrintSpanUtilStatsInPbtxt(
   }
 }
 
+template <class Forwarder>
+inline void CentralFreeList<Forwarder>::PrintNumSpansUsedInPbtxt(
+    PbtxtRegion& region) {
+  for (size_t i = 1; i <= kMaxObjectsToMove; ++i) {
+    PbtxtRegion span_allocations_tracker =
+        region.CreateSubRegion("span_allocations_tracker");
+    span_allocations_tracker.PrintI64("num_spans", i);
+    span_allocations_tracker.PrintI64("value",
+                                      span_allocations_tracker_[i - 1].value());
+  }
+}
+
 template <class Forwarder>
 inline void CentralFreeList<Forwarder>::PrintSpanLifetimeStatsInPbtxt(
     PbtxtRegion& region) {

tcmalloc/central_freelist_test.cc

@@ -37,6 +37,7 @@
 #include "absl/memory/memory.h"
 #include "absl/numeric/bits.h"
 #include "absl/random/random.h"
+#include "absl/strings/str_format.h"
 #include "absl/strings/str_join.h"
 #include "absl/synchronization/mutex.h"
 #include "absl/time/clock.h"
@@ -497,7 +498,7 @@ TEST_P(CentralFreeListTest, SinglePopulate) {
   TypeParam e(std::get<0>(GetParam()).size, std::get<0>(GetParam()).pages,
               std::get<0>(GetParam()).num_to_move, std::get<1>(GetParam()));
   // Try to fetch sufficiently large number of objects at startup.
-  const int num_objects_to_fetch = 10 * e.objects_per_span();
+  const int num_objects_to_fetch = kMaxObjectsToMove;
   std::vector<void*> objects(num_objects_to_fetch, nullptr);
   const size_t got = e.central_freelist().RemoveRange(
       absl::MakeSpan(objects.data(), num_objects_to_fetch));
@@ -817,6 +818,76 @@ TEST_P(CentralFreeListTest, SpanLifetime) {
   CheckLifetimeStats(e, {.completed = {{100000, 1}}});
 }
 
+TEST_P(CentralFreeListTest, SpanAllocationTracker) {
+  TypeParam e(std::get<0>(GetParam()).size, std::get<0>(GetParam()).pages,
+              std::get<0>(GetParam()).num_to_move, std::get<1>(GetParam()));
+
+  const int objects_per_span = e.objects_per_span();
+  if (objects_per_span == 1) return;
+
+  constexpr int kNumSpans = 5;
+  // Track objects allocated per span.
+  absl::FixedArray<std::vector<void*>> objects(kNumSpans);
+  void* batch[kMaxObjectsToMove];
+
+  const size_t to_fetch = objects_per_span;
+  // Allocate all objects from kNumSpans.
+  for (int span = 0; span < kNumSpans; ++span) {
+    size_t fetched = 0;
+    while (fetched < to_fetch) {
+      const size_t n = to_fetch - fetched;
+      int got = e.central_freelist().RemoveRange(
+          absl::MakeSpan(batch, std::min(n, e.batch_size())));
+      for (int i = 0; i < got; ++i) {
+        objects[span].push_back(batch[i]);
+      }
+      fetched += got;
+    }
+  }
+
+  // Perform deallocations so that each span contains only one free object.
+  for (int span = 0; span < kNumSpans; ++span) {
+    batch[0] = objects[span][0];
+    e.central_freelist().InsertRange({batch, 1});
+    objects[span].erase(objects[span].begin(), objects[span].begin() + 1);
+  }
+
+  // Calculate the expected number of allocations served by 1 span. These all
+  // come from the case where there are no non-empty spans and a new span has to
+  // be allocated.
+  int single_spans = objects_per_span / e.batch_size();
+  if (objects_per_span % e.batch_size() != 0) {
+    single_spans++;
+  }
+  single_spans *= kNumSpans;
+
+  // Check txt stats
+  int got = e.central_freelist().RemoveRange(absl::MakeSpan(batch, kNumSpans));
+  EXPECT_EQ(got, kNumSpans);
+  std::string buffer(1024 * 1024, '\0');
+  Printer printer(&*buffer.begin(), buffer.size());
+  e.central_freelist().PrintNumSpansUsed(printer);
+  buffer.resize(strlen(buffer.c_str()));
+  EXPECT_THAT(
+      buffer,
+      testing::AllOf(
+          testing::HasSubstr(absl::StrFormat("%v :        1", kNumSpans)),
+          testing::ContainsRegex(absl::StrFormat("1 : *%v", single_spans))));
+
+  // Check pbtxt stats
+  std::string buffer_pbtxt(1024 * 1024, '\0');
+  Printer printer_pbtxt(&*buffer_pbtxt.begin(), buffer_pbtxt.size());
+  PbtxtRegion region(printer_pbtxt,
+                     tcmalloc::tcmalloc_internal::PbtxtRegionType::kTop);
+  e.central_freelist().PrintNumSpansUsedInPbtxt(region);
+  buffer_pbtxt.resize(strlen(buffer_pbtxt.c_str()));
+  EXPECT_THAT(buffer_pbtxt,
+              testing::AllOf(testing::HasSubstr(absl::StrFormat(
+                                 "{ num_spans: %v value: 1}", kNumSpans)),
+                             testing::HasSubstr(absl::StrFormat(
+                                 "{ num_spans: 1 value: %v}", single_spans))));
+}
+
 TEST_P(CentralFreeListTest, MultipleSpans) {
   TypeParam e(std::get<0>(GetParam()).size, std::get<0>(GetParam()).pages,
               std::get<0>(GetParam()).num_to_move, std::get<1>(GetParam()));

tcmalloc/global_stats.cc

@@ -484,6 +484,17 @@ void DumpStats(Printer& out, int level) {
       tc_globals.central_freelist(size_class).PrintSpanLifetimeStats(out);
     }
 
+    out.printf("\n");
+    out.printf("------------------------------------------------\n");
+    out.printf("Central cache freelist: Number of spans used histogram\n");
+    out.printf(
+        "Number of spans used to fill a batch of allocations, from 0 to "
+        "N\n");
+    out.printf("------------------------------------------------\n");
+    for (int size_class = 1; size_class < kNumClasses; ++size_class) {
+      tc_globals.central_freelist(size_class).PrintNumSpansUsed(out);
+    }
+
     tc_globals.transfer_cache().Print(tc_globals.per_size_class_counts(), out);
     tc_globals.sharded_transfer_cache().Print(
         tc_globals.per_size_class_counts(), out);
@@ -694,6 +705,7 @@ void DumpStatsInPbtxt(Printer& out, int level) {
       tc_globals.central_freelist(size_class).PrintSpanUtilStatsInPbtxt(entry);
       tc_globals.central_freelist(size_class)
           .PrintSpanLifetimeStatsInPbtxt(entry);
+      tc_globals.central_freelist(size_class).PrintNumSpansUsedInPbtxt(entry);
     }
 
     tc_globals.transfer_cache().PrintInPbtxt(tc_globals.per_size_class_counts(),