No public description

PiperOrigin-RevId: 831070869
Change-Id: I80107296e0a36cc057f3f221b20c889eac02100c

Author: TCMalloc Team

tcmalloc/experiment_config.h

@@ -23,7 +23,8 @@ namespace tcmalloc {
 enum class Experiment : int {
   // clang-format off
   // go/keep-sorted start
-    TCMALLOC_EXTENDED_PRIORITY_LISTS_V1,  // TODO: b/437412600 - Complete experiment.
+  TCMALLOC_COLLAPSE_HEURISTICS,  // TODO: b/456564608 - Complete experiment.
+  TCMALLOC_EXTENDED_PRIORITY_LISTS_V1,  // TODO: b/437412600 - Complete experiment.
   TCMALLOC_RELEASE_FREE_SWAPPED,  // TODO: b/438767733 - Complete experiment.
   TEST_ONLY_L3_AWARE,  // TODO: b/239977380 - Complete experiment.
   TEST_ONLY_MM_VCPU,  // TODO: b/245776120 - Complete experiment.
@@ -48,6 +49,7 @@ struct ExperimentConfig {
 // clang-format off
 inline constexpr ExperimentConfig experiments[] = {
     // go/keep-sorted start
+    {Experiment::TCMALLOC_COLLAPSE_HEURISTICS, "TCMALLOC_COLLAPSE_HEURISTICS"},
     {Experiment::TCMALLOC_EXTENDED_PRIORITY_LISTS_V1, "TCMALLOC_EXTENDED_PRIORITY_LISTS_V1"},
     {Experiment::TCMALLOC_RELEASE_FREE_SWAPPED, "TCMALLOC_RELEASE_FREE_SWAPPED"},
     {Experiment::TEST_ONLY_L3_AWARE, "TEST_ONLY_L3_AWARE"},

tcmalloc/parameters.cc

@@ -200,8 +200,16 @@ ABSL_CONST_INIT std::atomic<double>
 ABSL_CONST_INIT std::atomic<int64_t> Parameters::profile_sampling_interval_(
     kDefaultProfileSamplingInterval);
 
-ABSL_CONST_INIT std::atomic<bool>
-    Parameters::use_userspace_collapse_heuristics_(false);
+static std::atomic<bool>& use_userspace_collapse_heuristics_enabled() {
+  ABSL_CONST_INIT static absl::once_flag flag;
+  ABSL_CONST_INIT static std::atomic<bool> v{false};
+  absl::base_internal::LowLevelCallOnce(&flag, [&]() {
+    if (IsExperimentActive(Experiment::TCMALLOC_COLLAPSE_HEURISTICS)) {
+      v.store(true, std::memory_order_relaxed);
+    }
+  });
+  return v;
+}
 
 static std::atomic<bool>& release_free_swapped_enabled() {
   ABSL_CONST_INIT static absl::once_flag flag;
@@ -247,6 +255,11 @@ bool Parameters::usermode_hugepage_collapse() {
   return usermode_hugepage_collapse_enabled_.load(std::memory_order_relaxed);
 }
 
+bool Parameters::use_userspace_collapse_heuristics() {
+  return use_userspace_collapse_heuristics_enabled().load(
+      std::memory_order_relaxed);
+}
+
 bool Parameters::release_free_swapped() {
   return release_free_swapped_enabled().load(std::memory_order_relaxed);
 }
@@ -273,10 +286,6 @@ int32_t Parameters::max_per_cpu_cache_size() {
   return tc_globals.cpu_cache().CacheLimit();
 }
 
-bool TCMalloc_Internal_GetUseUserspaceCollapseHeuristics() {
-  return Parameters::use_userspace_collapse_heuristics();
-}
-
 int ABSL_ATTRIBUTE_WEAK default_want_disable_dynamic_slabs();
 
 // TODO(b/271475288): remove the default_want_disable_dynamic_slabs opt-out
@@ -568,9 +577,13 @@ void TCMalloc_Internal_SetReleaseFreeSwapped(bool v) {
       v, std::memory_order_relaxed);
 }
 
+bool TCMalloc_Internal_GetUseUserspaceCollapseHeuristics() {
+  return Parameters::use_userspace_collapse_heuristics();
+}
+
 void TCMalloc_Internal_SetUseUserspaceCollapseHeuristics(bool v) {
-  Parameters::use_userspace_collapse_heuristics_.store(
-      v, std::memory_order_relaxed);
+  tcmalloc::tcmalloc_internal::use_userspace_collapse_heuristics_enabled()
+      .store(v, std::memory_order_relaxed);
 }
 
 }  // extern "C"

tcmalloc/parameters.h

@@ -99,9 +99,7 @@ class Parameters {
 
   static bool release_free_swapped();
 
-  static bool use_userspace_collapse_heuristics() {
-    return use_userspace_collapse_heuristics_.load(std::memory_order_relaxed);
-  }
+  static bool use_userspace_collapse_heuristics();
 
   static bool release_pages_from_huge_region() {
     return release_pages_from_huge_region_.load(std::memory_order_relaxed);
@@ -224,7 +222,6 @@ class Parameters {
   static std::atomic<double> per_cpu_caches_dynamic_slab_grow_threshold_;
   static std::atomic<double> per_cpu_caches_dynamic_slab_shrink_threshold_;
   static std::atomic<bool> usermode_hugepage_collapse_enabled_;
-  static std::atomic<bool> use_userspace_collapse_heuristics_;
 };
 
 }  // namespace tcmalloc_internal

tcmalloc/testing/get_stats_test.cc

@@ -133,7 +133,11 @@ TEST_F(GetStatsTest, Pbtxt) {
   } else {
     EXPECT_THAT(buf, HasSubstr("release_free_swapped: false"));
   }
-  EXPECT_THAT(buf, HasSubstr("use_userspace_collapse_heuristics: false"));
+  if (IsExperimentActive(Experiment::TCMALLOC_COLLAPSE_HEURISTICS)) {
+    EXPECT_THAT(buf, HasSubstr("use_userspace_collapse_heuristics: true"));
+  } else {
+    EXPECT_THAT(buf, HasSubstr("use_userspace_collapse_heuristics: false"));
+  }
   if (IsExperimentActive(
           Experiment::TEST_ONLY_TCMALLOC_EXTENDED_PRIORITY_LISTS_V1) ||
       IsExperimentActive(Experiment::TCMALLOC_EXTENDED_PRIORITY_LISTS_V1)) {
@@ -219,9 +223,17 @@ TEST_F(GetStatsTest, Parameters) {
       EXPECT_THAT(buf,
                   HasSubstr(R"(PARAMETER tcmalloc_release_free_swapped 0)"));
     }
-    EXPECT_THAT(
-        buf,
-        HasSubstr(R"(PARAMETER tcmalloc_use_userspace_collapse_heuristics 0)"));
+    if (IsExperimentActive(Experiment::TCMALLOC_COLLAPSE_HEURISTICS)) {
+      EXPECT_THAT(
+          buf,
+          HasSubstr(
+              R"(PARAMETER tcmalloc_use_userspace_collapse_heuristics 1)"));
+    } else {
+      EXPECT_THAT(
+          buf,
+          HasSubstr(
+              R"(PARAMETER tcmalloc_use_userspace_collapse_heuristics 0)"));
+    }
     if (IsExperimentActive(
             Experiment::TEST_ONLY_TCMALLOC_EXTENDED_PRIORITY_LISTS_V1) ||
         IsExperimentActive(Experiment::TCMALLOC_EXTENDED_PRIORITY_LISTS_V1)) {

tcmalloc/variants.bzl

@@ -265,6 +265,12 @@ test_variants = [
         "deps": ["//tcmalloc:common_8k_pages"],
         "env": {"BORG_EXPERIMENTS": "TEST_ONLY_TCMALLOC_EXTENDED_PRIORITY_LISTS_V1"},
     },
+    {
+        "name": "tcmalloc_collapse_heuristics",
+        "malloc": "//tcmalloc",
+        "deps": ["//tcmalloc:common_8k_pages"],
+        "env": {"BORG_EXPERIMENTS": "TCMALLOC_COLLAPSE_HEURISTICS"},
+    },
     {
         "name": "tcmalloc_release_free_swapped",
         "malloc": "//tcmalloc",