Wire up userspace collapse heuristic experiment.

PiperOrigin-RevId: 831061743
Change-Id: Ie4e7374c192d18f5cdccc3bbbd06d3ac3daae3ec

Author: TCMalloc Team

tcmalloc/experiment_config.h

@@ -23,8 +23,7 @@ namespace tcmalloc {
 enum class Experiment : int {
   // clang-format off
   // go/keep-sorted start
-  TCMALLOC_COLLAPSE_HEURISTICS,  // TODO: b/456564608 - Complete experiment.
-  TCMALLOC_EXTENDED_PRIORITY_LISTS_V1,  // TODO: b/437412600 - Complete experiment.
+    TCMALLOC_EXTENDED_PRIORITY_LISTS_V1,  // TODO: b/437412600 - Complete experiment.
   TCMALLOC_RELEASE_FREE_SWAPPED,  // TODO: b/438767733 - Complete experiment.
   TEST_ONLY_L3_AWARE,  // TODO: b/239977380 - Complete experiment.
   TEST_ONLY_MM_VCPU,  // TODO: b/245776120 - Complete experiment.
@@ -49,7 +48,6 @@ struct ExperimentConfig {
 // clang-format off
 inline constexpr ExperimentConfig experiments[] = {
     // go/keep-sorted start
-    {Experiment::TCMALLOC_COLLAPSE_HEURISTICS, "TCMALLOC_COLLAPSE_HEURISTICS"},
     {Experiment::TCMALLOC_EXTENDED_PRIORITY_LISTS_V1, "TCMALLOC_EXTENDED_PRIORITY_LISTS_V1"},
     {Experiment::TCMALLOC_RELEASE_FREE_SWAPPED, "TCMALLOC_RELEASE_FREE_SWAPPED"},
     {Experiment::TEST_ONLY_L3_AWARE, "TEST_ONLY_L3_AWARE"},

tcmalloc/parameters.cc

@@ -200,16 +200,8 @@ ABSL_CONST_INIT std::atomic<double>
 ABSL_CONST_INIT std::atomic<int64_t> Parameters::profile_sampling_interval_(
     kDefaultProfileSamplingInterval);
 
-static std::atomic<bool>& use_userspace_collapse_heuristics_enabled() {
-  ABSL_CONST_INIT static absl::once_flag flag;
-  ABSL_CONST_INIT static std::atomic<bool> v{false};
-  absl::base_internal::LowLevelCallOnce(&flag, [&]() {
-    if (IsExperimentActive(Experiment::TCMALLOC_COLLAPSE_HEURISTICS)) {
-      v.store(true, std::memory_order_relaxed);
-    }
-  });
-  return v;
-}
+ABSL_CONST_INIT std::atomic<bool>
+    Parameters::use_userspace_collapse_heuristics_(false);
 
 static std::atomic<bool>& release_free_swapped_enabled() {
   ABSL_CONST_INIT static absl::once_flag flag;
@@ -255,11 +247,6 @@ bool Parameters::usermode_hugepage_collapse() {
   return usermode_hugepage_collapse_enabled_.load(std::memory_order_relaxed);
 }
 
-bool Parameters::use_userspace_collapse_heuristics() {
-  return use_userspace_collapse_heuristics_enabled().load(
-      std::memory_order_relaxed);
-}
-
 bool Parameters::release_free_swapped() {
   return release_free_swapped_enabled().load(std::memory_order_relaxed);
 }
@@ -286,6 +273,10 @@ int32_t Parameters::max_per_cpu_cache_size() {
   return tc_globals.cpu_cache().CacheLimit();
 }
 
+bool TCMalloc_Internal_GetUseUserspaceCollapseHeuristics() {
+  return Parameters::use_userspace_collapse_heuristics();
+}
+
 int ABSL_ATTRIBUTE_WEAK default_want_disable_dynamic_slabs();
 
 // TODO(b/271475288): remove the default_want_disable_dynamic_slabs opt-out
@@ -577,13 +568,9 @@ void TCMalloc_Internal_SetReleaseFreeSwapped(bool v) {
       v, std::memory_order_relaxed);
 }
 
-bool TCMalloc_Internal_GetUseUserspaceCollapseHeuristics() {
-  return Parameters::use_userspace_collapse_heuristics();
-}
-
 void TCMalloc_Internal_SetUseUserspaceCollapseHeuristics(bool v) {
-  tcmalloc::tcmalloc_internal::use_userspace_collapse_heuristics_enabled()
-      .store(v, std::memory_order_relaxed);
+  Parameters::use_userspace_collapse_heuristics_.store(
+      v, std::memory_order_relaxed);
 }
 
 }  // extern "C"

tcmalloc/parameters.h

@@ -99,7 +99,9 @@ class Parameters {
 
   static bool release_free_swapped();
 
-  static bool use_userspace_collapse_heuristics();
+  static bool use_userspace_collapse_heuristics() {
+    return use_userspace_collapse_heuristics_.load(std::memory_order_relaxed);
+  }
 
   static bool release_pages_from_huge_region() {
     return release_pages_from_huge_region_.load(std::memory_order_relaxed);
@@ -222,6 +224,7 @@ class Parameters {
   static std::atomic<double> per_cpu_caches_dynamic_slab_grow_threshold_;
   static std::atomic<double> per_cpu_caches_dynamic_slab_shrink_threshold_;
   static std::atomic<bool> usermode_hugepage_collapse_enabled_;
+  static std::atomic<bool> use_userspace_collapse_heuristics_;
 };
 
 }  // namespace tcmalloc_internal

tcmalloc/testing/get_stats_test.cc

@@ -133,11 +133,7 @@ TEST_F(GetStatsTest, Pbtxt) {
   } else {
     EXPECT_THAT(buf, HasSubstr("release_free_swapped: false"));
   }
-  if (IsExperimentActive(Experiment::TCMALLOC_COLLAPSE_HEURISTICS)) {
-    EXPECT_THAT(buf, HasSubstr("use_userspace_collapse_heuristics: true"));
-  } else {
-    EXPECT_THAT(buf, HasSubstr("use_userspace_collapse_heuristics: false"));
-  }
+  EXPECT_THAT(buf, HasSubstr("use_userspace_collapse_heuristics: false"));
   if (IsExperimentActive(
           Experiment::TEST_ONLY_TCMALLOC_EXTENDED_PRIORITY_LISTS_V1) ||
       IsExperimentActive(Experiment::TCMALLOC_EXTENDED_PRIORITY_LISTS_V1)) {
@@ -223,17 +219,9 @@ TEST_F(GetStatsTest, Parameters) {
       EXPECT_THAT(buf,
                   HasSubstr(R"(PARAMETER tcmalloc_release_free_swapped 0)"));
     }
-    if (IsExperimentActive(Experiment::TCMALLOC_COLLAPSE_HEURISTICS)) {
-      EXPECT_THAT(
-          buf,
-          HasSubstr(
-              R"(PARAMETER tcmalloc_use_userspace_collapse_heuristics 1)"));
-    } else {
-      EXPECT_THAT(
-          buf,
-          HasSubstr(
-              R"(PARAMETER tcmalloc_use_userspace_collapse_heuristics 0)"));
-    }
+    EXPECT_THAT(
+        buf,
+        HasSubstr(R"(PARAMETER tcmalloc_use_userspace_collapse_heuristics 0)"));
     if (IsExperimentActive(
             Experiment::TEST_ONLY_TCMALLOC_EXTENDED_PRIORITY_LISTS_V1) ||
         IsExperimentActive(Experiment::TCMALLOC_EXTENDED_PRIORITY_LISTS_V1)) {

tcmalloc/variants.bzl

@@ -265,12 +265,6 @@ test_variants = [
         "deps": ["//tcmalloc:common_8k_pages"],
         "env": {"BORG_EXPERIMENTS": "TEST_ONLY_TCMALLOC_EXTENDED_PRIORITY_LISTS_V1"},
     },
-    {
-        "name": "tcmalloc_collapse_heuristics",
-        "malloc": "//tcmalloc",
-        "deps": ["//tcmalloc:common_8k_pages"],
-        "env": {"BORG_EXPERIMENTS": "TCMALLOC_COLLAPSE_HEURISTICS"},
-    },
     {
         "name": "tcmalloc_release_free_swapped",
         "malloc": "//tcmalloc",