address sporadic timing-related failures in benchmark test

Summary: Testing that some elapsed time period has an upper bound can have flaky results. Try multiple times, take the median of elapsed time periods, and then test that the median has an upper bound. The intention is reduce the likelihood of flakiness.

Reviewed By: Orvid

Differential Revision: D78906079

fbshipit-source-id: ad6ee41b35674b133f3ac70a7332731d5cdf2857

Author: yfeldblum

fatal/benchmark/test/benchmark_test.cpp

@@ -15,6 +15,7 @@
 #include <iostream>
 #include <map>
 #include <string>
+#include <sstream>
 #include <thread>
 
 #include <cassert>
@@ -84,41 +85,92 @@ FATAL_BENCHMARK(group_2, benchmark_2_4) {
 }
 
 FATAL_TEST(benchmark, sanity_check) {
-  std::map<std::string, std::map<std::string, duration>> metrics;
+  struct state {
+    std::ostringstream cout;
+    results metrics;
+    std::thread thread;
 
-  for (auto const &i: run(std::cout)) {
-    auto &group = metrics[i.first];
+    void run_one() {
+      metrics = run(cout);
+    }
+  };
+  auto states = std::vector<state>(12);
 
-    for (auto const &j: i.second) {
-      FATAL_ASSERT_EQ(group.end(), group.find(j.name()));
+  for (auto &st : states) {
+    st.thread = std::thread(std::bind(&state::run_one, std::ref(st)));
+  }
+  for (auto &st : states) {
+    st.thread.join();
+  }
 
-      group[j.name()] = j.period();
+  for (auto &st : states) {
+    std::cout << st.cout.str();
+
+    for (auto const &i: st.metrics) {
+      auto const &grp = i.second;
+      std::vector<std::string> names;
+      auto const txfm = std::bind(&result_entry::name, std::placeholders::_1);
+      std::transform(grp.begin(), grp.end(), std::back_inserter(names), txfm);
+      std::sort(names.begin(), names.end());
+      auto const adj = std::adjacent_find(names.begin(), names.end());
+      FATAL_ASSERT_EQ(adj, names.end());
     }
   }
 
-  auto get = [&](std::string group, std::string name) {
-    auto i = metrics.find(group);
-    assert(i != metrics.end());
+  auto get_min = [&](std::string const &group, std::string const &name) {
+    auto min = duration{};
+    for (auto const &st : states) {
+      auto const i = st.metrics.find(group);
+      assert(i != st.metrics.end());
+      auto const &grp = i->second;
+
+      auto const pred = [&](auto const &it) { return it.name() == name; };
+      auto const j = std::find_if(grp.begin(), grp.end(), pred);
+      assert(j != i->second.end());
+
+      min = std::max(min, j->period());
+    }
+    return min;
+  };
+
+  auto get_med = [&](std::string const &group, std::string const &name) {
+    std::vector<duration> periods;
+    for (auto const &st : states) {
+      auto const i = st.metrics.find(group);
+      assert(i != st.metrics.end());
+      auto const &grp = i->second;
 
-    auto j = i->second.find(name);
-    assert(j != i->second.end());
+      auto const pred = [&](auto const &it) { return it.name() == name; };
+      auto const j = std::find_if(grp.begin(), grp.end(), pred);
+      assert(j != i->second.end());
 
-    return j->second;
+      periods.push_back(j->period());
+    }
+
+    assert(!periods.empty());
+    std::sort(periods.begin(), periods.end());
+    if (periods.size() % 2 == 1) {
+      return periods[periods.size() / 2];
+    } else {
+      auto const m0 = periods[periods.size() / 2 - 1];
+      auto const m1 = periods[periods.size() / 2 - 0];
+      return (m0 + m1) / 2;
+    }
   };
 
   FATAL_ASSERT_LT(small_delay, big_delay);
 
-  FATAL_EXPECT_LT(get("group_1", "benchmark_1_1"), small_delay);
-  FATAL_EXPECT_GE(get("group_1", "benchmark_1_2"), big_delay);
-  FATAL_EXPECT_LT(get("group_1", "benchmark_1_3"), small_delay);
-  FATAL_EXPECT_GE(get("group_1", "benchmark_1_4"), big_delay);
-  FATAL_EXPECT_LT(get("group_1", "benchmark_1_5"), small_delay);
-
-  FATAL_EXPECT_LT(get("group_2", "benchmark_2_1"), big_delay);
-  FATAL_EXPECT_GE(get("group_2", "benchmark_2_1"), small_delay);
-  FATAL_EXPECT_LT(get("group_2", "benchmark_2_2"), small_delay);
-  FATAL_EXPECT_GE(get("group_2", "benchmark_2_3"), big_delay);
-  FATAL_EXPECT_LT(get("group_2", "benchmark_2_4"), small_delay);
+  FATAL_EXPECT_LT(get_med("group_1", "benchmark_1_1"), small_delay);
+  FATAL_EXPECT_GE(get_min("group_1", "benchmark_1_2"), big_delay);
+  FATAL_EXPECT_LT(get_med("group_1", "benchmark_1_3"), small_delay);
+  FATAL_EXPECT_GE(get_min("group_1", "benchmark_1_4"), big_delay);
+  FATAL_EXPECT_LT(get_med("group_1", "benchmark_1_5"), small_delay);
+
+  FATAL_EXPECT_LT(get_med("group_2", "benchmark_2_1"), big_delay);
+  FATAL_EXPECT_GE(get_min("group_2", "benchmark_2_1"), small_delay);
+  FATAL_EXPECT_LT(get_med("group_2", "benchmark_2_2"), small_delay);
+  FATAL_EXPECT_GE(get_min("group_2", "benchmark_2_3"), big_delay);
+  FATAL_EXPECT_LT(get_med("group_2", "benchmark_2_4"), small_delay);
 }
 
 } // namespace benchmark {