bench: add "priority level" to the benchmark framework

Will allow us to run certain benchmarks while skip
non-prioritized ones in 'make check'.

Author: furszy

src/bench/bench.cpp

@@ -6,6 +6,7 @@
 
 #include <fs.h>
 #include <test/util/setup_common.h>
+#include <util/string.h>
 
 #include <chrono>
 #include <fstream>
@@ -43,15 +44,37 @@ void GenerateTemplateResults(const std::vector<ankerl::nanobench::Result>& bench
 
 namespace benchmark {
 
+// map a label to one or multiple priority levels
+std::map<std::string, uint8_t> map_label_priority = {
+    {"high", PriorityLevel::HIGH},
+    {"low", PriorityLevel::LOW},
+    {"all", 0xff}
+};
+
+std::string ListPriorities()
+{
+    using item_t = std::pair<std::string, uint8_t>;
+    auto sort_by_priority = [](item_t a, item_t b){ return a.second < b.second; };
+    std::set<item_t, decltype(sort_by_priority)> sorted_priorities(map_label_priority.begin(), map_label_priority.end(), sort_by_priority);
+    return Join(sorted_priorities, ',', [](const auto& entry){ return entry.first; });
+}
+
+uint8_t StringToPriority(const std::string& str)
+{
+    auto it = map_label_priority.find(str);
+    if (it == map_label_priority.end()) throw std::runtime_error(strprintf("Unknown priority level %s", str));
+    return it->second;
+}
+
 BenchRunner::BenchmarkMap& BenchRunner::benchmarks()
 {
-    static std::map<std::string, BenchFunction> benchmarks_map;
+    static BenchmarkMap benchmarks_map;
     return benchmarks_map;
 }
 
-BenchRunner::BenchRunner(std::string name, BenchFunction func)
+BenchRunner::BenchRunner(std::string name, BenchFunction func, PriorityLevel level)
 {
-    benchmarks().insert(std::make_pair(name, func));
+    benchmarks().insert(std::make_pair(name, std::make_pair(func, level)));
 }
 
 void BenchRunner::RunAll(const Args& args)
@@ -64,33 +87,39 @@ void BenchRunner::RunAll(const Args& args)
     }
 
     std::vector<ankerl::nanobench::Result> benchmarkResults;
-    for (const auto& p : benchmarks()) {
-        if (!std::regex_match(p.first, baseMatch, reFilter)) {
+    for (const auto& [name, bench_func] : benchmarks()) {
+        const auto& [func, priority_level] = bench_func;
+
+        if (!(priority_level & args.priority)) {
+            continue;
+        }
+
+        if (!std::regex_match(name, baseMatch, reFilter)) {
             continue;
         }
 
         if (args.is_list_only) {
-            std::cout << p.first << std::endl;
+            std::cout << name << std::endl;
             continue;
         }
 
         Bench bench;
         if (args.sanity_check) {
             bench.epochs(1).epochIterations(1);
         }
-        bench.name(p.first);
+        bench.name(name);
         if (args.min_time > 0ms) {
             // convert to nanos before dividing to reduce rounding errors
             std::chrono::nanoseconds min_time_ns = args.min_time;
             bench.minEpochTime(min_time_ns / bench.epochs());
         }
 
         if (args.asymptote.empty()) {
-            p.second(bench);
+            func(bench);
         } else {
             for (auto n : args.asymptote) {
                 bench.complexityN(n);
-                p.second(bench);
+                func(bench);
             }
             std::cout << bench.complexityBigO() << std::endl;
         }

src/bench/bench.h

@@ -41,6 +41,16 @@ using ankerl::nanobench::Bench;
 
 typedef std::function<void(Bench&)> BenchFunction;
 
+enum PriorityLevel : uint8_t
+{
+    LOW = 1 << 0,
+    HIGH = 1 << 2,
+};
+
+// List priority labels, comma-separated and sorted by increasing priority
+std::string ListPriorities();
+uint8_t StringToPriority(const std::string& str);
+
 struct Args {
     bool is_list_only;
     bool sanity_check;
@@ -49,22 +59,24 @@ struct Args {
     fs::path output_csv;
     fs::path output_json;
     std::string regex_filter;
+    uint8_t priority;
 };
 
 class BenchRunner
 {
-    typedef std::map<std::string, BenchFunction> BenchmarkMap;
+    // maps from "name" -> (function, priority_level)
+    typedef std::map<std::string, std::pair<BenchFunction, PriorityLevel>> BenchmarkMap;
     static BenchmarkMap& benchmarks();
 
 public:
-    BenchRunner(std::string name, BenchFunction func);
+    BenchRunner(std::string name, BenchFunction func, PriorityLevel level);
 
     static void RunAll(const Args& args);
 };
 } // namespace benchmark
 
-// BENCHMARK(foo) expands to:  benchmark::BenchRunner bench_11foo("foo", foo);
+// BENCHMARK(foo) expands to:  benchmark::BenchRunner bench_11foo("foo", foo, priority_level);
 #define BENCHMARK(n) \
-    benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n);
+    benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n, benchmark::PriorityLevel::HIGH);
 
 #endif // BITCOIN_BENCH_BENCH_H

src/bench/bench_bitcoin.cpp

@@ -18,6 +18,8 @@
 
 static const char* DEFAULT_BENCH_FILTER = ".*";
 static constexpr int64_t DEFAULT_MIN_TIME_MS{10};
+/** Priority level default value, run "all" priority levels */
+static const std::string DEFAULT_PRIORITY{"all"};
 
 static void SetupBenchArgs(ArgsManager& argsman)
 {
@@ -30,6 +32,8 @@ static void SetupBenchArgs(ArgsManager& argsman)
     argsman.AddArg("-output-csv=<output.csv>", "Generate CSV file with the most important benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-output-json=<output.json>", "Generate JSON file with all benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-sanity-check", "Run benchmarks for only one iteration", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+    argsman.AddArg("-priority-level=<l1,l2,l3>", strprintf("Run benchmarks of one or multiple priority level(s) (%s), default: '%s'",
+                                                           benchmark::ListPriorities(), DEFAULT_PRIORITY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
 }
 
 // parses a comma separated list like "10,20,30,50"
@@ -45,6 +49,14 @@ static std::vector<double> parseAsymptote(const std::string& str) {
     return numbers;
 }
 
+static uint8_t parsePriorityLevel(const std::string& str) {
+    uint8_t levels{0};
+    for (const auto& level: SplitString(str, ',')) {
+        levels |= benchmark::StringToPriority(level);
+    }
+    return levels;
+}
+
 int main(int argc, char** argv)
 {
     ArgsManager argsman;
@@ -114,6 +126,7 @@ int main(int argc, char** argv)
     args.output_json = argsman.GetPathArg("-output-json");
     args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER);
     args.sanity_check = argsman.GetBoolArg("-sanity-check", false);
+    args.priority = parsePriorityLevel(argsman.GetArg("-priority-level", DEFAULT_PRIORITY));
 
     benchmark::BenchRunner::RunAll(args);