[benchmark] Run benchmarks using substring filters

Added support for running benchmarks using substring filters. Positional arguments prefixed with a single + or - sign are interpreted as benchmark name filters.

Excecutes all benchmarks whose names include any of the strings prefixed with a plus sign but none of the strings prefixed with a minus sign.

Author: palimondo

benchmark/utils/ArgParse.swift

@@ -112,7 +112,8 @@ class ArgumentParser<U> {
           .split(separator: "\n")
           .joined(separator: "\n" + padded(""))
        }
-      let positional = f("TEST", "name or number of the benchmark to measure")
+      let positional = f("TEST", "name or number of the benchmark to measure;\n"
+                         + "use +/- prefix to filter by substring match")
       let optional =  arguments.filter { $0.name != nil }
                                 .map { f($0.name!, $0.help ?? "") }
                                 .joined(separator: "\n")
@@ -152,7 +153,6 @@ class ArgumentParser<U> {
     /// We assume that optional switch args are of the form:
     ///
     ///     --opt-name[=opt-value]
-    ///     -opt-name[=opt-value]
     ///
     /// with `opt-name` and `opt-value` not containing any '=' signs. Any
     /// other option passed in is assumed to be a positional argument.
@@ -165,7 +165,7 @@ class ArgumentParser<U> {
       for arg in CommandLine.arguments[1..<CommandLine.arguments.count] {
         // If the argument doesn't match the optional argument pattern. Add
         // it to the positional argument list and continue...
-        if !arg.starts(with: "-") {
+        if !arg.starts(with: "--") {
           positionalArgs.append(arg)
           continue
         }

benchmark/utils/DriverUtils.swift

@@ -194,7 +194,7 @@ struct TestConfig {
     afterRunSleep = c.afterRunSleep
     action = c.action ?? .run
     tests = TestConfig.filterTests(registeredBenchmarks,
-                                    specifiedTests: Set(c.tests ?? []),
+                                    tests: c.tests ?? [],
                                     tags: c.tags ?? [],
                                     skipTags: c.skipTags ?? [.unstable, .skip])
 
@@ -241,10 +241,16 @@ struct TestConfig {
   ///     specified filtering conditions.
   static func filterTests(
     _ registeredBenchmarks: [BenchmarkInfo],
-    specifiedTests: Set<String>,
+    tests: [String],
     tags: Set<BenchmarkCategory>,
     skipTags: Set<BenchmarkCategory>
   ) -> [(index: String, info: BenchmarkInfo)] {
+    var t = tests
+    let filtersIndex = t.partition { $0.hasPrefix("+") || $0.hasPrefix("-") }
+    let excludesIndex = t[filtersIndex...].partition { $0.hasPrefix("-") }
+    let specifiedTests = Set(t[..<filtersIndex])
+    let includes = t[filtersIndex..<excludesIndex].map { $0.dropFirst() }
+    let excludes = t[excludesIndex...].map { $0.dropFirst() }
     let allTests = registeredBenchmarks.sorted()
     let indices = Dictionary(uniqueKeysWithValues:
       zip(allTests.map { $0.name },
@@ -256,14 +262,30 @@ struct TestConfig {
     }
     func byNamesOrIndices(b: BenchmarkInfo) -> Bool {
       return specifiedTests.contains(b.name) ||
-        specifiedTests.contains(indices[b.name]!)
-    } // !! "`allTests` have been assigned an index"
+        // !! "`allTests` have been assigned an index"
+        specifiedTests.contains(indices[b.name]!) ||
+        (includes.contains { b.name.contains($0) } &&
+          excludes.allSatisfy { !b.name.contains($0) } )
+    }
     return allTests
-      .filter(specifiedTests.isEmpty ? byTags : byNamesOrIndices)
+      .filter(tests.isEmpty ? byTags : byNamesOrIndices)
       .map { (index: indices[$0.name]!, info: $0) }
   }
 }
 
+extension String {
+  func contains(_ str: Substring) -> Bool {
+    guard let c = str.first else { return false }
+    var s = self[...]
+    repeat {
+      s = s[(s.firstIndex(of: c) ?? s.endIndex)...]
+      if s.starts(with: str) { return true }
+      s = s.dropFirst()
+    } while s.startIndex != s.endIndex
+    return false
+  }
+}
+
 struct Stats {
     var n: Int = 0
     var S: Double = 0.0

test/benchmark/Benchmark_O.test.md

@@ -100,6 +100,18 @@ ALPHASORT: FatCompactMap
 
 ````
 
+Substring filters using + and - prefix
+
+````
+RUN: %Benchmark_O --list -.A +Angry -Small AngryPhonebook.ASCII.Small \
+RUN:             | %FileCheck %s --check-prefix FILTERS
+FILTERS: AngryPhonebook.ASCII.Small
+FILTERS-NOT: AngryPhonebook.Armenian
+FILTERS-NOT: AngryPhonebook.Cyrillic.Small
+FILTERS: AngryPhonebook.Cyrillic
+FILTERS: AngryPhonebook.Strasse
+````
+
 ## Running Benchmarks
 By default, each real benchmark execution takes about a second per sample.
 To minimise the test time, multiple checks are combined into one run.