[leaks-checker] Add support for filtering tests and changing the number of samples/iters run. This will make it easy to bisect using the Benchmark_LeaksRunner script.

Author: gottesmm

benchmark/scripts/Benchmark_RuntimeLeaksRunner.in

@@ -76,25 +76,30 @@ class LeaksRunnerResult(perf_test_driver.Result):
 
 class LeaksRunnerBenchmarkDriver(perf_test_driver.BenchmarkDriver):
 
-    def __init__(self, binary, xfail_list):
+    def __init__(self, binary, xfail_list, num_samples, num_iters):
         perf_test_driver.BenchmarkDriver.__init__(
             self, binary, xfail_list,
             enable_parallel=True)
+        self.num_samples = num_samples
+        self.num_iters = num_iters
 
     def print_data_header(self, max_test_len):
         fmt = '{:<%d}{:<10}{:}' % (max_test_len + 5)
         print(fmt.format('Name', 'Result', 'RC Delta'))
 
+    # Propagate any data from this class that is needed for individual
+    # tests. The reason this is needed is to avoid issues with attempting to
+    # access a value in a different process.
     def prepare_input(self, name):
-        return {}
+        return {'num_samples': self.num_samples, 'num_iters': self.num_iters}
 
     def process_input(self, data):
         test_name = '({},{})'.format(data['opt'], data['test_name'])
         print("Running {}...".format(test_name))
         sys.stdout.flush()
         try:
-            p = subprocess.Popen([data['path'], "--run-all", "--num-samples=2",
-                                  "--num-iters={}".format(2), data['test_name']],
+            p = subprocess.Popen([data['path'], "--run-all", "--num-samples={}".format(data['num_samples']),
+                                  "--num-iters={}".format(data['num_iters']), data['test_name']],
                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE)
             error_out = p.communicate()[1].split("\n")
         except OSError:
@@ -121,9 +126,19 @@ class LeaksRunnerBenchmarkDriver(perf_test_driver.BenchmarkDriver):
 
 SWIFT_BIN_DIR = os.path.dirname(os.path.abspath(__file__))
 
+def parse_args():
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-filter', type=str, default=None,
+                        help='Filter out any test that does not match the given regex')
+    parser.add_argument('-num-samples', type=int, default=2)
+    parser.add_argument('-num-iters', type=int, default=2)
+    return parser.parse_args()
+
 if __name__ == "__main__":
-    l = LeaksRunnerBenchmarkDriver(SWIFT_BIN_DIR, XFAIL_LIST)
-    if l.run():
+    args = parse_args()
+    l = LeaksRunnerBenchmarkDriver(SWIFT_BIN_DIR, XFAIL_LIST, args.num_samples, args.num_iters)
+    if l.run(args.filter):
         sys.exit(0)
     else:
         sys.exit(-1)