Merge pull request swiftlang#32723 from eeckstein/fix-bench-script

benchmarks: fix run_smoke_bench to support new benchmark executable naming scheme

Author: eeckstein

benchmark/scripts/Benchmark_Driver

@@ -73,7 +73,25 @@ class BenchmarkDriver(object):
     def test_harness(self):
         """Full path to test harness binary."""
         suffix = self.args.optimization if hasattr(self.args, "optimization") else "O"
-        return os.path.join(self.args.tests, "Benchmark_" + suffix)
+        if hasattr(self.args, "architecture") and self.args.architecture:
+            suffix += "-" + self.args.architecture + "*"
+        pattern = os.path.join(self.args.tests, "Benchmark_" + suffix)
+        executables = []
+        if hasattr(self._subprocess, "test_mode") and self._subprocess.test_mode:
+            executables = [pattern]
+        else:
+            executables = glob.glob(pattern)
+        if len(executables) == 0:
+            raise ValueError(
+                "No benchmark executable for file name pattern " +
+                pattern + " found")
+        if len(executables) > 1:
+            raise ValueError(
+                "Multiple benchmark executables for file name pattern " +
+                pattern + " found\n" +
+                str(executables) +
+                "\nSpecify the architecture to select the right benchmark executable")
+        return executables[0]
 
     def _git(self, cmd):
         """Execute the Git command in the `swift-repo`."""

benchmark/scripts/run_smoke_bench

@@ -48,12 +48,13 @@ VERBOSE = False
 class DriverArgs(object):
     """Arguments for BenchmarkDriver."""
 
-    def __init__(self, tests, optimization="O"):
+    def __init__(self, tests, optimization="O", architecture=None):
         """Initialize with path to the build-dir and optimization level."""
         self.benchmarks = None
         self.filters = None
         self.tests = os.path.join(tests, "bin")
         self.optimization = optimization
+        self.architecture = architecture
 
 
 def log(msg):
@@ -126,6 +127,12 @@ def main():
         help="The number of re-runs until it's assumed to be a real change",
         default=8,
     )
+    argparser.add_argument(
+        "-arch",
+        type=str,
+        help="The architecture. The default is x86_64",
+        default="x86_64",
+    )
     argparser.add_argument(
         "-platform", type=str, help="The benchmark build platform", default="macosx"
     )
@@ -158,6 +165,7 @@ def test_opt_levels(args):
                 args.num_samples,
                 args.num_reruns,
                 output_file,
+                args.arch
             ):
                 changes = True
 
@@ -167,6 +175,7 @@ def test_opt_levels(args):
                 opt_level,
                 args.oldbuilddir[0],
                 args.newbuilddir[0],
+                args.arch,
                 args.platform,
                 output_file,
             ):
@@ -177,6 +186,7 @@ def test_opt_levels(args):
             "swiftlibs",
             args.oldbuilddir[0],
             args.newbuilddir[0],
+            args.arch,
             args.platform,
             output_file,
         ):
@@ -221,7 +231,8 @@ def merge(results, other_results):
 
 
 def test_performance(
-    opt_level, old_dir, new_dir, threshold, num_samples, num_reruns, output_file
+    opt_level, old_dir, new_dir, threshold, num_samples, num_reruns,
+    output_file, arch
 ):
     """Detect performance changes in benchmarks.
 
@@ -231,7 +242,7 @@ def test_performance(
 
     i, unchanged_length_count = 0, 0
     old, new = [
-        BenchmarkDriver(DriverArgs(dir, optimization=opt_level))
+        BenchmarkDriver(DriverArgs(dir, optimization=opt_level, architecture=arch))
         for dir in [old_dir, new_dir]
     ]
     results = [measure(driver, driver.tests, i) for driver in [old, new]]
@@ -260,12 +271,13 @@ def test_performance(
     )
 
 
-def report_code_size(opt_level, old_dir, new_dir, platform, output_file):
+def report_code_size(opt_level, old_dir, new_dir, architecture, platform, output_file):
     if opt_level == "swiftlibs":
         files = glob.glob(os.path.join(old_dir, "lib", "swift", platform, "*.dylib"))
     else:
         files = glob.glob(
-            os.path.join(old_dir, opt_level + "-*" + platform + "*", "*.o")
+            os.path.join(old_dir, opt_level + "-" + architecture + "*" +
+                         platform + "*", "*.o")
         )
 
     idx = 1
@@ -370,8 +382,8 @@ performance team (@eeckstein).
 
 
 def check_added(args, output_file=None):
-    old = BenchmarkDriver(DriverArgs(args.oldbuilddir[0]))
-    new = BenchmarkDriver(DriverArgs(args.newbuilddir[0]))
+    old = BenchmarkDriver(DriverArgs(args.oldbuilddir[0], architecture=args.arch))
+    new = BenchmarkDriver(DriverArgs(args.newbuilddir[0], architecture=args.arch))
     added = set(new.tests).difference(set(old.tests))
     new.tests = list(added)
     doctor = BenchmarkDoctor(args, driver=new)

benchmark/scripts/test_Benchmark_Driver.py

@@ -198,6 +198,9 @@ def record_and_respond(self, args, stdin, stdout, stderr, shell):
         self.calls.append(args)
         return self.respond.get(args, "")
 
+    def test_mode():
+        return True
+
 
 class TestBenchmarkDriverInitialization(unittest.TestCase):
     def setUp(self):
@@ -206,13 +209,19 @@ def setUp(self):
 
     def test_test_harness(self):
         self.assertEqual(
-            BenchmarkDriver(self.args, tests=["ignored"]).test_harness,
+            BenchmarkDriver(
+                self.args,
+                tests=["ignored"],
+                _subprocess=self.subprocess_mock).test_harness,
             "/benchmarks/Benchmark_O",
         )
         self.args.tests = "/path"
         self.args.optimization = "Suffix"
         self.assertEqual(
-            BenchmarkDriver(self.args, tests=["ignored"]).test_harness,
+            BenchmarkDriver(
+                self.args,
+                tests=["ignored"],
+                _subprocess=self.subprocess_mock).test_harness,
             "/path/Benchmark_Suffix",
         )
 
@@ -271,6 +280,7 @@ def test_log_file(self):
                 swift_repo=None,
             ),
             tests=["ignored"],
+            _subprocess=self.subprocess_mock
         )
         self.assertEqual(driver.log_file, "/path/Benchmark_Suffix-" + now + ".log")