Merge pull request swiftlang#68718 from tbkka/tbkka-build-script-benchmarks

Fix build-script -B

Author: tbkka

benchmark/cmake/modules/AddSwiftBenchmarkSuite.cmake

@@ -751,6 +751,26 @@ function(swift_benchmark_compile)
         DEPENDS ${platform_executables})
 
     if(NOT SWIFT_BENCHMARK_BUILT_STANDALONE AND "${SWIFT_BENCHMARK_COMPILE_PLATFORM}" STREQUAL "macosx")
+      set(SWIFT_BENCHMARK_ARGS)
+      list(APPEND SWIFT_BENCHMARK_ARGS "--output-dir")
+      list(APPEND SWIFT_BENCHMARK_ARGS "${CMAKE_CURRENT_BINARY_DIR}/logs")
+      list(APPEND SWIFT_BENCHMARK_ARGS "--swift-repo")
+      list(APPEND SWIFT_BENCHMARK_ARGS "${SWIFT_SOURCE_DIR}")
+      list(APPEND SWIFT_BENCHMARK_ARGS "--architecture")
+      list(APPEND SWIFT_BENCHMARK_ARGS "${arch}")
+
+      set(SWIFT_O_BENCHMARK_ARGS)
+      if(DEFINED SWIFT_BENCHMARK_NUM_O_ITERATIONS)
+        list(APPEND SWIFT_O_BENCHMARK_ARGS "--independent-samples")
+        list(APPEND SWIFT_O_BENCHMARK_ARGS "${SWIFT_BENCHMARK_NUM_O_ITERATIONS}")
+      endif()
+
+      set(SWIFT_ONONE_BENCHMARK_ARGS)
+      if(DEFINED SWIFT_BENCHMARK_NUM_ONONE_ITERATIONS)
+        list(APPEND SWIFT_O_BENCHMARK_ARGS "--independent-samples")
+        list(APPEND SWIFT_O_BENCHMARK_ARGS "${SWIFT_BENCHMARK_NUM_ONONE_ITERATIONS}")
+      endif()
+
       add_custom_command(
           TARGET "${executable_target}"
           POST_BUILD
@@ -759,15 +779,13 @@ function(swift_benchmark_compile)
 
       add_custom_target("check-${executable_target}"
           COMMAND "${swift-bin-dir}/Benchmark_Driver" "run"
-                  "-o" "O" "--output-dir" "${CMAKE_CURRENT_BINARY_DIR}/logs"
-                  "--architecture" "${arch}"
-                  "--swift-repo" "${SWIFT_SOURCE_DIR}"
-                  "--independent-samples" "${SWIFT_BENCHMARK_NUM_O_ITERATIONS}"
+                  "-o" "O"
+		  ${SWIFT_BENCHMARK_ARGS}
+		  ${SWIFT_O_BENCHMARK_ARGS}
           COMMAND "${swift-bin-dir}/Benchmark_Driver" "run"
-                  "-o" "Onone" "--output-dir" "${CMAKE_CURRENT_BINARY_DIR}/logs"
-                  "--swift-repo" "${SWIFT_SOURCE_DIR}"
-                  "--architecture" "${arch}"
-                  "--independent-samples" "${SWIFT_BENCHMARK_NUM_ONONE_ITERATIONS}"
+                  "-o" "Onone"
+		  ${SWIFT_BENCHMARK_ARGS}
+		  ${SWIFT_ONONE_BENCHMARK_ARGS}
           COMMAND "${swift-bin-dir}/Benchmark_Driver" "compare"
                   "--log-dir" "${CMAKE_CURRENT_BINARY_DIR}/logs"
                   "--swift-repo" "${SWIFT_SOURCE_DIR}"

benchmark/scripts/Benchmark_Driver

@@ -226,6 +226,8 @@ class BenchmarkDriver(object):
             cmd.extend([str(self.test_number.get(name, name)) for name in self.tests])
         if num_samples > 0:
             cmd.append("--num-samples={0}".format(num_samples))
+        else:
+            cmd.append("--min-samples=2")
         if num_iters > 0:
             cmd.append("--num-iters={0}".format(num_iters))
         if sample_time > 0:

benchmark/scripts/test_Benchmark_Driver.py

@@ -329,7 +329,7 @@ def setUp(self):
     def test_run_benchmark_with_multiple_samples(self):
         self.driver.run("b1")
         self.subprocess_mock.assert_called_with(
-            ("/benchmarks/Benchmark_O-*", "b1")
+            ("/benchmarks/Benchmark_O-*", "b1", "--min-samples=2")
         )
         self.driver.run("b2", num_samples=5)
         self.subprocess_mock.assert_called_with(
@@ -339,19 +339,19 @@ def test_run_benchmark_with_multiple_samples(self):
     def test_run_benchmark_with_specified_number_of_iterations(self):
         self.driver.run("b", num_iters=1)
         self.subprocess_mock.assert_called_with(
-            ("/benchmarks/Benchmark_O-*", "b", "--num-iters=1")
+            ("/benchmarks/Benchmark_O-*", "b", "--min-samples=2", "--num-iters=1")
         )
 
     def test_run_benchmark_for_specified_time(self):
         self.driver.run("b", sample_time=0.5)
         self.subprocess_mock.assert_called_with(
-            ("/benchmarks/Benchmark_O-*", "b", "--sample-time=0.5")
+            ("/benchmarks/Benchmark_O-*", "b", "--min-samples=2", "--sample-time=0.5")
         )
 
     def test_run_benchmark_in_verbose_mode(self):
         self.driver.run("b", verbose=True)
         self.subprocess_mock.assert_called_with(
-            ("/benchmarks/Benchmark_O-*", "b", "--verbose")
+            ("/benchmarks/Benchmark_O-*", "b", "--min-samples=2", "--verbose")
         )
 
     def test_run_batch(self):
@@ -363,7 +363,7 @@ def test_run_batch(self):
         self.driver.tests = ["b1", "bx"]
         self.driver.run()
         self.subprocess_mock.assert_called_with(
-            ("/benchmarks/Benchmark_O-*", "1", "bx")
+            ("/benchmarks/Benchmark_O-*", "1", "bx", "--min-samples=2")
         )
 
     def test_parse_results_from_running_benchmarks(self):
@@ -382,7 +382,7 @@ def test_parse_results_from_running_benchmarks(self):
     def test_measure_memory(self):
         self.driver.run("b", measure_memory=True)
         self.subprocess_mock.assert_called_with(
-            ("/benchmarks/Benchmark_O-*", "b", "--memory")
+            ("/benchmarks/Benchmark_O-*", "b", "--min-samples=2", "--memory")
         )
 
     def test_run_benchmark_independent_samples(self):
@@ -394,6 +394,7 @@ def test_run_benchmark_independent_samples(self):
                 (
                     "/benchmarks/Benchmark_O-*",
                     "b1",
+                    "--min-samples=2",
                     "--num-iters=1",
                     "--memory",
                 )