[benchmark] Recommended runtime should be < 1ms

* Lowered the threshold for healthy benchmark runtime to be under 1000 μs.
* Offer suitable divisor that is power of 10, in addition to the one that’s power of 2.
* Expanded the motivation in the docstring.

Author: palimondo

benchmark/scripts/Benchmark_Driver

@@ -347,15 +347,20 @@ class BenchmarkDoctor(object):
               for correction in [(setup / i) for i in [1, 2]]
               ] for result in i_series])
 
-        if 2500 < runtime:
-            log = (BenchmarkDoctor.log_runtime.warning if runtime < 500000 else
+        threshold = 1000
+        if threshold < runtime:
+            log = (BenchmarkDoctor.log_runtime.warning if runtime < 10000 else
                    BenchmarkDoctor.log_runtime.error)
             caveat = '' if setup == 0 else ' (excluding the setup overhead)'
             log("'%s' execution took at least %d μs%s.", name, runtime, caveat)
-            factor = int(pow(2, math.ceil(math.log(runtime / 2500.0, 2))))
+
+            def factor(base):  # suitable divisior that's integer power of base
+                return int(pow(base, math.ceil(
+                    math.log(runtime / float(threshold), base))))
+
             BenchmarkDoctor.log_runtime.info(
-                "Decrease the workload of '%s' by a factor of %d, "
-                "to be less than 2500 μs.", name, factor)
+                "Decrease the workload of '%s' by a factor of %d (%d), to be "
+                "less than %d μs.", name, factor(2), factor(10), threshold)
 
     @staticmethod
     def _setup_overhead(measurements):

benchmark/scripts/test_Benchmark_Driver.py

@@ -548,13 +548,18 @@ def test_benchmark_name_is_at_most_40_chars_long(self):
             self.logs['info'])
 
     def test_benchmark_runtime_range(self):
-        """Optimized benchmark should run in less then 2500 μs.
-
-        With runtimes less than 2500 μs there is better than 1:4 chance of
-        being interrupted in the middle of measurement due to elapsed 10 ms
-        quantum used by macos scheduler.
-
-        Warn about longer runtime. Runtimes over half a second are an error.
+        """Optimized benchmark should run in less then 1000 μs.
+
+        Even on calm machine, benchmark with runtime of 2500 μs has 1:4 chance
+        of being interrupted in the middle of measurement due to elapsed 10 ms
+        quantum used by macos scheduler. Linux scheduler's quantum is 6ms.
+        Driver yielding the process before the 10ms quantum elapses helped
+        a lot, but benchmarks with runtimes under 1ms usually exhibit a strong
+        mode which is best for accurate performance charaterization.
+        To minimize the number of involuntary context switches that corrupt our
+        measurements, we should strive to stay in the microbenchmark range.
+
+        Warn about longer runtime. Runtimes over 10ms are an error.
         """
         def measurements(name, runtime):
             return {'name': name,
@@ -564,7 +569,7 @@ def measurements(name, runtime):
         with captured_output() as (out, _):
             doctor = BenchmarkDoctor(self.args, BenchmarkDriverMock([]))
             doctor.analyze(measurements('Cheetah', 200))
-            doctor.analyze(measurements('Hare', 2501))
+            doctor.analyze(measurements('Hare', 1001))
             doctor.analyze(measurements('Tortoise', 500000))
             doctor.analyze({'name': 'OverheadTurtle',
                             'OverheadTurtle O i1a': _PTR(min=800000),
@@ -573,17 +578,17 @@ def measurements(name, runtime):
 
         self.assertIn('runtime: ', output)
         self.assertNotIn('Cheetah', output)
-        self.assert_contains(["'Hare' execution took at least 2501 μs."],
+        self.assert_contains(["'Hare' execution took at least 1001 μs."],
                              self.logs['warning'])
         self.assert_contains(
-            ["Decrease the workload of 'Hare' by a factor of 2, "
-             "to be less than 2500 μs."], self.logs['info'])
+            ["Decrease the workload of 'Hare' by a factor of 2 (10), "
+             "to be less than 1000 μs."], self.logs['info'])
         self.assert_contains(
             ["'Tortoise' execution took at least 500000 μs."],
             self.logs['error'])
         self.assert_contains(
-            ["Decrease the workload of 'Tortoise' by a factor of 256, "
-             "to be less than 2500 μs."], self.logs['info'])
+            ["Decrease the workload of 'Tortoise' by a factor of 512 (1000), "
+             "to be less than 1000 μs."], self.logs['info'])
         self.assert_contains(
             ["'OverheadTurtle' execution took at least 600000 μs"
              " (excluding the setup overhead)."],