Add configurable iteration modes to parsl-perf (#3985)

These are described in the documentation added in this PR.

The default mode is the previous behaviour (estimate mode).

This PR adds a new mode, exponential, which is useful for looking at
scaling behaviour over batch sizes and for more consistent batch sizes
across runs.

# Changed Behaviour

new options, old behaviour should be preserved

## Type of change

- New feature

Author: benclifford

docs/userguide/advanced/parsl_perf.rst

@@ -6,8 +6,9 @@ Measuring performance with parsl-perf
 ``parsl-perf`` is tool for making basic performance measurements of Parsl
 configurations.
 
-It runs increasingly large numbers of no-op apps until a batch takes
-(by default) 120 seconds, giving a measurement of tasks per second.
+It runs repeated batches of no-op apps, giving a measurement of tasks per
+second. By default, ``parsl-perf`` will attempt to estimate a batch size
+that will take 2 minutes to execute.
 
 This can give a basic measurement of some of the overheads in task
 execution.
@@ -23,6 +24,20 @@ argument.
 To change the target runtime from the default of 120 seconds, add a
 ``--time`` parameter.
 
+To change the iteration algorithm used to pick the next batch size and to
+decide when to stop, use the ``--iterate`` parameter:
+
+ * ``--iterate=estimate`` (the default) will run a small batch, then
+   repeatedly re-estimate a new batch size until it finds one that takes at least
+   75% of the target run time. This will rapidly find a batch size of around
+   the target size.
+
+ * ``--iterate=exponential`` will start with a small batch, then double the size
+   in each subsequent batch until reaching the target run time. This is useful
+   for understanding how a configuration scales down as batch sizes increase,
+   which is sometimes sub-linear, and to get more consistent batch sizes across
+   runs of ``parsl-perf``.
+
 For example:
 
 .. code-block:: bash

parsl/benchmark/perf.py

@@ -6,6 +6,7 @@
 
 import parsl
 from parsl.dataflow.dflow import DataFlowKernel
+from parsl.errors import InternalConsistencyError
 
 min_iterations = 2
 
@@ -40,19 +41,18 @@ def app(extra_payload: Any, parsl_resource_specification: Dict = {}) -> int:
     return 7
 
 
-def performance(*, resources: dict, target_t: float, args_extra_size: int) -> None:
+def performance(*, resources: dict, target_t: float, args_extra_size: int, iterate_mode: str) -> None:
     n = 10
 
     delta_t: float
-    delta_t = 0
-
-    threshold_t = int(0.75 * target_t)
 
     iteration = 1
 
     args_extra_payload = "x" * args_extra_size
 
-    while delta_t < threshold_t or iteration <= min_iterations:
+    iterate = True
+
+    while iterate:
         print(f"==== Iteration {iteration} ====")
         print(f"Will run {n} tasks to target {target_t} seconds runtime")
         start_t = time.time()
@@ -78,10 +78,20 @@ def performance(*, resources: dict, target_t: float, args_extra_size: int) -> No
         print(f"Runtime: actual {delta_t:.3f}s vs target {target_t}s")
         print(f"Tasks per second: {rate:.3f}")
 
-        n = max(1, int(target_t * rate))
-
         iteration += 1
 
+        # decide upon next iteration
+
+        match iterate_mode:
+            case "estimate":
+                n = max(1, int(target_t * rate))
+                iterate = delta_t < (0.75 * target_t) or iteration <= min_iterations
+            case "exponential":
+                n = int(n * 2)
+                iterate = delta_t < target_t or iteration <= min_iterations
+            case _:
+                raise InternalConsistencyError(f"Bad iterate mode {iterate_mode} - should have been validated at arg parse time")
+
 
 def cli_run() -> None:
     parser = argparse.ArgumentParser(
@@ -96,6 +106,12 @@ def cli_run() -> None:
     parser.add_argument("--time", metavar="SECONDS", help="target number of seconds for an iteration", default=120, type=float)
     parser.add_argument("--argsize", metavar="BYTES", help="extra bytes to add into app invocation arguments", default=0, type=int)
     parser.add_argument("--version", action="version", version=f"parsl-perf from Parsl {parsl.__version__}")
+    parser.add_argument("--iterate",
+                        metavar="MODE",
+                        help="Iteration mode: estimate, exponential",
+                        type=str,
+                        default="estimate",
+                        choices=("estimate", "exponential"))
 
     args = parser.parse_args()
 
@@ -105,7 +121,7 @@ def cli_run() -> None:
         resources = {}
 
     with load_dfk_from_config(args.config):
-        performance(resources=resources, target_t=args.time, args_extra_size=args.argsize)
+        performance(resources=resources, target_t=args.time, args_extra_size=args.argsize, iterate_mode=args.iterate)
         print("Tests complete - leaving DFK block")
     print("The end")