Standardize benchmark warm-up steps

Use step terminology across runtime and play interfaces. Record the excluded count in environment-step timing so schema and flat formatter consumers share one contract.

Author: AntoineRichard

docs/source/testing/benchmarks.rst

@@ -149,7 +149,7 @@ Measure environment stepping performance without any RL library:
        --task Isaac-Cartpole \
        --num_envs 4096 \
        --num_frames 1000 \
-       --warmup_frames 50 \
+       --warmup_steps 50 \
        --benchmark_formatter json \
        --output_path ./results
 
@@ -218,7 +218,9 @@ Environment-Step Timing Semantics
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Runtime, training, and play benchmarks report
-:class:`~isaaclab.benchmark.EnvironmentStepTiming`. The default
+:class:`~isaaclab.benchmark.EnvironmentStepTiming`. Its ``warmup_steps`` field
+records the exact number of initial ``env.step()`` calls excluded from
+timing. The default
 ``host_return`` measurement mode records wall time until ``env.step()`` returns
 without forcing queued device work to complete. It describes the host-visible
 call boundary, not a device-complete boundary; asynchronously queued work can be
@@ -413,7 +415,7 @@ Non-RL / Runtime Benchmark Arguments
    * - ``--num_frames``
      - ``1000``
      - Number of environment steps to measure
-   * - ``--warmup_frames``
+   * - ``--warmup_steps``
      - ``50``
      - Exact number of environment steps to exclude from timing; zero measures the first step
    * - ``--measure_sync_step``
@@ -474,7 +476,7 @@ RL Play Arguments
    * - ``--num_frames``
      - ``100``
      - Number of measured inference steps
-   * - ``--warmup_frames``
+   * - ``--warmup_steps``
      - ``1``
      - Number of preceding environment steps to exclude from timing and throughput
    * - ``--checkpoint``
@@ -487,15 +489,16 @@ RL Play Arguments
      - ``false``
      - Collect the serialized synchronized diagnostic described above
 
-Runtime and play execute ``warmup_frames + num_frames`` environment steps. Thus,
+Runtime and play execute ``warmup_steps + num_frames`` environment steps. Thus,
 the requested ``num_frames`` is always the exact number of measured steps. Play
-warmup frames are excluded only from timing and throughput; they still contribute
+warm-up steps are excluded only from timing and throughput; they still contribute
 to reward, episode-length, success-rate, and resource measurements.
 
-Runtime warmup frames are excluded from steady-state timing, throughput, and
-synchronized environment-step measurements. When warmup is nonzero, the first
-warmup frame ordinary wall-clock time is retained separately as the ``first_step`` startup diagnostic.
-With zero warmup, the first measured frame supplies that diagnostic.
+Runtime warm-up steps are excluded from steady-state timing, throughput, and
+synchronized environment-step measurements. When warm-up is nonzero, the first
+warm-up step's ordinary wall-clock time is retained separately as the
+``first_step`` startup diagnostic. With zero warm-up, the first measured step
+supplies that diagnostic.
 
 Measurement Types
 -----------------

scripts/benchmarks/test/test_benchmark_smoke.py

@@ -62,7 +62,7 @@ def test_adapters_reject_non_positive_workloads(library: str, workflow: str, arg
 def test_adapters_reject_negative_warmup_steps(library: str, workflow: str, monkeypatch, capsys):
     """Benchmark adapters reject negative warm-up counts."""
     module = _load_adapter(library, workflow)
-    warmup_argument = "--warmup_frames" if workflow == "play" else "--warmup_steps"
+    warmup_argument = "--warmup_steps"
     argv = ["--task", _TASK, warmup_argument, "-1", "--headless"]
     monkeypatch.setattr(sys, "argv", ["benchmark", *argv])
 
@@ -83,7 +83,7 @@ def test_adapters_default_to_one_warmup_step(library: str, workflow: str, monkey
 
     args = module._parse_args(argv)[0]
 
-    warmup_count = args.warmup_frames if workflow == "play" else args.warmup_steps
+    warmup_count = args.warmup_steps
     assert warmup_count == 1
 
 
@@ -104,13 +104,13 @@ def test_adapters_accept_short_synchronized_step_flag(library: str, workflow: st
 def test_play_adapters_accept_warmup_larger_than_measured_workload(library: str, monkeypatch):
     """Play warm-up adds calls without consuming the measured workload."""
     module = _load_adapter(library, "play")
-    argv = ["--task", _TASK, "--num_frames", "2", "--warmup_frames", "3", "--headless"]
+    argv = ["--task", _TASK, "--num_frames", "2", "--warmup_steps", "3", "--headless"]
     monkeypatch.setattr(sys, "argv", ["benchmark", *argv])
 
     args = module._parse_args(argv)[0]
 
     assert args.num_frames == 2
-    assert args.warmup_frames == 3
+    assert args.warmup_steps == 3
 
 
 @pytest.mark.parametrize(
@@ -179,6 +179,7 @@ def test_training_and_play_write_bundles(
     assert training_data["runtime"]["total_fps"]["mean"] > 0
     training_timing = training_data["runtime"]["environment_step_timing"]
     assert training_timing["environment_step_calls"] > 0
+    assert training_timing["warmup_steps"] == 1
     assert training_timing["environment_step_fps"]["mean"] > 0
     assert training_timing["simulation_step_calls"] is None
     assert training_timing["simulation_step_time_s"] is None
@@ -213,6 +214,7 @@ def test_training_and_play_write_bundles(
     assert play_data["runtime"]["total_fps"]["mean"] > 0
     play_timing = play_data["runtime"]["environment_step_timing"]
     assert play_timing["environment_step_calls"] == 250
+    assert play_timing["warmup_steps"] == 1
     assert play_timing["environment_step_fps"]["mean"] > 0
     assert play_timing["simulation_step_calls"] is None
     assert play_timing["simulation_step_time_s"] is None
@@ -244,6 +246,7 @@ def test_training_and_play_write_bundles(
         synchronized_play_data = _load_play_bundle(synchronized_play_output)
         synchronized_timing = synchronized_play_data["runtime"]["environment_step_timing"]
         assert synchronized_timing["environment_step_calls"] == 10
+        assert synchronized_timing["warmup_steps"] == 1
         assert synchronized_timing["simulation_step_calls"] > 0
         assert synchronized_timing["simulation_step_time_s"]["mean"] > 0.0
         assert synchronized_timing["outside_simulation_step_time_s"]["mean"] >= 0.0
@@ -259,6 +262,8 @@ def test_training_and_play_write_bundles(
         assert play_omniperf["runtime"]["Mean Total FPS"] == pytest.approx(play_data["runtime"]["total_fps"]["mean"])
         assert training_omniperf["benchmark_info"]["environment_step_measurement_mode"] == "host_return"
         assert play_omniperf["benchmark_info"]["environment_step_measurement_mode"] == "host_return"
+        assert training_omniperf["benchmark_info"]["environment_step_warmup_steps"] == 1
+        assert play_omniperf["benchmark_info"]["environment_step_warmup_steps"] == 1
         if library == "rsl_rl":
             synchronized_omniperf = json.loads(next(synchronized_play_output.glob("*_omniperf.json")).read_text())
             assert (

scripts/benchmarks/test/test_runtime_smoke.py

@@ -29,7 +29,7 @@ def test_runtime_writes_all_requested_formats(tmp_path, measure_sync_step: bool)
         "16",
         "--num_frames",
         "20",
-        "--warmup_frames",
+        "--warmup_steps",
         "0",
         "--seed",
         "0",
@@ -59,9 +59,10 @@ def test_runtime_writes_all_requested_formats(tmp_path, measure_sync_step: bool)
     schema_data = json.loads(schema_files[0].read_text())
     assert schema_data["run"]["config"]["physics_backend"] == "newton_mjwarp"
     assert schema_data["runtime"]["iterations_completed"] == 20
-    assert schema_data["extra"]["warmup_frames"] == 0
+    assert schema_data["extra"] is None
     assert schema_data["runtime"]["startup_time_s"]["first_step"] > 0.0
     timing = schema_data["runtime"]["environment_step_timing"]
+    assert timing["warmup_steps"] == 0
     assert timing["environment_step_calls"] == 20
     assert timing["environment_step_fps"]["mean"] > 0
     if measure_sync_step:
@@ -79,6 +80,7 @@ def test_runtime_writes_all_requested_formats(tmp_path, measure_sync_step: bool)
     assert timing["environment_step_time_s"]["std"] >= 0.0
     omniperf_data = json.loads(omniperf_files[0].read_text())
     assert omniperf_data["benchmark_info"]["environment_step_measurement_mode"] == timing["measurement_mode"]
+    assert omniperf_data["benchmark_info"]["environment_step_warmup_steps"] == 0
     if measure_sync_step:
         assert "Mean Serialized Diagnostic Total FPS" in omniperf_data["runtime"]
         assert "Mean Total FPS" not in omniperf_data["runtime"]

source/isaaclab/changelog.d/antoiner-benchmark-warmup-steps.major.rst

@@ -0,0 +1,10 @@
+Changed
+^^^^^^^
+
+* **Breaking:** Renamed ``--warmup_frames`` to ``--warmup_steps`` for runtime
+  and play benchmarks, and renamed ``warmup_frames`` to ``warmup_steps`` in
+  :class:`~isaaclab.benchmark.BenchmarkRuntimeRequest` and
+  :class:`~isaaclab.benchmark.BenchmarkPlayRequest`. Use the new step-based
+  names when invoking or configuring benchmarks. Benchmark bundles now record
+  the excluded count in
+  :attr:`~isaaclab.benchmark.EnvironmentStepTiming.warmup_steps`.

source/isaaclab/isaaclab/benchmark/api.py

@@ -84,7 +84,7 @@ class BenchmarkRuntimeRequest:
         task: Registered Gym task identifier.
         num_envs: Number of parallel environments.
         num_frames: Number of measured environment steps.
-        warmup_frames: Number of warm-up steps excluded from throughput measurements.
+        warmup_steps: Number of warm-up steps excluded from throughput measurements.
         seed: Environment seed.
         measure_synchronized_step_breakdown: Whether to collect serialized synchronized
             environment/simulation step diagnostics.
@@ -97,7 +97,7 @@ class BenchmarkRuntimeRequest:
     task: str
     num_envs: int | None = None
     num_frames: int = 1000
-    warmup_frames: int = 50
+    warmup_steps: int = 50
     seed: int | None = None
     measure_synchronized_step_breakdown: bool = False
     presets: tuple[str, ...] = field(default_factory=tuple)
@@ -224,7 +224,7 @@ class BenchmarkPlayRequest:
         agent: Optional task agent configuration entry point.
         num_envs: Number of parallel environments.
         num_frames: Number of measured inference steps.
-        warmup_frames: Number of initial environment steps excluded from environment-step timing.
+        warmup_steps: Number of initial environment steps excluded from environment-step timing.
         seed: Environment seed.
         measure_synchronized_step_breakdown: Whether to collect serialized synchronized
             environment/simulation step diagnostics.
@@ -241,7 +241,7 @@ class BenchmarkPlayRequest:
     agent: str | None = None
     num_envs: int | None = None
     num_frames: int = 100
-    warmup_frames: int = 1
+    warmup_steps: int = 1
     seed: int | None = None
     measure_synchronized_step_breakdown: bool = False
     presets: tuple[str, ...] = field(default_factory=tuple)

source/isaaclab/isaaclab/benchmark/builders.py

@@ -132,6 +132,7 @@ def build_runtime(
     steps_per_iteration: int,
     aggregate_throughput: bool = False,
     frames_per_environment_step: int | None = None,
+    environment_step_warmup_steps: int = 0,
     environment_step_times_s: Sequence[float] | None = None,
     simulation_step_times_s: Sequence[float] | None = None,
     simulation_step_calls: int | None = None,
@@ -150,6 +151,7 @@ def build_runtime(
             remains the ordinary sample deviation of the per-iteration rates,
             and peak remains the maximum per-iteration throughput.
         frames_per_environment_step: Number of environment frames processed by each vectorized ``env.step()`` call.
+        environment_step_warmup_steps: Number of initial environment-step calls excluded from timing.
         environment_step_times_s: Positive per-environment-step wall times [s].
         simulation_step_times_s: Synchronized simulation wall times per environment step [s].
         simulation_step_calls: Number of measured simulation-step calls.
@@ -230,6 +232,7 @@ def build_runtime(
             environment_step_calls=len(environment_samples),
             simulation_step_calls=simulation_step_calls,
             measurement_mode=("serialized_synchronized" if simulation_step_times_s is not None else "host_return"),
+            warmup_steps=environment_step_warmup_steps,
         )
 
     return Runtime(

source/isaaclab/isaaclab/benchmark/dispatch.py

@@ -130,7 +130,7 @@ def _request_argv(request: BenchmarkRequest) -> list[str]:
 
     if request.workflow == "runtime":
         _append_value(argv, "--num_frames", request.num_frames)
-        _append_value(argv, "--warmup_frames", request.warmup_frames)
+        _append_value(argv, "--warmup_steps", request.warmup_steps)
     elif request.workflow == "startup":
         _append_value(argv, "--top_n", request.top_n)
         _append_value(argv, "--whitelist_config", request.whitelist_config)
@@ -163,7 +163,7 @@ def _request_argv(request: BenchmarkRequest) -> list[str]:
         _append_value(argv, "--checkpoint", request.checkpoint)
         _append_value(argv, "--agent", request.agent)
         _append_value(argv, "--num_frames", request.num_frames)
-        _append_value(argv, "--warmup_frames", request.warmup_frames)
+        _append_value(argv, "--warmup_steps", request.warmup_steps)
 
     if getattr(request, "measure_synchronized_step_breakdown", False):
         argv.append("--measure_sync_step")

source/isaaclab/isaaclab/benchmark/entrypoints/backends/rl_games/benchmark_play_rl_games.py

@@ -66,7 +66,7 @@ def _parse_args(argv: list[str]):
         help="Measure a serialized synchronized simulation and outside-simulation step breakdown.",
     )
     parser.add_argument(
-        "--warmup_frames",
+        "--warmup_steps",
         type=parse_non_negative_int,
         default=1,
         help="Exclude the first N env.step() calls from environment-step timing. Default 1 removes cold start.",
@@ -175,7 +175,7 @@ def run(argv: list[str]) -> BenchmarkResult:
                             "name": "environment_step_measurement_mode",
                             "data": ("serialized_synchronized" if args_cli.measure_sync_step else "host_return"),
                         },
-                        {"name": "environment_step_warmup_frames", "data": args_cli.warmup_frames},
+                        {"name": "environment_step_warmup_steps", "data": args_cli.warmup_steps},
                         {"name": "presets", "data": ",".join(cfg.presets)},
                     ]
                 },
@@ -243,14 +243,14 @@ def policy(obs):
             environment_step_timer = stepping.EnvironmentStepTimingRecorder(
                 env,
                 measure_synchronized_step_breakdown=args_cli.measure_sync_step,
-                warmup_steps=args_cli.warmup_frames,
+                warmup_steps=args_cli.warmup_steps,
             )
-            total_frames = args_cli.warmup_frames + args_cli.num_frames
+            total_frames = args_cli.warmup_steps + args_cli.num_frames
             with environment_step_timer, BenchmarkMonitor(benchmark, interval=1.0):
                 all_step_times, reward, ep_length, success_rate = stepping.run_play_loop(env, policy, total_frames)
 
             first_step_s = all_step_times[0]
-            step_times = all_step_times[args_cli.warmup_frames :]
+            step_times = all_step_times[args_cli.warmup_steps :]
 
             benchmark.update_manual_recorders()
 
@@ -268,6 +268,7 @@ def policy(obs):
                 total_fps=fps,
                 steps_per_iteration=num_envs,
                 frames_per_environment_step=env.unwrapped.num_envs,
+                environment_step_warmup_steps=args_cli.warmup_steps,
                 environment_step_times_s=environment_step_timer.step_times_s,
                 simulation_step_times_s=environment_step_timer.simulation_step_times_s,
                 simulation_step_calls=environment_step_timer.simulation_step_calls,

source/isaaclab/isaaclab/benchmark/entrypoints/backends/rl_games/benchmark_train_rl_games.py

@@ -306,6 +306,7 @@ def run(argv: list[str]) -> BenchmarkResult:
                 total_fps=total_fps_series,
                 steps_per_iteration=steps_per_iteration,
                 frames_per_environment_step=env.unwrapped.num_envs,
+                environment_step_warmup_steps=args_cli.warmup_steps,
                 environment_step_times_s=environment_step_timer.step_times_s,
                 simulation_step_times_s=environment_step_timer.simulation_step_times_s,
                 simulation_step_calls=environment_step_timer.simulation_step_calls,

source/isaaclab/isaaclab/benchmark/entrypoints/backends/rsl_rl/benchmark_play_rsl_rl.py

@@ -67,7 +67,7 @@ def _parse_args(argv: list[str]) -> tuple[argparse.Namespace, list[str]]:
         help="Measure a serialized synchronized simulation and outside-simulation step breakdown.",
     )
     parser.add_argument(
-        "--warmup_frames",
+        "--warmup_steps",
         type=parse_non_negative_int,
         default=1,
         help="Exclude the first N env.step() calls from environment-step timing. Default 1 removes cold start.",
@@ -172,7 +172,7 @@ def run(argv: list[str]) -> BenchmarkResult:
                             "name": "environment_step_measurement_mode",
                             "data": ("serialized_synchronized" if args.measure_sync_step else "host_return"),
                         },
-                        {"name": "environment_step_warmup_frames", "data": args.warmup_frames},
+                        {"name": "environment_step_warmup_steps", "data": args.warmup_steps},
                         {"name": "presets", "data": ",".join(cfg.presets)},
                     ]
                 },
@@ -200,14 +200,14 @@ def run(argv: list[str]) -> BenchmarkResult:
             environment_step_timer = stepping.EnvironmentStepTimingRecorder(
                 env,
                 measure_synchronized_step_breakdown=args.measure_sync_step,
-                warmup_steps=args.warmup_frames,
+                warmup_steps=args.warmup_steps,
             )
-            total_frames = args.warmup_frames + args.num_frames
+            total_frames = args.warmup_steps + args.num_frames
             with environment_step_timer, BenchmarkMonitor(benchmark, interval=1.0):
                 all_step_times, reward, ep_length, success_rate = stepping.run_play_loop(env, policy, total_frames)
 
             first_step_s = all_step_times[0]
-            step_times = all_step_times[args.warmup_frames :]
+            step_times = all_step_times[args.warmup_steps :]
 
             benchmark.update_manual_recorders()
 
@@ -225,6 +225,7 @@ def run(argv: list[str]) -> BenchmarkResult:
                 total_fps=fps,
                 steps_per_iteration=num_envs,
                 frames_per_environment_step=env.unwrapped.num_envs,
+                environment_step_warmup_steps=args.warmup_steps,
                 environment_step_times_s=environment_step_timer.step_times_s,
                 simulation_step_times_s=environment_step_timer.simulation_step_times_s,
                 simulation_step_calls=environment_step_timer.simulation_step_calls,