diff --git a/benchmarks/samplers/benchmark_samplers.py b/benchmarks/samplers/benchmark_samplers.py index c8c5a50d0..1ea363ed3 100644 --- a/benchmarks/samplers/benchmark_samplers.py +++ b/benchmarks/samplers/benchmark_samplers.py @@ -16,16 +16,22 @@ def bench(f, *args, num_exp=100, warmup=0, **kwargs): for _ in range(warmup): f(*args, **kwargs) + num_frames = None times = [] for _ in range(num_exp): start = perf_counter_ns() - f(*args, **kwargs) + clips = f(*args, **kwargs) end = perf_counter_ns() times.append(end - start) - return torch.tensor(times).float() + num_frames = ( + clips.data.shape[0] * clips.data.shape[1] + ) # should be constant across calls + return torch.tensor(times).float(), num_frames + +def report_stats(times, num_frames, unit="ms"): + fps = num_frames * 1e9 / torch.median(times) -def report_stats(times, unit="ms"): mul = { "ns": 1, "µs": 1e-3, @@ -35,13 +41,13 @@ def report_stats(times, unit="ms"): times = times * mul std = times.std().item() med = times.median().item() - print(f"{med = :.2f}{unit} +- {std:.2f}") - return med + print(f"{med = :.2f}{unit} +- {std:.2f} med fps = {fps:.1f}") + return med, fps def sample(sampler, **kwargs): decoder = VideoDecoder(VIDEO_PATH) - sampler( + return sampler( decoder, num_frames_per_clip=10, **kwargs, @@ -56,34 +62,34 @@ def sample(sampler, **kwargs): print(f"{num_clips = }") print("clips_at_random_indices ", end="") - times = bench( + times, num_frames = bench( sample, clips_at_random_indices, num_clips=num_clips, num_exp=NUM_EXP, warmup=2 ) - report_stats(times, unit="ms") + report_stats(times, num_frames, unit="ms") print("clips_at_regular_indices ", end="") - times = bench( + times, num_frames = bench( sample, clips_at_regular_indices, num_clips=num_clips, num_exp=NUM_EXP, warmup=2 ) - report_stats(times, unit="ms") + report_stats(times, num_frames, unit="ms") print("clips_at_random_timestamps ", end="") - times = bench( + times, num_frames = bench( sample, clips_at_random_timestamps, num_clips=num_clips, num_exp=NUM_EXP, warmup=2, ) - report_stats(times, unit="ms") + report_stats(times, num_frames, unit="ms") print("clips_at_regular_timestamps ", end="") seconds_between_clip_starts = 13 / num_clips # approximate. video is 13s long - times = bench( + times, num_frames = bench( sample, clips_at_regular_timestamps, seconds_between_clip_starts=seconds_between_clip_starts, num_exp=NUM_EXP, warmup=2, ) - report_stats(times, unit="ms") + report_stats(times, num_frames, unit="ms")