Skip to content

Commit c4681c0

Browse files
committed
Generation new chart, add more next benchmarks
1 parent 79589f2 commit c4681c0

File tree

5 files changed

+92
-81
lines changed

5 files changed

+92
-81
lines changed

benchmarks/decoders/benchmark_decoders.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -145,8 +145,9 @@ def main() -> None:
145145
decoder_dict,
146146
video_paths,
147147
num_uniform_samples,
148-
args.bm_video_speed_min_run_seconds,
149-
args.bm_video_creation,
148+
num_sequential_frames_from_start=[1, 10, 100],
149+
min_runtime_seconds=args.bm_video_speed_min_run_seconds,
150+
benchmark_video_creation=args.bm_video_creation,
150151
)
151152
plot_data(df_data, args.plot_path)
152153

benchmarks/decoders/benchmark_decoders_library.py

Lines changed: 26 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -472,28 +472,37 @@ def get_metadata(video_file_path: str) -> VideoStreamMetadata:
472472
return VideoDecoder(video_file_path).metadata
473473

474474
def run_benchmarks(
475-
decoder_dict,
476-
video_files_paths,
477-
num_samples,
478-
min_runtime_seconds,
479-
benchmark_video_creation,
475+
decoder_dict: dict[str, AbstractDecoder],
476+
video_files_paths: list[str],
477+
num_samples: int,
478+
num_sequential_frames_from_start: list[int],
479+
min_runtime_seconds: float,
480+
benchmark_video_creation: bool,
480481
) -> list[dict[str, str | float | int]]:
482+
# Ensure that we have the same seed across benchmark runs.
483+
torch.manual_seed(0)
484+
485+
print(f"video_files_paths={video_files_paths}")
486+
481487
results = []
482488
df_data = []
483-
print(f"video_files_paths={video_files_paths}")
484489
verbose = False
485-
for decoder_name, decoder in decoder_dict.items():
486-
for video_file_path in video_files_paths:
487-
print(f"video={video_file_path}, decoder={decoder_name}")
488-
metadata = get_metadata(video_file_path)
489-
metadata_label = f"{metadata.codec} {metadata.width}x{metadata.height}, {metadata.duration_seconds}s {metadata.average_fps}fps"
490+
for video_file_path in video_files_paths:
491+
metadata = get_metadata(video_file_path)
492+
metadata_label = f"{metadata.codec} {metadata.width}x{metadata.height}, {metadata.duration_seconds}s {metadata.average_fps}fps"
490493

491-
duration = metadata.duration_seconds
492-
uniform_pts_list = [
493-
i * duration / num_samples for i in range(num_samples)
494-
]
494+
duration = metadata.duration_seconds
495+
uniform_pts_list = [
496+
i * duration / num_samples for i in range(num_samples)
497+
]
495498

496-
random_pts_list = (torch.rand(num_samples) * duration).tolist()
499+
# Note that we are using the same random pts values for all decoders for the same
500+
# video. However, because we use the duration as part of this calculation, we
501+
# are using different random pts values across videos.
502+
random_pts_list = (torch.rand(num_samples) * duration).tolist()
503+
504+
for decoder_name, decoder in decoder_dict.items():
505+
print(f"video={video_file_path}, decoder={decoder_name}")
497506

498507
for kind, pts_list in [("uniform", uniform_pts_list), ("random", random_pts_list)]:
499508
if verbose:
@@ -527,7 +536,7 @@ def run_benchmarks(
527536
df_item["fps_p25"] = 1.0 * num_samples / results[-1]._p25
528537
df_data.append(df_item)
529538

530-
for num_consecutive_nexts in [100]:
539+
for num_consecutive_nexts in num_sequential_frames_from_start:
531540
consecutive_frames_result = benchmark.Timer(
532541
stmt="decoder.get_consecutive_frames_from_video(video_file, consecutive_frames_to_extract)",
533542
globals={
137 Bytes
Loading

benchmarks/decoders/benchmark_readme_data.json

Lines changed: 60 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -2,144 +2,144 @@
22
{
33
"decoder": "TorchCodec",
44
"description": "uniform 10 seek()+next()",
5-
"fps": 314.3002734250253,
6-
"fps_p25": 322.3704500761085,
7-
"fps_p75": 304.61961295439414,
5+
"fps": 315.05924655387844,
6+
"fps_p25": 323.6567367722293,
7+
"fps_p75": 303.4217234978234,
88
"frame_count": 10,
9-
"iqr": 0.001807614229619503,
10-
"median": 0.03181670792400837,
9+
"iqr": 0.0020604978781193495,
10+
"median": 0.031740061938762665,
1111
"type": "uniform:seek()+next()",
1212
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
1313
},
1414
{
1515
"decoder": "TorchCodec",
1616
"description": "random 10 seek()+next()",
17-
"fps": 318.6332392061074,
18-
"fps_p25": 326.61265301425016,
19-
"fps_p75": 307.9272223416659,
17+
"fps": 312.8775910187949,
18+
"fps_p25": 316.65411586172576,
19+
"fps_p75": 304.67664202566044,
2020
"frame_count": 10,
21-
"iqr": 0.0018578986637294292,
22-
"median": 0.031384045258164406,
21+
"iqr": 0.0012414834462106256,
22+
"median": 0.03196138134226203,
2323
"type": "random:seek()+next()",
2424
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
2525
},
2626
{
2727
"decoder": "TorchCodec",
2828
"description": "100 next()",
29-
"fps": 963.8588500259038,
30-
"fps_p25": 1232.7462278510554,
31-
"fps_p75": 897.72507501799,
29+
"fps": 948.3281062356014,
30+
"fps_p25": 1211.2939721426944,
31+
"fps_p75": 884.5162407819654,
3232
"frame_count": 100,
33-
"iqr": 0.03027298115193844,
34-
"median": 0.10374963097274303,
33+
"iqr": 0.030499806627631187,
34+
"median": 0.10544873587787151,
3535
"type": "next()",
3636
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
3737
},
3838
{
3939
"decoder": "TorchCodec[num_threads=1]",
4040
"description": "uniform 10 seek()+next()",
41-
"fps": 131.988339357579,
42-
"fps_p25": 133.26652650624064,
43-
"fps_p75": 130.3643453501947,
41+
"fps": 131.3991915256709,
42+
"fps_p25": 132.80657677770856,
43+
"fps_p75": 130.0875105195256,
4444
"frame_count": 10,
45-
"iqr": 0.0016704928129911423,
46-
"median": 0.07576426863670349,
45+
"iqr": 0.0015738545916974545,
46+
"median": 0.07610396901145577,
4747
"type": "uniform:seek()+next()",
4848
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
4949
},
5050
{
5151
"decoder": "TorchCodec[num_threads=1]",
5252
"description": "random 10 seek()+next()",
53-
"fps": 122.75317946016999,
54-
"fps_p25": 124.22201311180707,
55-
"fps_p75": 121.19633847694604,
53+
"fps": 131.76952597486445,
54+
"fps_p25": 133.43827127306128,
55+
"fps_p75": 129.72933699334547,
5656
"frame_count": 10,
57-
"iqr": 0.0020097135566174984,
58-
"median": 0.08146428503096104,
57+
"iqr": 0.0021425478626042604,
58+
"median": 0.0758900810033083,
5959
"type": "random:seek()+next()",
6060
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
6161
},
6262
{
6363
"decoder": "TorchCodec[num_threads=1]",
6464
"description": "100 next()",
65-
"fps": 809.9441914595772,
66-
"fps_p25": 820.5478107543202,
67-
"fps_p75": 801.8920090548505,
65+
"fps": 767.4262194030912,
66+
"fps_p25": 954.5964481242703,
67+
"fps_p75": 751.3156802763621,
6868
"frame_count": 100,
69-
"iqr": 0.0028352681547403336,
70-
"median": 0.12346529681235552,
69+
"iqr": 0.028343535726889968,
70+
"median": 0.13030568603426218,
7171
"type": "next()",
7272
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
7373
},
7474
{
7575
"decoder": "TorchVision[backend=video_reader]",
7676
"description": "uniform 10 seek()+next()",
77-
"fps": 7.818673617539736,
78-
"fps_p25": 7.914887876132339,
79-
"fps_p75": 7.72635851907541,
77+
"fps": 7.87802293808524,
78+
"fps_p25": 7.9221952370727795,
79+
"fps_p75": 7.816352619742458,
8080
"frame_count": 10,
81-
"iqr": 0.0308289946988225,
82-
"median": 1.2789893131703138,
81+
"iqr": 0.017092708498239517,
82+
"median": 1.2693540090695024,
8383
"type": "uniform:seek()+next()",
8484
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
8585
},
8686
{
8787
"decoder": "TorchVision[backend=video_reader]",
8888
"description": "random 10 seek()+next()",
89-
"fps": 8.174175797820915,
90-
"fps_p25": 8.198128262246657,
91-
"fps_p75": 8.089292632708272,
89+
"fps": 7.257307771543773,
90+
"fps_p25": 7.305370582206469,
91+
"fps_p75": 7.209328679013935,
9292
"frame_count": 10,
93-
"iqr": 0.016411407850682735,
94-
"median": 1.2233649296686053,
93+
"iqr": 0.018235752126201987,
94+
"median": 1.3779214434325695,
9595
"type": "random:seek()+next()",
9696
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
9797
},
9898
{
9999
"decoder": "TorchVision[backend=video_reader]",
100100
"description": "100 next()",
101-
"fps": 708.5626871155907,
102-
"fps_p25": 714.9065385245668,
103-
"fps_p75": 701.1653993140255,
101+
"fps": 843.6237746060216,
102+
"fps_p25": 852.9895625098609,
103+
"fps_p75": 830.8058002517034,
104104
"frame_count": 100,
105-
"iqr": 0.002741277450695634,
106-
"median": 0.14113077335059643,
105+
"iqr": 0.0031303432770073414,
106+
"median": 0.11853625159710646,
107107
"type": "next()",
108108
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
109109
},
110110
{
111111
"decoder": "TorchAudio",
112112
"description": "uniform 10 seek()+next()",
113-
"fps": 28.25481341811952,
114-
"fps_p25": 28.472751002542807,
115-
"fps_p75": 28.027683266448527,
113+
"fps": 28.283386183212908,
114+
"fps_p25": 28.48861769505288,
115+
"fps_p75": 27.962052080094207,
116116
"frame_count": 10,
117-
"iqr": 0.005577113479375839,
118-
"median": 0.35392199736088514,
117+
"iqr": 0.006610161624848843,
118+
"median": 0.3535644542425871,
119119
"type": "uniform:seek()+next()",
120120
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
121121
},
122122
{
123123
"decoder": "TorchAudio",
124124
"description": "random 10 seek()+next()",
125-
"fps": 29.222482794708707,
126-
"fps_p25": 29.434989975510586,
127-
"fps_p75": 28.911024515041184,
125+
"fps": 26.009247898010745,
126+
"fps_p25": 26.13538377314808,
127+
"fps_p75": 25.757236346453418,
128128
"frame_count": 10,
129-
"iqr": 0.006157086929306388,
130-
"median": 0.34220227180048823,
129+
"iqr": 0.00561736966483295,
130+
"median": 0.38447863003239036,
131131
"type": "random:seek()+next()",
132132
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
133133
},
134134
{
135135
"decoder": "TorchAudio",
136136
"description": "100 next()",
137-
"fps": 719.0499213961119,
138-
"fps_p25": 725.4279593070022,
139-
"fps_p75": 710.8406389605324,
137+
"fps": 659.7600283811723,
138+
"fps_p25": 668.0071480761926,
139+
"fps_p75": 652.052492189031,
140140
"frame_count": 100,
141-
"iqr": 0.002828843891620636,
142-
"median": 0.13907240238040686,
141+
"iqr": 0.003662889124825597,
142+
"median": 0.15157026145607233,
143143
"type": "next()",
144144
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
145145
},

benchmarks/decoders/generate_readme_data.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,9 @@ def main() -> None:
6161
decoder_dict,
6262
video_files_paths,
6363
num_samples,
64-
30,
65-
False,
64+
num_sequential_frames_from_start=[100],
65+
min_runtime_seconds=30,
66+
benchmark_video_creation=False,
6667
)
6768
df_data.append(
6869
{

0 commit comments

Comments
 (0)