Skip to content

Commit 9e6ff29

Browse files
ahmadsharif1ahmadsharif1
committed
Merge branch 'main' of https://github.com/pytorch/torchcodec into cuda15
2 parents c25eaa7 + aa9b872 commit 9e6ff29

File tree

9 files changed

+277
-99
lines changed

9 files changed

+277
-99
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ The instructions below assume you're on Linux.
120120

121121
## Benchmark Results
122122

123-
The following was generated by running [our benchmark script](./benchmarks/decoders/generate_readme_chart.py) on a lightly loaded 22-core machine.
123+
The following was generated by running [our benchmark script](./benchmarks/decoders/generate_readme_data.py) on a lightly loaded 56-core machine.
124124

125125
![benchmark_results](./benchmarks/decoders/benchmark_readme_chart.png)
126126

‎benchmarks/decoders/benchmark_decoders_library.py‎

Lines changed: 24 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import os
44
import subprocess
55
import timeit
6-
from concurrent.futures import ThreadPoolExecutor
6+
from concurrent.futures import ThreadPoolExecutor, wait
77
from itertools import product
88

99
import matplotlib.pyplot as plt
@@ -306,6 +306,7 @@ def generate_video(command):
306306
print(command)
307307
print(" ".join(command))
308308
subprocess.check_call(command)
309+
return True
309310

310311

311312
def generate_videos(
@@ -321,6 +322,7 @@ def generate_videos(
321322
executor = ThreadPoolExecutor(max_workers=20)
322323
video_count = 0
323324

325+
futures = []
324326
for resolution, duration, fps, gop_size, encoding, pix_fmt in product(
325327
resolutions, durations, fpses, gop_sizes, encodings, pix_fmts
326328
):
@@ -342,9 +344,12 @@ def generate_videos(
342344
pix_fmt,
343345
outfile,
344346
]
345-
executor.submit(generate_video, command)
347+
futures.append(executor.submit(generate_video, command))
346348
video_count += 1
347349

350+
wait(futures)
351+
for f in futures:
352+
assert f.result()
348353
executor.shutdown(wait=True)
349354
print(f"Generated {video_count} videos")
350355

@@ -442,21 +447,21 @@ def plot_data(df_data, plot_path):
442447

443448
def run_benchmarks(
444449
decoder_dict,
445-
video_paths,
450+
video_files_paths,
446451
num_uniform_samples,
447452
min_runtime_seconds,
448453
benchmark_video_creation,
449-
):
454+
) -> list[dict[str, str | float | int]]:
450455
results = []
451456
df_data = []
452-
print(f"video_paths={video_paths}")
457+
print(f"video_files_paths={video_files_paths}")
453458
verbose = False
454459
for decoder_name, decoder in decoder_dict.items():
455-
for video_path in video_paths:
456-
print(f"video={video_path}, decoder={decoder_name}")
460+
for video_file_path in video_files_paths:
461+
print(f"video={video_file_path}, decoder={decoder_name}")
457462
# We only use the VideoDecoder to get the metadata and get
458463
# the list of PTS values to seek to.
459-
simple_decoder = VideoDecoder(video_path)
464+
simple_decoder = VideoDecoder(video_file_path)
460465
duration = simple_decoder.metadata.duration_seconds
461466
pts_list = [
462467
i * duration / num_uniform_samples for i in range(num_uniform_samples)
@@ -465,16 +470,16 @@ def run_benchmarks(
465470
metadata_string = f"{metadata.codec} {metadata.width}x{metadata.height}, {metadata.duration_seconds}s {metadata.average_fps}fps"
466471
if verbose:
467472
print(
468-
f"video={video_path}, decoder={decoder_name}, pts_list={pts_list}"
473+
f"video={video_file_path}, decoder={decoder_name}, pts_list={pts_list}"
469474
)
470475
seeked_result = benchmark.Timer(
471476
stmt="decoder.get_frames_from_video(video_file, pts_list)",
472477
globals={
473-
"video_file": video_path,
478+
"video_file": video_file_path,
474479
"pts_list": pts_list,
475480
"decoder": decoder,
476481
},
477-
label=f"video={video_path} {metadata_string}",
482+
label=f"video={video_file_path} {metadata_string}",
478483
sub_label=decoder_name,
479484
description=f"{num_uniform_samples} seek()+next()",
480485
)
@@ -483,7 +488,7 @@ def run_benchmarks(
483488
)
484489
df_item = {}
485490
df_item["decoder"] = decoder_name
486-
df_item["video"] = video_path
491+
df_item["video"] = video_file_path
487492
df_item["description"] = results[-1].description
488493
df_item["frame_count"] = num_uniform_samples
489494
df_item["median"] = results[-1].median
@@ -498,11 +503,11 @@ def run_benchmarks(
498503
consecutive_frames_result = benchmark.Timer(
499504
stmt="decoder.get_consecutive_frames_from_video(video_file, consecutive_frames_to_extract)",
500505
globals={
501-
"video_file": video_path,
506+
"video_file": video_file_path,
502507
"consecutive_frames_to_extract": num_consecutive_nexts,
503508
"decoder": decoder,
504509
},
505-
label=f"video={video_path} {metadata_string}",
510+
label=f"video={video_file_path} {metadata_string}",
506511
sub_label=decoder_name,
507512
description=f"{num_consecutive_nexts} next()",
508513
)
@@ -513,7 +518,7 @@ def run_benchmarks(
513518
)
514519
df_item = {}
515520
df_item["decoder"] = decoder_name
516-
df_item["video"] = video_path
521+
df_item["video"] = video_file_path
517522
df_item["description"] = results[-1].description
518523
df_item["frame_count"] = num_consecutive_nexts
519524
df_item["median"] = results[-1].median
@@ -524,18 +529,18 @@ def run_benchmarks(
524529
df_item["fps_p25"] = 1.0 * num_consecutive_nexts / results[-1]._p25
525530
df_data.append(df_item)
526531

527-
first_video_path = video_paths[0]
532+
first_video_file_path = video_files_paths[0]
528533
if benchmark_video_creation:
529-
simple_decoder = VideoDecoder(first_video_path)
534+
simple_decoder = VideoDecoder(first_video_file_path)
530535
metadata = simple_decoder.metadata
531536
metadata_string = f"{metadata.codec} {metadata.width}x{metadata.height}, {metadata.duration_seconds}s {metadata.average_fps}fps"
532537
creation_result = benchmark.Timer(
533538
stmt="create_torchcodec_decoder_from_file(video_file)",
534539
globals={
535-
"video_file": first_video_path,
540+
"video_file": first_video_file_path,
536541
"create_torchcodec_decoder_from_file": create_torchcodec_decoder_from_file,
537542
},
538-
label=f"video={first_video_path} {metadata_string}",
543+
label=f"video={first_video_file_path} {metadata_string}",
539544
sub_label="TorchcodecNonCompiled",
540545
description="create()+next()",
541546
)

‎benchmarks/decoders/benchmark_readme_chart.png‎

3.23 KB
Loading

‎benchmarks/decoders/benchmark_readme_data.json‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
[
2+
{
3+
"decoder": "TorchCodec",
4+
"description": "10 seek()+next()",
5+
"fps": 296.7497929852154,
6+
"fps_p25": 304.82592121401444,
7+
"fps_p75": 286.39866868882336,
8+
"frame_count": 10,
9+
"iqr": 0.0021107543725520372,
10+
"median": 0.03369842283427715,
11+
"type": "seek()+next()",
12+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
13+
},
14+
{
15+
"decoder": "TorchCodec",
16+
"description": "1 next()",
17+
"fps": 141.80053650468176,
18+
"fps_p25": 144.12265279456386,
19+
"fps_p75": 128.28507218641042,
20+
"frame_count": 1,
21+
"iqr": 0.0008566047297790648,
22+
"median": 0.007052159495651722,
23+
"type": "next()",
24+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
25+
},
26+
{
27+
"decoder": "TorchCodec",
28+
"description": "10 next()",
29+
"fps": 638.7876251007046,
30+
"fps_p25": 647.626546889273,
31+
"fps_p75": 629.6538548699413,
32+
"frame_count": 10,
33+
"iqr": 0.00044074421748518944,
34+
"median": 0.015654655173420906,
35+
"type": "next()",
36+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
37+
},
38+
{
39+
"decoder": "TorchCodec[num_threads=1]",
40+
"description": "10 seek()+next()",
41+
"fps": 126.6773946375974,
42+
"fps_p25": 128.33021600580943,
43+
"fps_p75": 124.2963412180317,
44+
"frame_count": 10,
45+
"iqr": 0.00252892030403018,
46+
"median": 0.07894068257883191,
47+
"type": "seek()+next()",
48+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
49+
},
50+
{
51+
"decoder": "TorchCodec[num_threads=1]",
52+
"description": "1 next()",
53+
"fps": 410.70043288059617,
54+
"fps_p25": 416.39979027639464,
55+
"fps_p75": 405.8298526819803,
56+
"frame_count": 1,
57+
"iqr": 6.25486485660077e-05,
58+
"median": 0.0024348647333681584,
59+
"type": "next()",
60+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
61+
},
62+
{
63+
"decoder": "TorchCodec[num_threads=1]",
64+
"description": "10 next()",
65+
"fps": 758.7565583035099,
66+
"fps_p25": 766.9872077345758,
67+
"fps_p75": 751.3583938952637,
68+
"frame_count": 10,
69+
"iqr": 0.00027120066806674004,
70+
"median": 0.013179457746446133,
71+
"type": "next()",
72+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
73+
},
74+
{
75+
"decoder": "TorchVision[backend=VideoReader]",
76+
"description": "10 seek()+next()",
77+
"fps": 7.880664295730362,
78+
"fps_p25": 7.924876540397429,
79+
"fps_p75": 7.827668314297991,
80+
"frame_count": 10,
81+
"iqr": 0.01567032840102911,
82+
"median": 1.2689285604283214,
83+
"type": "seek()+next()",
84+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
85+
},
86+
{
87+
"decoder": "TorchVision[backend=VideoReader]",
88+
"description": "1 next()",
89+
"fps": 209.3834723742803,
90+
"fps_p25": 211.77546088016425,
91+
"fps_p75": 199.25812670019334,
92+
"frame_count": 1,
93+
"iqr": 0.0002966334810480479,
94+
"median": 0.00477592614479363,
95+
"type": "next()",
96+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
97+
},
98+
{
99+
"decoder": "TorchVision[backend=VideoReader]",
100+
"description": "10 next()",
101+
"fps": 531.7221665034224,
102+
"fps_p25": 538.7259880033903,
103+
"fps_p75": 522.8300241450709,
104+
"frame_count": 10,
105+
"iqr": 0.0005643628537654877,
106+
"median": 0.018806814216077328,
107+
"type": "next()",
108+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
109+
},
110+
{
111+
"decoder": "TorchAudio",
112+
"description": "10 seek()+next()",
113+
"fps": 27.403418366590216,
114+
"fps_p25": 27.57563496650987,
115+
"fps_p75": 27.276339015442346,
116+
"frame_count": 10,
117+
"iqr": 0.003979140194132924,
118+
"median": 0.36491797724738717,
119+
"type": "seek()+next()",
120+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
121+
},
122+
{
123+
"decoder": "TorchAudio",
124+
"description": "1 next()",
125+
"fps": 243.36030386646544,
126+
"fps_p25": 246.1182470856226,
127+
"fps_p75": 240.91573311529928,
128+
"frame_count": 1,
129+
"iqr": 8.774134330451558e-05,
130+
"median": 0.004109133593738079,
131+
"type": "next()",
132+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
133+
},
134+
{
135+
"decoder": "TorchAudio",
136+
"description": "10 next()",
137+
"fps": 564.5307153840273,
138+
"fps_p25": 572.2932075367765,
139+
"fps_p75": 558.1691247911658,
140+
"frame_count": 10,
141+
"iqr": 0.00044215633533895016,
142+
"median": 0.01771382801234722,
143+
"type": "next()",
144+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
145+
},
146+
{
147+
"cpu_count": 56,
148+
"machine": "x86_64",
149+
"processor": "x86_64",
150+
"python_version": "3.11.10",
151+
"system": "Linux"
152+
}
153+
]

‎benchmarks/decoders/generate_readme_chart.py‎

Lines changed: 7 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -4,81 +4,20 @@
44
# This source code is licensed under the BSD-style license found in the
55
# LICENSE file in the root directory of this source tree.
66

7-
import glob
8-
import importlib.resources
9-
import os
10-
import shutil
11-
from pathlib import Path
12-
13-
from benchmark_decoders_library import (
14-
generate_videos,
15-
plot_data,
16-
run_benchmarks,
17-
TorchAudioDecoder,
18-
TorchcodecNonCompiledWithOptions,
19-
TVNewAPIDecoderWithBackend,
20-
)
21-
22-
23-
def in_fbcode() -> bool:
24-
return "FB_PAR_RUNTIME_FILES" in os.environ
7+
import json
258

9+
from pathlib import Path
2610

27-
def get_test_resource_path(filename: str) -> str:
28-
if in_fbcode():
29-
resource = importlib.resources.files(__package__).joinpath(filename)
30-
with importlib.resources.as_file(resource) as path:
31-
return os.fspath(path)
32-
33-
return str(Path(__file__).parent / f"../../test/resources/{filename}")
11+
from benchmark_decoders_library import plot_data
3412

3513

3614
def main() -> None:
37-
"""Benchmarks the performance of a few video decoders on synthetic videos"""
38-
39-
resolutions = ["640x480"]
40-
encodings = ["libx264"]
41-
fpses = [30]
42-
gop_sizes = [600]
43-
durations = [10]
44-
pix_fmts = ["yuv420p"]
45-
ffmpeg_path = "/usr/local/bin/ffmpeg"
46-
videos_path = "/tmp/videos"
47-
shutil.rmtree(videos_path)
48-
os.makedirs(videos_path)
49-
generate_videos(
50-
resolutions,
51-
encodings,
52-
fpses,
53-
gop_sizes,
54-
durations,
55-
pix_fmts,
56-
ffmpeg_path,
57-
videos_path,
58-
)
59-
video_paths = glob.glob(f"{videos_path}/*.mp4")
60-
61-
decoder_dict = {}
62-
decoder_dict["TorchCodec"] = TorchcodecNonCompiledWithOptions()
63-
decoder_dict["TorchCodec[num_threads=1]"] = TorchcodecNonCompiledWithOptions(
64-
num_threads=1
65-
)
66-
decoder_dict["TorchVision[backend=VideoReader]"] = TVNewAPIDecoderWithBackend(
67-
"video_reader"
68-
)
69-
decoder_dict["TorchAudio"] = TorchAudioDecoder()
15+
data_json = Path(__file__).parent / "benchmark_readme_data.json"
16+
with open(data_json, "r") as read_file:
17+
data_from_file = json.load(read_file)
7018

7119
output_png = Path(__file__).parent / "benchmark_readme_chart.png"
72-
# These are the number of uniform seeks we do in the seek+decode benchmark.
73-
num_uniform_samples = 10
74-
df_data = run_benchmarks(
75-
decoder_dict,
76-
video_paths,
77-
num_uniform_samples,
78-
10,
79-
False,
80-
)
81-
plot_data(df_data, output_png)
20+
plot_data(data_from_file, output_png)
8221

8322

8423
if __name__ == "__main__":

0 commit comments

Comments
 (0)