Skip to content

Commit aa9b872

Browse files
scottsscotts
authored
Seperate out readme experiments from chart generation (#328)
1 parent b839529 commit aa9b872

File tree

6 files changed

+245
-70
lines changed

6 files changed

+245
-70
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ The instructions below assume you're on Linux.
120120

121121
## Benchmark Results
122122

123-
The following was generated by running [our benchmark script](./benchmarks/decoders/generate_readme_chart.py) on a lightly loaded 22-core machine.
123+
The following was generated by running [our benchmark script](./benchmarks/decoders/generate_readme_data.py) on a lightly loaded 56-core machine.
124124

125125
![benchmark_results](./benchmarks/decoders/benchmark_readme_chart.png)
126126

‎benchmarks/decoders/benchmark_decoders_library.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -451,7 +451,7 @@ def run_benchmarks(
451451
num_uniform_samples,
452452
min_runtime_seconds,
453453
benchmark_video_creation,
454-
):
454+
) -> list[dict[str, str | float | int]]:
455455
results = []
456456
df_data = []
457457
print(f"video_files_paths={video_files_paths}")

‎benchmarks/decoders/benchmark_readme_chart.png‎

3.23 KB
Loading

‎benchmarks/decoders/benchmark_readme_data.json‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
[
2+
{
3+
"decoder": "TorchCodec",
4+
"description": "10 seek()+next()",
5+
"fps": 296.7497929852154,
6+
"fps_p25": 304.82592121401444,
7+
"fps_p75": 286.39866868882336,
8+
"frame_count": 10,
9+
"iqr": 0.0021107543725520372,
10+
"median": 0.03369842283427715,
11+
"type": "seek()+next()",
12+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
13+
},
14+
{
15+
"decoder": "TorchCodec",
16+
"description": "1 next()",
17+
"fps": 141.80053650468176,
18+
"fps_p25": 144.12265279456386,
19+
"fps_p75": 128.28507218641042,
20+
"frame_count": 1,
21+
"iqr": 0.0008566047297790648,
22+
"median": 0.007052159495651722,
23+
"type": "next()",
24+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
25+
},
26+
{
27+
"decoder": "TorchCodec",
28+
"description": "10 next()",
29+
"fps": 638.7876251007046,
30+
"fps_p25": 647.626546889273,
31+
"fps_p75": 629.6538548699413,
32+
"frame_count": 10,
33+
"iqr": 0.00044074421748518944,
34+
"median": 0.015654655173420906,
35+
"type": "next()",
36+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
37+
},
38+
{
39+
"decoder": "TorchCodec[num_threads=1]",
40+
"description": "10 seek()+next()",
41+
"fps": 126.6773946375974,
42+
"fps_p25": 128.33021600580943,
43+
"fps_p75": 124.2963412180317,
44+
"frame_count": 10,
45+
"iqr": 0.00252892030403018,
46+
"median": 0.07894068257883191,
47+
"type": "seek()+next()",
48+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
49+
},
50+
{
51+
"decoder": "TorchCodec[num_threads=1]",
52+
"description": "1 next()",
53+
"fps": 410.70043288059617,
54+
"fps_p25": 416.39979027639464,
55+
"fps_p75": 405.8298526819803,
56+
"frame_count": 1,
57+
"iqr": 6.25486485660077e-05,
58+
"median": 0.0024348647333681584,
59+
"type": "next()",
60+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
61+
},
62+
{
63+
"decoder": "TorchCodec[num_threads=1]",
64+
"description": "10 next()",
65+
"fps": 758.7565583035099,
66+
"fps_p25": 766.9872077345758,
67+
"fps_p75": 751.3583938952637,
68+
"frame_count": 10,
69+
"iqr": 0.00027120066806674004,
70+
"median": 0.013179457746446133,
71+
"type": "next()",
72+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
73+
},
74+
{
75+
"decoder": "TorchVision[backend=VideoReader]",
76+
"description": "10 seek()+next()",
77+
"fps": 7.880664295730362,
78+
"fps_p25": 7.924876540397429,
79+
"fps_p75": 7.827668314297991,
80+
"frame_count": 10,
81+
"iqr": 0.01567032840102911,
82+
"median": 1.2689285604283214,
83+
"type": "seek()+next()",
84+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
85+
},
86+
{
87+
"decoder": "TorchVision[backend=VideoReader]",
88+
"description": "1 next()",
89+
"fps": 209.3834723742803,
90+
"fps_p25": 211.77546088016425,
91+
"fps_p75": 199.25812670019334,
92+
"frame_count": 1,
93+
"iqr": 0.0002966334810480479,
94+
"median": 0.00477592614479363,
95+
"type": "next()",
96+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
97+
},
98+
{
99+
"decoder": "TorchVision[backend=VideoReader]",
100+
"description": "10 next()",
101+
"fps": 531.7221665034224,
102+
"fps_p25": 538.7259880033903,
103+
"fps_p75": 522.8300241450709,
104+
"frame_count": 10,
105+
"iqr": 0.0005643628537654877,
106+
"median": 0.018806814216077328,
107+
"type": "next()",
108+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
109+
},
110+
{
111+
"decoder": "TorchAudio",
112+
"description": "10 seek()+next()",
113+
"fps": 27.403418366590216,
114+
"fps_p25": 27.57563496650987,
115+
"fps_p75": 27.276339015442346,
116+
"frame_count": 10,
117+
"iqr": 0.003979140194132924,
118+
"median": 0.36491797724738717,
119+
"type": "seek()+next()",
120+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
121+
},
122+
{
123+
"decoder": "TorchAudio",
124+
"description": "1 next()",
125+
"fps": 243.36030386646544,
126+
"fps_p25": 246.1182470856226,
127+
"fps_p75": 240.91573311529928,
128+
"frame_count": 1,
129+
"iqr": 8.774134330451558e-05,
130+
"median": 0.004109133593738079,
131+
"type": "next()",
132+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
133+
},
134+
{
135+
"decoder": "TorchAudio",
136+
"description": "10 next()",
137+
"fps": 564.5307153840273,
138+
"fps_p25": 572.2932075367765,
139+
"fps_p75": 558.1691247911658,
140+
"frame_count": 10,
141+
"iqr": 0.00044215633533895016,
142+
"median": 0.01771382801234722,
143+
"type": "next()",
144+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
145+
},
146+
{
147+
"cpu_count": 56,
148+
"machine": "x86_64",
149+
"processor": "x86_64",
150+
"python_version": "3.11.10",
151+
"system": "Linux"
152+
}
153+
]

‎benchmarks/decoders/generate_readme_chart.py‎

Lines changed: 7 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -4,81 +4,20 @@
44
# This source code is licensed under the BSD-style license found in the
55
# LICENSE file in the root directory of this source tree.
66

7-
import glob
8-
import importlib.resources
9-
import os
10-
import shutil
11-
from pathlib import Path
12-
13-
from benchmark_decoders_library import (
14-
generate_videos,
15-
plot_data,
16-
run_benchmarks,
17-
TorchAudioDecoder,
18-
TorchcodecNonCompiledWithOptions,
19-
TVNewAPIDecoderWithBackend,
20-
)
21-
22-
23-
def in_fbcode() -> bool:
24-
return "FB_PAR_RUNTIME_FILES" in os.environ
7+
import json
258

9+
from pathlib import Path
2610

27-
def get_test_resource_path(filename: str) -> str:
28-
if in_fbcode():
29-
resource = importlib.resources.files(__package__).joinpath(filename)
30-
with importlib.resources.as_file(resource) as path:
31-
return os.fspath(path)
32-
33-
return str(Path(__file__).parent / f"../../test/resources/{filename}")
11+
from benchmark_decoders_library import plot_data
3412

3513

3614
def main() -> None:
37-
"""Benchmarks the performance of a few video decoders on synthetic videos"""
38-
39-
resolutions = ["640x480"]
40-
encodings = ["libx264"]
41-
fpses = [30]
42-
gop_sizes = [600]
43-
durations = [10]
44-
pix_fmts = ["yuv420p"]
45-
ffmpeg_path = "ffmpeg"
46-
videos_dir_path = "/tmp/torchcodec_benchmarking_videos"
47-
shutil.rmtree(videos_dir_path, ignore_errors=True)
48-
os.makedirs(videos_dir_path)
49-
generate_videos(
50-
resolutions,
51-
encodings,
52-
fpses,
53-
gop_sizes,
54-
durations,
55-
pix_fmts,
56-
ffmpeg_path,
57-
videos_dir_path,
58-
)
59-
video_files_paths = glob.glob(f"{videos_dir_path}/*.mp4")
60-
61-
decoder_dict = {}
62-
decoder_dict["TorchCodec"] = TorchcodecNonCompiledWithOptions()
63-
decoder_dict["TorchCodec[num_threads=1]"] = TorchcodecNonCompiledWithOptions(
64-
num_threads=1
65-
)
66-
decoder_dict["TorchVision[backend=VideoReader]"] = TVNewAPIDecoderWithBackend(
67-
"video_reader"
68-
)
69-
decoder_dict["TorchAudio"] = TorchAudioDecoder()
15+
data_json = Path(__file__).parent / "benchmark_readme_data.json"
16+
with open(data_json, "r") as read_file:
17+
data_from_file = json.load(read_file)
7018

7119
output_png = Path(__file__).parent / "benchmark_readme_chart.png"
72-
# These are the number of uniform seeks we do in the seek+decode benchmark.
73-
num_uniform_samples = 10
74-
df_data = run_benchmarks(
75-
decoder_dict,
76-
video_files_paths,
77-
num_uniform_samples,
78-
10,
79-
False,
80-
)
81-
plot_data(df_data, output_png)
20+
plot_data(data_from_file, output_png)
8221

8322

8423
if __name__ == "__main__":

‎benchmarks/decoders/generate_readme_data.py‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import glob
8+
import json
9+
import os
10+
import platform
11+
import shutil
12+
from pathlib import Path
13+
14+
from benchmark_decoders_library import (
15+
generate_videos,
16+
run_benchmarks,
17+
TorchAudioDecoder,
18+
TorchcodecNonCompiledWithOptions,
19+
TVNewAPIDecoderWithBackend,
20+
)
21+
22+
23+
def main() -> None:
24+
"""Benchmarks the performance of a few video decoders on synthetic videos"""
25+
26+
resolutions = ["640x480"]
27+
encodings = ["libx264"]
28+
fpses = [30]
29+
gop_sizes = [600]
30+
durations = [10]
31+
pix_fmts = ["yuv420p"]
32+
ffmpeg_path = "ffmpeg"
33+
videos_dir_path = "/tmp/torchcodec_benchmarking_videos"
34+
shutil.rmtree(videos_dir_path, ignore_errors=True)
35+
os.makedirs(videos_dir_path)
36+
generate_videos(
37+
resolutions,
38+
encodings,
39+
fpses,
40+
gop_sizes,
41+
durations,
42+
pix_fmts,
43+
ffmpeg_path,
44+
videos_dir_path,
45+
)
46+
video_files_paths = glob.glob(f"{videos_dir_path}/*.mp4")
47+
48+
decoder_dict = {}
49+
decoder_dict["TorchCodec"] = TorchcodecNonCompiledWithOptions()
50+
decoder_dict["TorchCodec[num_threads=1]"] = TorchcodecNonCompiledWithOptions(
51+
num_threads=1
52+
)
53+
decoder_dict["TorchVision[backend=VideoReader]"] = TVNewAPIDecoderWithBackend(
54+
"video_reader"
55+
)
56+
decoder_dict["TorchAudio"] = TorchAudioDecoder()
57+
58+
# These are the number of uniform seeks we do in the seek+decode benchmark.
59+
num_uniform_samples = 10
60+
df_data = run_benchmarks(
61+
decoder_dict,
62+
video_files_paths,
63+
num_uniform_samples,
64+
10,
65+
False,
66+
)
67+
df_data.append(
68+
{
69+
"cpu_count": os.cpu_count(),
70+
"system": platform.system(),
71+
"machine": platform.machine(),
72+
"processor": platform.processor(),
73+
"python_version": str(platform.python_version()),
74+
}
75+
)
76+
77+
data_json = Path(__file__).parent / "benchmark_readme_data.json"
78+
with open(data_json, "w") as write_file:
79+
json.dump(df_data, write_file, sort_keys=True, indent=4)
80+
81+
82+
if __name__ == "__main__":
83+
main()

0 commit comments

Comments
 (0)