Skip to content

Commit 066663c

Browse files
scottsscotts
committed
Seperate out readme experiments from chart generation
1 parent b839529 commit 066663c

File tree

5 files changed

+226
-70
lines changed

5 files changed

+226
-70
lines changed

‎benchmarks/decoders/benchmark_decoders_library.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -451,7 +451,7 @@ def run_benchmarks(
451451
num_uniform_samples,
452452
min_runtime_seconds,
453453
benchmark_video_creation,
454-
):
454+
) -> list[dict[str, str | float | int]]:
455455
results = []
456456
df_data = []
457457
print(f"video_files_paths={video_files_paths}")

‎benchmarks/decoders/benchmark_readme_chart.png‎

2.73 KB
Loading

‎benchmarks/decoders/benchmark_readme_data.json‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
[
2+
{
3+
"decoder": "TorchCodec",
4+
"description": "10 seek()+next()",
5+
"fps": 287.76819838503394,
6+
"fps_p25": 296.7112274366146,
7+
"fps_p75": 278.56593088985414,
8+
"frame_count": 10,
9+
"iqr": 0.0021953415125608444,
10+
"median": 0.0347501914948225,
11+
"type": "seek()+next()",
12+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
13+
},
14+
{
15+
"decoder": "TorchCodec",
16+
"description": "1 next()",
17+
"fps": 143.23418093152685,
18+
"fps_p25": 144.9163143919374,
19+
"fps_p75": 139.4945872229059,
20+
"frame_count": 1,
21+
"iqr": 0.00026820262428373185,
22+
"median": 0.006981573766097426,
23+
"type": "next()",
24+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
25+
},
26+
{
27+
"decoder": "TorchCodec",
28+
"description": "10 next()",
29+
"fps": 636.9483478857736,
30+
"fps_p25": 648.4013013535854,
31+
"fps_p75": 625.0224538297709,
32+
"frame_count": 10,
33+
"iqr": 0.0005768775008618832,
34+
"median": 0.01569986017420888,
35+
"type": "next()",
36+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
37+
},
38+
{
39+
"decoder": "TorchCodec[num_threads=1]",
40+
"description": "10 seek()+next()",
41+
"fps": 126.3244720565993,
42+
"fps_p25": 127.9167991114483,
43+
"fps_p75": 124.01060571029214,
44+
"frame_count": 10,
45+
"iqr": 0.0024624494835734367,
46+
"median": 0.0791612253524363,
47+
"type": "seek()+next()",
48+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
49+
},
50+
{
51+
"decoder": "TorchCodec[num_threads=1]",
52+
"description": "1 next()",
53+
"fps": 407.44392494869396,
54+
"fps_p25": 412.74345620298266,
55+
"fps_p75": 402.8417995660793,
56+
"frame_count": 1,
57+
"iqr": 5.955155938863754e-05,
58+
"median": 0.0024543254636228084,
59+
"type": "next()",
60+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
61+
},
62+
{
63+
"decoder": "TorchCodec[num_threads=1]",
64+
"description": "10 next()",
65+
"fps": 753.9423314510905,
66+
"fps_p25": 767.9119180622625,
67+
"fps_p75": 674.3038471753032,
68+
"frame_count": 10,
69+
"iqr": 0.0018077827990055084,
70+
"median": 0.013263613916933537,
71+
"type": "next()",
72+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
73+
},
74+
{
75+
"decoder": "TorchVision[backend=VideoReader]",
76+
"description": "10 seek()+next()",
77+
"fps": 7.810381194748724,
78+
"fps_p25": 7.856542709397942,
79+
"fps_p75": 7.642085464749366,
80+
"frame_count": 10,
81+
"iqr": 0.03571884077973664,
82+
"median": 1.2803472392261028,
83+
"type": "seek()+next()",
84+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
85+
},
86+
{
87+
"decoder": "TorchVision[backend=VideoReader]",
88+
"description": "1 next()",
89+
"fps": 208.7929367227745,
90+
"fps_p25": 211.6872277728345,
91+
"fps_p75": 203.7724491459089,
92+
"frame_count": 1,
93+
"iqr": 0.00018348419107496773,
94+
"median": 0.004789434047415853,
95+
"type": "next()",
96+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
97+
},
98+
{
99+
"decoder": "TorchVision[backend=VideoReader]",
100+
"description": "10 next()",
101+
"fps": 552.8383200630532,
102+
"fps_p25": 583.4686750060439,
103+
"fps_p75": 538.7105006175086,
104+
"frame_count": 10,
105+
"iqr": 0.0014239652082324028,
106+
"median": 0.01808847114443779,
107+
"type": "next()",
108+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
109+
},
110+
{
111+
"decoder": "TorchAudio",
112+
"description": "10 seek()+next()",
113+
"fps": 27.53516195793989,
114+
"fps_p25": 27.855660335461184,
115+
"fps_p75": 27.088720957512532,
116+
"frame_count": 10,
117+
"iqr": 0.01016386947594583,
118+
"median": 0.3631720058619976,
119+
"type": "seek()+next()",
120+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
121+
},
122+
{
123+
"decoder": "TorchAudio",
124+
"description": "1 next()",
125+
"fps": 241.6594091958052,
126+
"fps_p25": 244.01496866823945,
127+
"fps_p75": 238.4751784315965,
128+
"frame_count": 1,
129+
"iqr": 9.519928134977835e-05,
130+
"median": 0.004138055304065347,
131+
"type": "next()",
132+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
133+
},
134+
{
135+
"decoder": "TorchAudio",
136+
"description": "10 next()",
137+
"fps": 559.8508874362404,
138+
"fps_p25": 567.2986262102045,
139+
"fps_p75": 550.7422646602039,
140+
"frame_count": 10,
141+
"iqr": 0.0005299132317304611,
142+
"median": 0.017861898988485336,
143+
"type": "next()",
144+
"video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
145+
}
146+
]

‎benchmarks/decoders/generate_readme_chart.py‎

Lines changed: 6 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -4,81 +4,18 @@
44
# This source code is licensed under the BSD-style license found in the
55
# LICENSE file in the root directory of this source tree.
66

7-
import glob
8-
import importlib.resources
9-
import os
10-
import shutil
11-
from pathlib import Path
7+
import json
128

13-
from benchmark_decoders_library import (
14-
generate_videos,
15-
plot_data,
16-
run_benchmarks,
17-
TorchAudioDecoder,
18-
TorchcodecNonCompiledWithOptions,
19-
TVNewAPIDecoderWithBackend,
20-
)
21-
22-
23-
def in_fbcode() -> bool:
24-
return "FB_PAR_RUNTIME_FILES" in os.environ
25-
26-
27-
def get_test_resource_path(filename: str) -> str:
28-
if in_fbcode():
29-
resource = importlib.resources.files(__package__).joinpath(filename)
30-
with importlib.resources.as_file(resource) as path:
31-
return os.fspath(path)
32-
33-
return str(Path(__file__).parent / f"../../test/resources/{filename}")
9+
from benchmark_decoders_library import plot_data
3410

3511

3612
def main() -> None:
37-
"""Benchmarks the performance of a few video decoders on synthetic videos"""
38-
39-
resolutions = ["640x480"]
40-
encodings = ["libx264"]
41-
fpses = [30]
42-
gop_sizes = [600]
43-
durations = [10]
44-
pix_fmts = ["yuv420p"]
45-
ffmpeg_path = "ffmpeg"
46-
videos_dir_path = "/tmp/torchcodec_benchmarking_videos"
47-
shutil.rmtree(videos_dir_path, ignore_errors=True)
48-
os.makedirs(videos_dir_path)
49-
generate_videos(
50-
resolutions,
51-
encodings,
52-
fpses,
53-
gop_sizes,
54-
durations,
55-
pix_fmts,
56-
ffmpeg_path,
57-
videos_dir_path,
58-
)
59-
video_files_paths = glob.glob(f"{videos_dir_path}/*.mp4")
60-
61-
decoder_dict = {}
62-
decoder_dict["TorchCodec"] = TorchcodecNonCompiledWithOptions()
63-
decoder_dict["TorchCodec[num_threads=1]"] = TorchcodecNonCompiledWithOptions(
64-
num_threads=1
65-
)
66-
decoder_dict["TorchVision[backend=VideoReader]"] = TVNewAPIDecoderWithBackend(
67-
"video_reader"
68-
)
69-
decoder_dict["TorchAudio"] = TorchAudioDecoder()
13+
data_json = Path(__file__).parent / "benchmark_readme_data.json"
14+
with open(data_json, "r") as read_file:
15+
data_from_file = json.load(read_file)
7016

7117
output_png = Path(__file__).parent / "benchmark_readme_chart.png"
72-
# These are the number of uniform seeks we do in the seek+decode benchmark.
73-
num_uniform_samples = 10
74-
df_data = run_benchmarks(
75-
decoder_dict,
76-
video_files_paths,
77-
num_uniform_samples,
78-
10,
79-
False,
80-
)
81-
plot_data(df_data, output_png)
18+
plot_data(data_from_file, output_png)
8219

8320

8421
if __name__ == "__main__":

‎benchmarks/decoders/generate_readme_data.py‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import glob
8+
import json
9+
import os
10+
import shutil
11+
from pathlib import Path
12+
13+
from benchmark_decoders_library import (
14+
generate_videos,
15+
run_benchmarks,
16+
TorchAudioDecoder,
17+
TorchcodecNonCompiledWithOptions,
18+
TVNewAPIDecoderWithBackend,
19+
)
20+
21+
22+
def main() -> None:
23+
"""Benchmarks the performance of a few video decoders on synthetic videos"""
24+
25+
resolutions = ["640x480"]
26+
encodings = ["libx264"]
27+
fpses = [30]
28+
gop_sizes = [600]
29+
durations = [10]
30+
pix_fmts = ["yuv420p"]
31+
ffmpeg_path = "ffmpeg"
32+
videos_dir_path = "/tmp/torchcodec_benchmarking_videos"
33+
shutil.rmtree(videos_dir_path, ignore_errors=True)
34+
os.makedirs(videos_dir_path)
35+
generate_videos(
36+
resolutions,
37+
encodings,
38+
fpses,
39+
gop_sizes,
40+
durations,
41+
pix_fmts,
42+
ffmpeg_path,
43+
videos_dir_path,
44+
)
45+
video_files_paths = glob.glob(f"{videos_dir_path}/*.mp4")
46+
47+
decoder_dict = {}
48+
decoder_dict["TorchCodec"] = TorchcodecNonCompiledWithOptions()
49+
decoder_dict["TorchCodec[num_threads=1]"] = TorchcodecNonCompiledWithOptions(
50+
num_threads=1
51+
)
52+
decoder_dict["TorchVision[backend=VideoReader]"] = TVNewAPIDecoderWithBackend(
53+
"video_reader"
54+
)
55+
decoder_dict["TorchAudio"] = TorchAudioDecoder()
56+
57+
# These are the number of uniform seeks we do in the seek+decode benchmark.
58+
num_uniform_samples = 10
59+
df_data = run_benchmarks(
60+
decoder_dict,
61+
video_files_paths,
62+
num_uniform_samples,
63+
10,
64+
False,
65+
)
66+
67+
data_json = Path(__file__).parent / "benchmark_readme_data.json"
68+
with open(data_json, "w") as write_file:
69+
json.dump(df_data, write_file, sort_keys=True, indent=4)
70+
71+
72+
if __name__ == "__main__":
73+
main()

0 commit comments

Comments
 (0)