Added benchmark chart to README.md (#315)

Author: ahmadsharif1

README.md

@@ -118,6 +118,12 @@ The instructions below assume you're on Linux.
      pip install torchcodec
      ```
 
+## Benchmark Results
+
+The following was generated by running [our benchmark script](./benchmarks/decoders/generate_readme_chart.py) on a lightly loaded 22-core machine.
+
+![benchmark_results](./benchmarks/decoders/benchmark_readme_chart.png)
+
 ## Planned future work
 
 We are actively working on the following features:

benchmarks/decoders/benchmark_decoders_library.py

@@ -1,7 +1,10 @@
 import abc
 import json
 import os
+import subprocess
 import timeit
+from concurrent.futures import ThreadPoolExecutor
+from itertools import product
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -299,6 +302,53 @@ def create_torchcodec_decoder_from_file(video_file):
     return video_decoder
 
 
+def generate_video(command):
+    print(command)
+    print(" ".join(command))
+    subprocess.check_call(command)
+
+
+def generate_videos(
+    resolutions,
+    encodings,
+    fpses,
+    gop_sizes,
+    durations,
+    pix_fmts,
+    ffmpeg_cli,
+    output_dir,
+):
+    executor = ThreadPoolExecutor(max_workers=20)
+    video_count = 0
+
+    for resolution, duration, fps, gop_size, encoding, pix_fmt in product(
+        resolutions, durations, fpses, gop_sizes, encodings, pix_fmts
+    ):
+        outfile = f"{output_dir}/{resolution}_{duration}s_{fps}fps_{gop_size}gop_{encoding}_{pix_fmt}.mp4"
+        command = [
+            ffmpeg_cli,
+            "-y",
+            "-f",
+            "lavfi",
+            "-i",
+            f"color=c=blue:s={resolution}:d={duration}",
+            "-c:v",
+            encoding,
+            "-r",
+            f"{fps}",
+            "-g",
+            f"{gop_size}",
+            "-pix_fmt",
+            pix_fmt,
+            outfile,
+        ]
+        executor.submit(generate_video, command)
+        video_count += 1
+
+    executor.shutdown(wait=True)
+    print(f"Generated {video_count} videos")
+
+
 def plot_data(df_data, plot_path):
     # Creating the DataFrame
     df = pd.DataFrame(df_data)

benchmarks/decoders/benchmark_readme_chart.png

@@ -0,0 +1,85 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import glob
+import importlib.resources
+import os
+import shutil
+from pathlib import Path
+
+from benchmark_decoders_library import (
+    generate_videos,
+    plot_data,
+    run_benchmarks,
+    TorchAudioDecoder,
+    TorchcodecNonCompiledWithOptions,
+    TVNewAPIDecoderWithBackend,
+)
+
+
+def in_fbcode() -> bool:
+    return "FB_PAR_RUNTIME_FILES" in os.environ
+
+
+def get_test_resource_path(filename: str) -> str:
+    if in_fbcode():
+        resource = importlib.resources.files(__package__).joinpath(filename)
+        with importlib.resources.as_file(resource) as path:
+            return os.fspath(path)
+
+    return str(Path(__file__).parent / f"../../test/resources/{filename}")
+
+
+def main() -> None:
+    """Benchmarks the performance of a few video decoders on synthetic videos"""
+
+    resolutions = ["640x480"]
+    encodings = ["libx264"]
+    fpses = [30]
+    gop_sizes = [600]
+    durations = [10]
+    pix_fmts = ["yuv420p"]
+    ffmpeg_path = "/usr/local/bin/ffmpeg"
+    videos_path = "/tmp/videos"
+    shutil.rmtree(videos_path)
+    os.makedirs(videos_path)
+    generate_videos(
+        resolutions,
+        encodings,
+        fpses,
+        gop_sizes,
+        durations,
+        pix_fmts,
+        ffmpeg_path,
+        videos_path,
+    )
+    video_paths = glob.glob(f"{videos_path}/*.mp4")
+
+    decoder_dict = {}
+    decoder_dict["TorchCodec"] = TorchcodecNonCompiledWithOptions()
+    decoder_dict["TorchCodec[num_threads=1]"] = TorchcodecNonCompiledWithOptions(
+        num_threads=1
+    )
+    decoder_dict["TorchVision[backend=VideoReader]"] = TVNewAPIDecoderWithBackend(
+        "video_reader"
+    )
+    decoder_dict["TorchAudio"] = TorchAudioDecoder()
+
+    output_png = Path(__file__).parent / "benchmark_readme_chart.png"
+    # These are the number of uniform seeks we do in the seek+decode benchmark.
+    num_uniform_samples = 10
+    df_data = run_benchmarks(
+        decoder_dict,
+        video_paths,
+        num_uniform_samples,
+        10,
+        False,
+    )
+    plot_data(df_data, output_png)
+
+
+if __name__ == "__main__":
+    main()