Generation new chart, add more next benchmarks

scotts · scotts · commit c4681c0b0e07 · 2024-11-04T07:56:36.000-08:00
diff --git a/benchmarks/decoders/benchmark_decoders.py b/benchmarks/decoders/benchmark_decoders.py
@@ -145,8 +145,9 @@ def main() -> None:
         decoder_dict,
         video_paths,
         num_uniform_samples,
-        args.bm_video_speed_min_run_seconds,
-        args.bm_video_creation,
+        num_sequential_frames_from_start=[1, 10, 100],
+        min_runtime_seconds=args.bm_video_speed_min_run_seconds,
+        benchmark_video_creation=args.bm_video_creation,
     )
     plot_data(df_data, args.plot_path)
 
diff --git a/benchmarks/decoders/benchmark_decoders_library.py b/benchmarks/decoders/benchmark_decoders_library.py
@@ -472,28 +472,37 @@ def get_metadata(video_file_path: str) -> VideoStreamMetadata:
     return VideoDecoder(video_file_path).metadata
 
 def run_benchmarks(
-    decoder_dict,
-    video_files_paths,
-    num_samples,
-    min_runtime_seconds,
-    benchmark_video_creation,
+    decoder_dict: dict[str, AbstractDecoder],
+    video_files_paths: list[str],
+    num_samples: int,
+    num_sequential_frames_from_start: list[int],
+    min_runtime_seconds: float,
+    benchmark_video_creation: bool,
 ) -> list[dict[str, str | float | int]]:
+    # Ensure that we have the same seed across benchmark runs.
+    torch.manual_seed(0)
+
+    print(f"video_files_paths={video_files_paths}")
+
     results = []
     df_data = []
-    print(f"video_files_paths={video_files_paths}")
     verbose = False
-    for decoder_name, decoder in decoder_dict.items():
-        for video_file_path in video_files_paths:
-            print(f"video={video_file_path}, decoder={decoder_name}")
-            metadata = get_metadata(video_file_path)
-            metadata_label = f"{metadata.codec} {metadata.width}x{metadata.height}, {metadata.duration_seconds}s {metadata.average_fps}fps"
+    for video_file_path in video_files_paths:
+        metadata = get_metadata(video_file_path)
+        metadata_label = f"{metadata.codec} {metadata.width}x{metadata.height}, {metadata.duration_seconds}s {metadata.average_fps}fps"
 
-            duration = metadata.duration_seconds
-            uniform_pts_list = [
-                i * duration / num_samples for i in range(num_samples)
-            ]
+        duration = metadata.duration_seconds
+        uniform_pts_list = [
+            i * duration / num_samples for i in range(num_samples)
+        ]
 
-            random_pts_list = (torch.rand(num_samples) * duration).tolist()
+        # Note that we are using the same random pts values for all decoders for the same
+        # video. However, because we use the duration as part of this calculation, we
+        # are using different random pts values across videos.
+        random_pts_list = (torch.rand(num_samples) * duration).tolist()
+
+        for decoder_name, decoder in decoder_dict.items():
+            print(f"video={video_file_path}, decoder={decoder_name}")
 
             for kind, pts_list in [("uniform", uniform_pts_list), ("random", random_pts_list)]:
                 if verbose:
@@ -527,7 +536,7 @@ def run_benchmarks(
                 df_item["fps_p25"] = 1.0 * num_samples / results[-1]._p25
                 df_data.append(df_item)
 
-            for num_consecutive_nexts in [100]:
+            for num_consecutive_nexts in num_sequential_frames_from_start:
                 consecutive_frames_result = benchmark.Timer(
                     stmt="decoder.get_consecutive_frames_from_video(video_file, consecutive_frames_to_extract)",
                     globals={
diff --git a/benchmarks/decoders/benchmark_readme_chart.png b/benchmarks/decoders/benchmark_readme_chart.png
diff --git a/benchmarks/decoders/benchmark_readme_data.json b/benchmarks/decoders/benchmark_readme_data.json
@@ -2,144 +2,144 @@
     {
         "decoder": "TorchCodec",
         "description": "uniform 10 seek()+next()",
-        "fps": 314.3002734250253,
-        "fps_p25": 322.3704500761085,
-        "fps_p75": 304.61961295439414,
+        "fps": 315.05924655387844,
+        "fps_p25": 323.6567367722293,
+        "fps_p75": 303.4217234978234,
         "frame_count": 10,
-        "iqr": 0.001807614229619503,
-        "median": 0.03181670792400837,
+        "iqr": 0.0020604978781193495,
+        "median": 0.031740061938762665,
         "type": "uniform:seek()+next()",
         "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
     },
     {
         "decoder": "TorchCodec",
         "description": "random 10 seek()+next()",
-        "fps": 318.6332392061074,
-        "fps_p25": 326.61265301425016,
-        "fps_p75": 307.9272223416659,
+        "fps": 312.8775910187949,
+        "fps_p25": 316.65411586172576,
+        "fps_p75": 304.67664202566044,
         "frame_count": 10,
-        "iqr": 0.0018578986637294292,
-        "median": 0.031384045258164406,
+        "iqr": 0.0012414834462106256,
+        "median": 0.03196138134226203,
         "type": "random:seek()+next()",
         "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
     },
     {
         "decoder": "TorchCodec",
         "description": "100 next()",
-        "fps": 963.8588500259038,
-        "fps_p25": 1232.7462278510554,
-        "fps_p75": 897.72507501799,
+        "fps": 948.3281062356014,
+        "fps_p25": 1211.2939721426944,
+        "fps_p75": 884.5162407819654,
         "frame_count": 100,
-        "iqr": 0.03027298115193844,
-        "median": 0.10374963097274303,
+        "iqr": 0.030499806627631187,
+        "median": 0.10544873587787151,
         "type": "next()",
         "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
     },
     {
         "decoder": "TorchCodec[num_threads=1]",
         "description": "uniform 10 seek()+next()",
-        "fps": 131.988339357579,
-        "fps_p25": 133.26652650624064,
-        "fps_p75": 130.3643453501947,
+        "fps": 131.3991915256709,
+        "fps_p25": 132.80657677770856,
+        "fps_p75": 130.0875105195256,
         "frame_count": 10,
-        "iqr": 0.0016704928129911423,
-        "median": 0.07576426863670349,
+        "iqr": 0.0015738545916974545,
+        "median": 0.07610396901145577,
         "type": "uniform:seek()+next()",
         "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
     },
     {
         "decoder": "TorchCodec[num_threads=1]",
         "description": "random 10 seek()+next()",
-        "fps": 122.75317946016999,
-        "fps_p25": 124.22201311180707,
-        "fps_p75": 121.19633847694604,
+        "fps": 131.76952597486445,
+        "fps_p25": 133.43827127306128,
+        "fps_p75": 129.72933699334547,
         "frame_count": 10,
-        "iqr": 0.0020097135566174984,
-        "median": 0.08146428503096104,
+        "iqr": 0.0021425478626042604,
+        "median": 0.0758900810033083,
         "type": "random:seek()+next()",
         "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
     },
     {
         "decoder": "TorchCodec[num_threads=1]",
         "description": "100 next()",
-        "fps": 809.9441914595772,
-        "fps_p25": 820.5478107543202,
-        "fps_p75": 801.8920090548505,
+        "fps": 767.4262194030912,
+        "fps_p25": 954.5964481242703,
+        "fps_p75": 751.3156802763621,
         "frame_count": 100,
-        "iqr": 0.0028352681547403336,
-        "median": 0.12346529681235552,
+        "iqr": 0.028343535726889968,
+        "median": 0.13030568603426218,
         "type": "next()",
         "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
     },
     {
         "decoder": "TorchVision[backend=video_reader]",
         "description": "uniform 10 seek()+next()",
-        "fps": 7.818673617539736,
-        "fps_p25": 7.914887876132339,
-        "fps_p75": 7.72635851907541,
+        "fps": 7.87802293808524,
+        "fps_p25": 7.9221952370727795,
+        "fps_p75": 7.816352619742458,
         "frame_count": 10,
-        "iqr": 0.0308289946988225,
-        "median": 1.2789893131703138,
+        "iqr": 0.017092708498239517,
+        "median": 1.2693540090695024,
         "type": "uniform:seek()+next()",
         "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
     },
     {
         "decoder": "TorchVision[backend=video_reader]",
         "description": "random 10 seek()+next()",
-        "fps": 8.174175797820915,
-        "fps_p25": 8.198128262246657,
-        "fps_p75": 8.089292632708272,
+        "fps": 7.257307771543773,
+        "fps_p25": 7.305370582206469,
+        "fps_p75": 7.209328679013935,
         "frame_count": 10,
-        "iqr": 0.016411407850682735,
-        "median": 1.2233649296686053,
+        "iqr": 0.018235752126201987,
+        "median": 1.3779214434325695,
         "type": "random:seek()+next()",
         "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
     },
     {
         "decoder": "TorchVision[backend=video_reader]",
         "description": "100 next()",
-        "fps": 708.5626871155907,
-        "fps_p25": 714.9065385245668,
-        "fps_p75": 701.1653993140255,
+        "fps": 843.6237746060216,
+        "fps_p25": 852.9895625098609,
+        "fps_p75": 830.8058002517034,
         "frame_count": 100,
-        "iqr": 0.002741277450695634,
-        "median": 0.14113077335059643,
+        "iqr": 0.0031303432770073414,
+        "median": 0.11853625159710646,
         "type": "next()",
         "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
     },
     {
         "decoder": "TorchAudio",
         "description": "uniform 10 seek()+next()",
-        "fps": 28.25481341811952,
-        "fps_p25": 28.472751002542807,
-        "fps_p75": 28.027683266448527,
+        "fps": 28.283386183212908,
+        "fps_p25": 28.48861769505288,
+        "fps_p75": 27.962052080094207,
         "frame_count": 10,
-        "iqr": 0.005577113479375839,
-        "median": 0.35392199736088514,
+        "iqr": 0.006610161624848843,
+        "median": 0.3535644542425871,
         "type": "uniform:seek()+next()",
         "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
     },
     {
         "decoder": "TorchAudio",
         "description": "random 10 seek()+next()",
-        "fps": 29.222482794708707,
-        "fps_p25": 29.434989975510586,
-        "fps_p75": 28.911024515041184,
+        "fps": 26.009247898010745,
+        "fps_p25": 26.13538377314808,
+        "fps_p75": 25.757236346453418,
         "frame_count": 10,
-        "iqr": 0.006157086929306388,
-        "median": 0.34220227180048823,
+        "iqr": 0.00561736966483295,
+        "median": 0.38447863003239036,
         "type": "random:seek()+next()",
         "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
     },
     {
         "decoder": "TorchAudio",
         "description": "100 next()",
-        "fps": 719.0499213961119,
-        "fps_p25": 725.4279593070022,
-        "fps_p75": 710.8406389605324,
+        "fps": 659.7600283811723,
+        "fps_p25": 668.0071480761926,
+        "fps_p75": 652.052492189031,
         "frame_count": 100,
-        "iqr": 0.002828843891620636,
-        "median": 0.13907240238040686,
+        "iqr": 0.003662889124825597,
+        "median": 0.15157026145607233,
         "type": "next()",
         "video": "/tmp/torchcodec_benchmarking_videos/640x480_10s_30fps_600gop_libx264_yuv420p.mp4"
     },
diff --git a/benchmarks/decoders/generate_readme_data.py b/benchmarks/decoders/generate_readme_data.py
@@ -61,8 +61,9 @@ def main() -> None:
         decoder_dict,
         video_files_paths,
         num_samples,
-        30,
-        False,
+        num_sequential_frames_from_start=[100],
+        min_runtime_seconds=30,
+        benchmark_video_creation=False,
     )
     df_data.append(
         {

Original file line number	Diff line number	Diff line change
`@@ -61,8 +61,9 @@ def main() -> None:`
`61`	`61`	`decoder_dict,`
`62`	`62`	`video_files_paths,`
`63`	`63`	`num_samples,`
`64`		`- 30,`
`65`		`- False,`
	`64`	`+ num_sequential_frames_from_start=[100],`
	`65`	`+ min_runtime_seconds=30,`
	`66`	`+ benchmark_video_creation=False,`
`66`	`67`	`)`
`67`	`68`	`df_data.append(`
`68`	`69`	`{`