Allow multiple backends, invert 'relative' column to show speed-up against first row

Author: NicolasHug

benchmarks/benchmark_transforms.py

@@ -231,7 +231,7 @@ def main():
     )
     all_backends = ["tv", "tv-compiled", "opencv", "pil", "albumentations", "kornia"]
     parser.add_argument(
-        "--backend", type=str.lower, choices=all_backends + ["all"], default="all", help="Backend to benchmark"
+        "--backends", type=str, default="all", help="Backends to benchmark (comma-separated list or 'all'). First backend is used as reference for comparison."
     )
     parser.add_argument("-v", "--verbose", action="store_true", help="Enable verbose output")
     parser.add_argument("--device", type=str, default="cpu", help="Device to use: cpu or cuda (default: cpu)")
@@ -240,7 +240,18 @@ def main():
 
     print_benchmark_info(args)
 
-    backends_to_run = all_backends if args.backend.lower() == "all" else [args.backend]
+    # Parse backends parameter
+    if args.backends.lower() == "all":
+        backends_to_run = all_backends
+    else:
+        backends_to_run = [backend.strip().lower() for backend in args.backends.split(",")]
+        # Validate backends
+        invalid_backends = [b for b in backends_to_run if b not in all_backends]
+        if invalid_backends:
+            print(f"ERROR: Invalid backends: {', '.join(invalid_backends)}")
+            print(f"Available backends: {', '.join(all_backends)}")
+            return
+
     results = []
 
     for backend in backends_to_run:

benchmarks/utils.py

@@ -86,12 +86,13 @@ def report_stats(times: torch.Tensor, unit: str, verbose: bool = True) -> Dict[s
 
 
 def print_comparison_table(results: List[Dict[str, Any]]) -> None:
-    torchvision_median = next((r["stats"]["median"] for r in results if r["backend"].lower() == "tv"), None)
+    # Use first backend as reference for relative comparison
+    reference_median = results[0]["stats"]["median"] if results else None
 
     table_data = []
     for result in results:
         stats = result["stats"]
-        relative = f"{stats['median'] / torchvision_median:.2f}x" if torchvision_median else "N/A"
+        speed_up = f"{reference_median / stats['median']:.2f}x" if reference_median else "N/A"
 
         table_data.append(
             {
@@ -101,7 +102,7 @@ def print_comparison_table(results: List[Dict[str, Any]]) -> None:
                 "Mean (ms)": f"{stats['mean']:.2f}",
                 "Min (ms)": f"{stats['min']:.2f}",
                 "Max (ms)": f"{stats['max']:.2f}",
-                "Relative": relative,
+                "Speed-up": speed_up,
             }
         )