Operator Count Analysis (#79)

Author: msaroufim

.gitignore

@@ -8,6 +8,5 @@ CLAUDE.md
 venv/
 ops/
 uv.lock
-
-# Pre-commit
+pytorch_operator_coverage.csv
 .pre-commit-cache/

BackendBench/eval.py

@@ -2,7 +2,12 @@
 
 import torch
 
-import triton.testing
+try:
+    import triton.testing
+
+    TRITON_AVAILABLE = True
+except ImportError:
+    TRITON_AVAILABLE = False
 
 
 from BackendBench.utils import uses_cuda_stream
@@ -69,7 +74,9 @@ def cpu_bench(fn, num_runs=100):
 
 
 def eval_performance(op, impl, tests):
-    bench_fn = triton.testing.do_bench if torch.cuda.is_available() else cpu_bench
+    bench_fn = (
+        triton.testing.do_bench if TRITON_AVAILABLE and torch.cuda.is_available() else cpu_bench
+    )
     base_times = []
     test_times = []
     for test in tests:

BackendBench/opinfo_suite.py

@@ -57,19 +57,24 @@ def build_op_tests(device, dtype, filter=None):
             continue
 
         op_indices = defaultdict(list)
-        for idx, test in enumerate(op.sample_inputs(device, dtype)):
+        try:
+            sample_inputs = list(op.sample_inputs(device, dtype))
+        except Exception:
+            continue
+
+        for idx, test in enumerate(sample_inputs):
             # print(f"{idx=} {test.input=} {test.args=} {test.kwargs=}")
-            with OpTracerMode() as tracer:
-                ref = op.op(test.input, *test.args, **test.kwargs)
-            if len(tracer.ops) == 1:
-                try:
+            try:
+                with OpTracerMode() as tracer:
+                    ref = op.op(test.input, *test.args, **test.kwargs)
+                if len(tracer.ops) == 1:
                     res = tracer.ops[0](test.input, *test.args, **test.kwargs)
                     if allclose(ref, res):
                         op_indices[tracer.ops[0]].append(idx)
-                except Exception:
-                    logger.debug(f"opinfo {op.name} couldn't run underlying op {tracer.ops[0]}")
-            else:
-                logger.debug(f"opinfo {op.name} has {len(tracer.ops)} ops")
+                else:
+                    logger.debug(f"opinfo {op.name} has {len(tracer.ops)} ops")
+            except Exception:
+                continue
 
         for overload, indices in op_indices.items():
             if len(indices) > 0:

BackendBench/scripts/generate_operator_coverage_csv.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+"""Generate comprehensive operator coverage CSV for BackendBench"""
+
+import csv
+import torch
+
+from torch.testing._internal.common_methods_invocations import op_db
+from BackendBench.scripts.pytorch_operators import (
+    get_pytorch_operators,
+    extract_aten_ops,
+    extract_operator_name,
+)
+from BackendBench.opinfo_suite import OpInfoTestSuite
+from BackendBench.torchbench_suite import TorchBenchTestSuite
+
+
+def get_torchbench_ops():
+    """Get operations from TorchBench suite"""
+    suite = TorchBenchTestSuite("torchbench", None)
+    ops = set()
+    for optest in suite:
+        op_str = str(optest.op)
+        op_name = extract_operator_name(op_str)
+        ops.add(op_name)
+    return ops
+
+
+def generate_coverage_csv():
+    """Generate comprehensive operator coverage CSV"""
+    print("Gathering operator data...")
+
+    # Get all operators and core operators in one call
+    all_native_ops, core_ops = get_pytorch_operators()
+
+    # Get OpInfo operators
+    print("Building OpInfo tests for device=cpu, dtype=torch.float32")
+    suite = OpInfoTestSuite("opinfo", "cpu", torch.float32)
+    opinfo_successful_ops = [str(optest.op) for optest in suite]
+    print("\nOpInfo loading results:")
+    print(f"  Total ops in op_db: {len(op_db)}")
+    print(f"  Successful operations found: {len(opinfo_successful_ops)}")
+    print(f"  Unique successful ops: {len(set(opinfo_successful_ops))}")
+
+    opinfo_ops = set(extract_aten_ops(opinfo_successful_ops))
+    torchbench_ops = get_torchbench_ops()
+
+    print("\nOperator counts:")
+    print(f"- Total native functions: {len(all_native_ops)}")
+    print(f"- Core operators: {len(core_ops)}")
+    print(f"- OpInfo: {len(opinfo_ops)}")
+    print(f"- TorchBench: {len(torchbench_ops)}")
+
+    # Create comprehensive operator list
+    all_operators = set(all_native_ops) | set(core_ops) | opinfo_ops | torchbench_ops
+    core_ops_set = set(core_ops)
+
+    # Generate CSV
+    csv_data = [["op_name", "is_core", "is_in_opinfo", "is_in_torchbench"]]
+
+    for op in sorted(all_operators):
+        row = [
+            op,
+            True if op in core_ops_set else False,
+            True if op in opinfo_ops else False,
+            True if op in torchbench_ops else False,
+        ]
+        csv_data.append(row)
+
+    csv_filename = "pytorch_operator_coverage.csv"
+    with open(csv_filename, "w", newline="") as csvfile:
+        writer = csv.writer(csvfile)
+        writer.writerows(csv_data)
+
+    print(f"\nCSV generated: {csv_filename}")
+
+    # Analysis
+    core_in_opinfo = core_ops_set & opinfo_ops
+    core_in_torchbench = core_ops_set & torchbench_ops
+    core_in_either = core_ops_set & (opinfo_ops | torchbench_ops)
+    core_missing_both = core_ops_set - (opinfo_ops | torchbench_ops)
+
+    print(
+        f"\nCore in OpInfo: {len(core_in_opinfo)}/{len(core_ops)} ({len(core_in_opinfo) / len(core_ops) * 100:.1f}%)"
+    )
+    print(
+        f"Core in TorchBench: {len(core_in_torchbench)}/{len(core_ops)} ({len(core_in_torchbench) / len(core_ops) * 100:.1f}%)"
+    )
+    print(
+        f"Combined coverage: {len(core_in_either)}/{len(core_ops)} ({len(core_in_either) / len(core_ops) * 100:.1f}%)"
+    )
+    print(f"Missing from both: {sorted(core_missing_both)}")
+
+    return csv_filename
+
+
+if __name__ == "__main__":
+    csv_file = generate_coverage_csv()
+    print(f"\nAnalysis complete! CSV saved as: {csv_file}")

BackendBench/scripts/pytorch_operators.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""PyTorch operator utilities for BackendBench analysis"""
+
+import urllib.request
+import yaml
+from typing import List
+
+
+def extract_operator_name(op_str: str) -> str:
+    """Extract clean operator name from various operator string formats.
+
+    Note: We don't care about overloads - we treat all overloads of an operator
+    (e.g., add.Tensor, add.Scalar, add.out) as the same base operator.
+
+    Examples:
+        "aten.relu.default" -> "relu"
+        "torch.ops.aten.add.Tensor" -> "add"
+        "add.Tensor" -> "add"
+        "relu" -> "relu"
+    """
+    if "aten." in op_str:
+        return op_str.split("aten.")[-1].split(".")[0]
+    elif "." in op_str:
+        return op_str.split(".")[0]
+    else:
+        return op_str
+
+
+def get_deprecated_operators():
+    """Get deprecated operators from PyTorch's deprecated.yaml"""
+    url = "https://raw.githubusercontent.com/pytorch/pytorch/refs/heads/main/tools/autograd/deprecated.yaml"
+
+    deprecated_ops = set()
+    try:
+        print("Downloading deprecated.yaml...")
+        with urllib.request.urlopen(url) as response:
+            yaml_content = response.read().decode("utf-8")
+
+        deprecated_functions = yaml.safe_load(yaml_content)
+
+        if deprecated_functions:
+            for func_def in deprecated_functions:
+                if isinstance(func_def, dict) and "name" in func_def:
+                    func_name = func_def["name"]
+                    base_name = extract_operator_name(func_name)
+                    deprecated_ops.add(base_name)
+
+        print(f"Found {len(deprecated_ops)} deprecated operators")
+    except Exception as e:
+        print(f"Warning: Could not fetch deprecated operators: {e}")
+
+    return deprecated_ops
+
+
+def get_pytorch_operators():
+    """Get all operators and core operators from PyTorch's native_functions.yaml, excluding deprecated ones"""
+    url = "https://raw.githubusercontent.com/pytorch/pytorch/refs/heads/main/aten/src/ATen/native/native_functions.yaml"
+
+    print("Downloading native_functions.yaml...")
+    with urllib.request.urlopen(url) as response:
+        yaml_content = response.read().decode("utf-8")
+
+    functions = yaml.safe_load(yaml_content)
+    print(f"Found {len(functions)} function definitions")
+
+    # Get deprecated operators to exclude
+    deprecated_ops = get_deprecated_operators()
+
+    all_ops = set()
+    core_ops = set()
+
+    for func_def in functions:
+        if isinstance(func_def, dict) and "func" in func_def:
+            func_signature = func_def["func"]
+            func_name = func_signature.split("(")[0].strip()
+
+            base_name = extract_operator_name(func_name)
+
+            # Skip deprecated operators
+            if base_name in deprecated_ops:
+                continue
+
+            all_ops.add(base_name)
+
+            if "core" in func_def.get("tags", []):
+                core_ops.add(base_name)
+
+    all_ops_list = sorted([op for op in all_ops if op and not op.isspace()])
+    core_ops_list = sorted([op for op in core_ops if op and not op.isspace()])
+
+    print(f"Extracted {len(all_ops_list)} unique operators (excluding deprecated)")
+    print(f"Found {len(core_ops_list)} core operators (excluding deprecated)")
+
+    return all_ops_list, core_ops_list
+
+
+def extract_aten_ops(ops_list: List[str]) -> List[str]:
+    """Extract aten operation names from ops list"""
+    aten_ops = set()
+    for op_str in ops_list:
+        if "aten." in op_str:
+            aten_ops.add(extract_operator_name(op_str))
+    return sorted(aten_ops)