Refactor suite namespace (#103)

Author: jiannanWang

BackendBench/scripts/generate_operator_coverage_csv.py

@@ -17,8 +17,7 @@
     extract_aten_ops,
     extract_operator_name,
 )
-from BackendBench.opinfo_suite import OpInfoTestSuite
-from BackendBench.torchbench_suite import TorchBenchTestSuite
+from BackendBench.suite import OpInfoTestSuite, TorchBenchTestSuite
 
 
 def get_torchbench_ops():

BackendBench/scripts/get_tests_stat.py

@@ -11,9 +11,7 @@
 import statistics
 
 import torch
-from BackendBench.facto_suite import FactoTestSuite
-from BackendBench.opinfo_suite import OpInfoTestSuite
-from BackendBench.torchbench_suite import TorchBenchTestSuite
+from BackendBench.suite import OpInfoTestSuite, TorchBenchTestSuite, FactoTestSuite
 from BackendBench.scripts.pytorch_operators import extract_operator_name
 
 

BackendBench/scripts/main.py

@@ -15,11 +15,14 @@
 import click
 import torch
 
-from BackendBench.facto_suite import FactoTestSuite
 from BackendBench.llm_client import ClaudeKernelGenerator, LLMKernelGenerator
-from BackendBench.opinfo_suite import OpInfoTestSuite
-from BackendBench.suite import SmokeTestSuite
-from BackendBench.torchbench_suite import DEFAULT_HUGGINGFACE_URL, TorchBenchTestSuite
+from BackendBench.suite import (
+    SmokeTestSuite,
+    OpInfoTestSuite,
+    DEFAULT_HUGGINGFACE_URL,
+    TorchBenchTestSuite,
+    FactoTestSuite,
+)
 
 logger = logging.getLogger(__name__)
 

BackendBench/suite/__init__.py

@@ -0,0 +1,33 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD 3-Clause license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+BackendBench suites submodule.
+
+This module provides various test suite implementations for benchmarking
+PyTorch operations across different backends. Each test suite defines a
+collection of tests to evaluate the correctness and/or performacne of
+backend implementations by comparing them against PyTorch operations.
+"""
+
+from .base import Test, OpTest, TestSuite
+from .facto import FactoTestSuite
+from .opinfo import OpInfoTestSuite
+from .smoke import SmokeTestSuite, randn
+from .torchbench import TorchBenchOpTest, TorchBenchTestSuite, DEFAULT_HUGGINGFACE_URL
+
+__all__ = [
+    "Test",
+    "OpTest",
+    "TestSuite",
+    "FactoTestSuite",
+    "OpInfoTestSuite",
+    "SmokeTestSuite",
+    "randn",
+    "TorchBenchOpTest",
+    "TorchBenchTestSuite",
+    "DEFAULT_HUGGINGFACE_URL",
+]

BackendBench/suite.py renamed to BackendBench/suite/base.py

@@ -4,14 +4,6 @@
 # This source code is licensed under the BSD 3-Clause license found in the
 # LICENSE file in the root directory of this source tree.
 
-import torch
-
-from BackendBench.opregistry import get_operator
-
-
-def randn(*args, **kwargs):
-    return lambda: torch.randn(*args, **kwargs)
-
 
 class Test:
     def __init__(self, *args, **kwargs):
@@ -42,19 +34,3 @@ def __init__(self, name, optests):
     def __iter__(self):
         for optest in self.optests:
             yield optest
-
-
-SmokeTestSuite = TestSuite(
-    "smoke",
-    [
-        OpTest(
-            get_operator(torch.ops.aten.relu.default),
-            [
-                Test(randn(2, device="cpu")),
-            ],
-            [
-                Test(randn(2**28, device="cpu")),
-            ],
-        )
-    ],
-)

BackendBench/facto_suite.py renamed to BackendBench/suite/facto.py

@@ -20,9 +20,9 @@
     SpecDictDB = None
 
 
-from .eval import allclose
-from .opregistry import get_operator
-from .suite import OpTest, TestSuite
+from BackendBench.eval import allclose
+from BackendBench.opregistry import get_operator
+from .base import OpTest, TestSuite
 
 logger = logging.getLogger(__name__)
 

BackendBench/opinfo_suite.py renamed to BackendBench/suite/opinfo.py

@@ -10,8 +10,8 @@
 from torch.testing._internal.common_methods_invocations import op_db
 from torch.utils._python_dispatch import TorchDispatchMode
 
-from .eval import allclose
-from .suite import OpTest, TestSuite
+from BackendBench.eval import allclose
+from .base import OpTest, TestSuite
 
 logger = logging.getLogger(__name__)
 

BackendBench/suite/smoke.py

@@ -0,0 +1,30 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD 3-Clause license found in the
+# LICENSE file in the root directory of this source tree.
+
+import torch
+
+from BackendBench.opregistry import get_operator
+from .base import Test, OpTest, TestSuite
+
+
+def randn(*args, **kwargs):
+    return lambda: torch.randn(*args, **kwargs)
+
+
+SmokeTestSuite = TestSuite(
+    "smoke",
+    [
+        OpTest(
+            get_operator(torch.ops.aten.relu.default),
+            [
+                Test(randn(2, device="cpu")),
+            ],
+            [
+                Test(randn(2**28, device="cpu")),
+            ],
+        )
+    ],
+)

BackendBench/torchbench_suite.py renamed to BackendBench/suite/torchbench.py

@@ -5,7 +5,7 @@
 # LICENSE file in the root directory of this source tree.
 
 import pytest
-from BackendBench.torchbench_suite import TorchBenchOpTest
+from BackendBench.suite import TorchBenchOpTest
 import BackendBench.multiprocessing_eval as multiprocessing_eval
 import BackendBench.backends as backends
 import torch