Add basic smoke test to CI (#3)

Author: msaroufim

.github/workflows/smoke-test.yml

@@ -0,0 +1,31 @@
+name: Smoke Test
+
+on:
+  push:
+
+jobs:
+  smoke-test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'
+    
+    - name: Cache pip dependencies
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    
+    - name: Install dependencies
+      run: |
+        pip install -r requirements.txt
+    
+    - name: Run smoke test
+      run: |
+        PYTHONPATH=. python scripts/main.py --suite smoke --backend aten 

BackendBench/eval.py

@@ -44,9 +44,27 @@ def eval_correctness(op, impl, tests):
     return correct / total
 
 
+def cpu_bench(fn, num_runs=100):
+    """Simple CPU benchmarking using time.perf_counter."""
+    import time
+
+    for _ in range(10):
+        fn()
+
+    start = time.perf_counter()
+    for _ in range(num_runs):
+        fn()
+    return (time.perf_counter() - start) / num_runs
+
+
 def eval_performance(op, impl, tests):
-    base_times = [do_bench(lambda: op(*test.args, **test.kwargs)) for test in tests]
-    test_times = [do_bench(lambda: impl(*test.args, **test.kwargs)) for test in tests]
+    if torch.cuda.is_available():
+        base_times = [do_bench(lambda: op(*test.args, **test.kwargs)) for test in tests]
+        test_times = [do_bench(lambda: impl(*test.args, **test.kwargs)) for test in tests]
+    else:
+        base_times = [cpu_bench(lambda: op(*test.args, **test.kwargs)) for test in tests]
+        test_times = [cpu_bench(lambda: impl(*test.args, **test.kwargs)) for test in tests]
+
     speedups = torch.tensor(test_times) / torch.tensor(base_times)
     # geometric mean of speedups
     return speedups.log().mean().exp()

BackendBench/opinfo_suite.py

@@ -67,9 +67,7 @@ def build_op_tests(device, dtype, filter=None):
                     if allclose(ref, res):
                         op_indices[tracer.ops[0]].append(idx)
                 except Exception:
-                    logger.debug(
-                        f"opinfo {op.name} couldn't run underlying op {tracer.ops[0]}"
-                    )
+                    logger.debug(f"opinfo {op.name} couldn't run underlying op {tracer.ops[0]}")
             else:
                 logger.debug(f"opinfo {op.name} has {len(tracer.ops)} ops")
 

BackendBench/suite.py

@@ -42,10 +42,10 @@ def __iter__(self):
         OpTest(
             torch.ops.aten.relu.default,
             [
-                Test(randn(2, device="cuda")),
+                Test(randn(2, device="cpu")),
             ],
             [
-                Test(randn(2**28, device="cuda")),
+                Test(randn(2**28, device="cpu")),
             ],
         )
     ],

pyproject.toml

@@ -0,0 +1,4 @@
+[tool.ruff]
+line-length = 100
+
+[tool.ruff.format]

requirements.txt

@@ -0,0 +1,4 @@
+torch
+click 
+numpy
+expecttest

scripts/main.py

@@ -70,9 +70,7 @@ def cli(suite, backend, ops):
 
     mean_correctness = torch.tensor(overall_correctness).mean().item()
     geomean_perf = torch.tensor(overall_performance).log().mean().exp().item()
-    print(
-        f"correctness score (mean pass rate over all operators): {mean_correctness:.2f}"
-    )
+    print(f"correctness score (mean pass rate over all operators): {mean_correctness:.2f}")
     print(f"performance score (geomean speedup over all operators): {geomean_perf:.2f}")