ci: WIP Benchmarks (#733)

Author: DeaMariaLeon

.github/workflows/ci.yml

@@ -53,23 +53,7 @@ jobs:
         if: always()
         with:
           files: ./**/coverage*.xml
-  benchmarks:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-          cache: 'pip'
-      - name: Install asv
-        run: |
-          pip install asv
-          asv machine --yes
-      - name: Run benchmarks
-        run: |
-          asv run --quick
+
   examples:
     runs-on: ubuntu-latest
     steps:

.github/workflows/codespeed.yml

@@ -0,0 +1,28 @@
+name: codspeed-benchmarks
+
+on:
+  push:
+    branches:
+      #- "main" # or "master"
+      - "benchmarks" # or "master"
+  pull_request:
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: pip install ".[all]"
+
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v3
+        with:
+          run: pytest benchmarks/ --codspeed

benchmarks/test_benchmark_coo.py

@@ -0,0 +1,15 @@
+import sparse
+
+import numpy as np
+
+
+def test_matmul(benchmark):
+    rng = np.random.default_rng(seed=42)
+    x = sparse.random((40, 40), density=0.01, random_state=rng)
+    y = sparse.random((40, 40), density=0.01, random_state=rng)
+
+    x @ y  # Numba compilation
+
+    @benchmark
+    def test_matmul():
+        x @ y

benchmarks_original/__init__.py

@@ -0,0 +1,69 @@
+import importlib
+import operator
+import os
+
+import sparse
+
+from utils import benchmark
+
+import numpy as np
+import scipy.sparse as sps
+
+LEN = 10000
+DENSITY = 0.001
+ITERS = 3
+rng = np.random.default_rng(0)
+
+
+if __name__ == "__main__":
+    print("Elementwise Example:\n")
+
+    for func_name in ["multiply", "add", "greater_equal"]:
+        print(f"{func_name} benchmark:\n")
+
+        s1_sps = sps.random(LEN, LEN, format="csr", density=DENSITY, random_state=rng) * 10
+        s1_sps.sum_duplicates()
+        s2_sps = sps.random(LEN, LEN, format="csr", density=DENSITY, random_state=rng) * 10
+        s2_sps.sum_duplicates()
+
+        # ======= Finch =======
+        os.environ[sparse._ENV_VAR_NAME] = "Finch"
+        importlib.reload(sparse)
+
+        s1 = sparse.asarray(s1_sps.asformat("csc"), format="csc")
+        s2 = sparse.asarray(s2_sps.asformat("csc"), format="csc")
+
+        func = getattr(sparse, func_name)
+
+        # Compile & Benchmark
+        result_finch = benchmark(func, args=[s1, s2], info="Finch", iters=ITERS)
+
+        # ======= Numba =======
+        os.environ[sparse._ENV_VAR_NAME] = "Numba"
+        importlib.reload(sparse)
+
+        s1 = sparse.asarray(s1_sps)
+        s2 = sparse.asarray(s2_sps)
+
+        func = getattr(sparse, func_name)
+
+        # Compile & Benchmark
+        result_numba = benchmark(func, args=[s1, s2], info="Numba", iters=ITERS)
+
+        # ======= SciPy =======
+        s1 = s1_sps
+        s2 = s2_sps
+
+        if func_name == "multiply":
+            func, args = s1.multiply, [s2]
+        elif func_name == "add":
+            func, args = operator.add, [s1, s2]
+        elif func_name == "greater_equal":
+            func, args = operator.ge, [s1, s2]
+
+        # Compile & Benchmark
+        result_scipy = benchmark(func, args=args, info="SciPy", iters=ITERS)
+
+        np.testing.assert_allclose(result_numba.todense(), result_scipy.toarray())
+        np.testing.assert_allclose(result_finch.todense(), result_numba.todense())
+        np.testing.assert_allclose(result_finch.todense(), result_scipy.toarray())

benchmarks/benchmark_coo.py renamed to benchmarks_original/benchmark_coo.py

@@ -0,0 +1,64 @@
+import importlib
+import os
+
+import sparse
+
+from utils import benchmark
+
+import numpy as np
+import scipy.sparse as sps
+
+LEN = 100000
+DENSITY = 0.00001
+ITERS = 3
+rng = np.random.default_rng(0)
+
+
+if __name__ == "__main__":
+    print("Matmul Example:\n")
+
+    a_sps = sps.random(LEN, LEN - 10, format="csr", density=DENSITY, random_state=rng) * 10
+    a_sps.sum_duplicates()
+    b_sps = sps.random(LEN - 10, LEN, format="csr", density=DENSITY, random_state=rng) * 10
+    b_sps.sum_duplicates()
+
+    # ======= Finch =======
+    os.environ[sparse._ENV_VAR_NAME] = "Finch"
+    importlib.reload(sparse)
+
+    a = sparse.asarray(a_sps)
+    b = sparse.asarray(b_sps)
+
+    @sparse.compiled
+    def sddmm_finch(a, b):
+        return a @ b
+
+    # Compile & Benchmark
+    result_finch = benchmark(sddmm_finch, args=[a, b], info="Finch", iters=ITERS)
+
+    # ======= Numba =======
+    os.environ[sparse._ENV_VAR_NAME] = "Numba"
+    importlib.reload(sparse)
+
+    a = sparse.asarray(a_sps)
+    b = sparse.asarray(b_sps)
+
+    def sddmm_numba(a, b):
+        return a @ b
+
+    # Compile & Benchmark
+    result_numba = benchmark(sddmm_numba, args=[a, b], info="Numba", iters=ITERS)
+
+    # ======= SciPy =======
+    def sddmm_scipy(a, b):
+        return a @ b
+
+    a = a_sps
+    b = b_sps
+
+    # Compile & Benchmark
+    result_scipy = benchmark(sddmm_scipy, args=[a, b], info="SciPy", iters=ITERS)
+
+    # np.testing.assert_allclose(result_numba.todense(), result_scipy.toarray())
+    # np.testing.assert_allclose(result_finch.todense(), result_numba.todense())
+    # np.testing.assert_allclose(result_finch.todense(), result_scipy.toarray())