Merge pull request #34 from Aharoni-Lab/restructure-tests

Restructure Testing Framework

Author: kaghi

.github/workflows/ci.yml

@@ -10,17 +10,15 @@ jobs:
   format:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.head_ref }}
+      - uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 
       - name: Cache PDM dependencies
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/pdm
@@ -36,31 +34,32 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pdm install --no-lock -G test
+          pdm install --check --no-lock -G test
 
-      - name: Run black
+      - name: Check code formatting (Black)
+        id: black
         run: |
-          pdm run black .
+          pdm run black --check .
 
-      - name: Commit changes
-        uses: stefanzweifel/git-auto-commit-action@v4
-        with:
-          commit_message: "style: format code with black"
-          branch: ${{ github.head_ref }}
+      - name: Help - Formatting
+        if: failure() && steps.black.outcome == 'failure'
+        run: |
+          echo "::error::❌ Code formatting check failed."
+          echo "To fix this, run the following command locally:"
+          echo "  pdm run black ."
 
   lint:
-    needs: format
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 
       - name: Cache PDM dependencies
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/pdm
@@ -74,43 +73,58 @@ jobs:
           python -m pip install --upgrade pip
           pip install pdm
 
-      - name: Lock dependencies
-        run: |
-          pdm fix
-          pdm lock --group :all
-
       - name: Install dependencies
         run: |
-          pdm install --no-lock -G test
+          pdm install --check --no-lock -G test
 
       - name: Run flake8
+        id: flake8
         run: |
           pdm run flake8
 
+      - name: Help - Flake8
+        if: failure() && steps.flake8.outcome == 'failure'
+        run: |
+          echo "::error::❌ Flake8 check failed."
+          echo "To fix this, run the following command locally:"
+          echo "  pdm run flake8"
+
+      - name: Run mypy
+        id: mypy
+        continue-on-error: true # Allow failure until codebase is fully typed
+        run: |
+          pdm run mypy .
+
+      - name: Help - Mypy
+        if: steps.mypy.outcome == 'failure'
+        run: |
+          echo "::warning::⚠️ Mypy check failed (non-blocking)."
+          echo "To view these errors locally, run:"
+          echo "  pdm run mypy ."
+
   test:
-    needs: lint
     runs-on: ubuntu-latest
-    timeout-minutes: 30 # Add timeout to prevent hanging
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.11"]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Cache PDM dependencies
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/pdm
             .pdm-build
-          key: ${{ runner.os }}-pdm-${{ hashFiles('**/pdm.lock') }}-${{ matrix.python-version }}
+          key: ${{ runner.os }}-pdm-${{ matrix.python-version }}-${{ hashFiles('**/pdm.lock') }}
           restore-keys: |
             ${{ runner.os }}-pdm-${{ matrix.python-version }}-
 
@@ -119,28 +133,30 @@ jobs:
           python -m pip install --upgrade pip
           pip install pdm
 
-      - name: Lock dependencies
-        run: |
-          pdm fix
-          pdm lock --group :all
-
       - name: Install dependencies
         run: |
-          pdm install --no-lock -G test
+          pdm install --check --no-lock -G test
 
-      - name: Run tests
-        run: |
-          pdm run pytest tests/ -v -m "not slow" -n auto --cov=indeca --cov-report=xml
-          pdm run plot_test
+      - name: Run Unit Tests
+        run: pdm run pytest tests/unit -v -n auto --cov=indeca --cov-report=xml
+
+      - name: Run Integration & Regression Tests
+        run: pdm run pytest tests/integration tests/regression -v -n auto -m "not slow" --cov=indeca --cov-report=xml --cov-append
+
+      - name: Run Other Tests (Robustness, Validation, Demo)
+        run: pdm run pytest tests/robustness tests/validation tests/demo -v -n auto -m "not slow" --cov=indeca --cov-report=xml --cov-append
+
+      - name: Generate Plot Artifacts
+        run: pdm run plot_test
 
       - uses: actions/upload-artifact@v4
-        continue-on-error: true
+        if: always() # Upload artifacts even if tests fail
         with:
-          name: test_output
+          name: test_output_${{ matrix.python-version }}
           path: tests/output
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           file: ./coverage.xml
-          fail_ci_if_error: false # Don't fail if coverage upload fails
+          fail_ci_if_error: false

benchmarks/routine/utils.py

@@ -12,7 +12,7 @@
 from scipy.spatial.distance import cdist
 from sklearn.metrics.pairwise import cosine_similarity
 
-from indeca.deconv import max_thres
+from indeca.core.deconv import max_thres
 from indeca.utils import norm
 
 

benchmarks/s00_benchmark_realds.py

@@ -14,7 +14,7 @@
 from routine.utils import compute_ROC
 
 from indeca import set_package_log_level
-from indeca.deconv import construct_R
+from indeca.core.deconv import construct_R
 from indeca.pipeline import pipeline_bin
 from tests.testing_utils.io import download_realds, load_gt_ds
 

benchmarks/s00_simulate_data.py

@@ -4,7 +4,7 @@
 
 import numpy as np
 
-from indeca.simulation import generate_data
+from indeca.core.simulation import generate_data
 
 OUT_PATH = "./intermediate/simulated/"
 PARAM_TAU_D = 6

benchmarks/s01_benchmark_AR.py

@@ -7,14 +7,14 @@
 import xarray as xr
 from tqdm.auto import tqdm
 
-from indeca.AR_kernel import (
+from indeca.core.AR_kernel import (
     convolve_g,
     convolve_h,
     estimate_coefs,
     solve_fit_h,
     solve_g,
 )
-from indeca.simulation import AR2exp, eval_exp, find_dhm, tau2AR
+from indeca.core.simulation import AR2exp, eval_exp, find_dhm, tau2AR
 from indeca.utils import scal_like
 
 IN_PATH = "./intermediate/simulated/simulated.nc"

benchmarks/s01_benchmark_bin.py

@@ -16,8 +16,8 @@
 )
 from tqdm.auto import tqdm
 
-from indeca.AR_kernel import estimate_coefs
-from indeca.deconv import construct_G, construct_R, max_thres
+from indeca.core.AR_kernel import estimate_coefs
+from indeca.core.deconv import construct_G, construct_R, max_thres
 from tests.testing_utils.cnmf import solve_deconv
 
 IN_PATH = {

benchmarks/s01_benchmark_pipeline.py

@@ -23,7 +23,7 @@
 from routine.utils import compute_ROC
 
 from indeca.pipeline import pipeline_bin
-from indeca.simulation import exp_pulse
+from indeca.core.simulation import exp_pulse
 from tests.testing_utils.cnmf import pipeline_cnmf
 
 IN_PATH = {
@@ -325,8 +325,11 @@
 from plotly.subplots import make_subplots
 from scipy.linalg import convolution_matrix
 
-from indeca.simulation import ar_pulse, exp_pulse, tau2AR
-from indeca.update_bin import max_thres, scal_lstsq, solve_deconv
+from indeca.core.simulation import ar_pulse, exp_pulse, tau2AR
+from indeca.core.deconv import max_thres
+
+# from indeca.core.deconv import solve_deconv  # TODO: missing function
+from indeca.utils.utils import scal_lstsq
 
 uid = 0
 iiter = 0
@@ -422,9 +425,12 @@
 from plotly.subplots import make_subplots
 from scipy.linalg import convolution_matrix
 
-from indeca.AR_kernel import solve_fit_h
-from indeca.simulation import ar_pulse, exp_pulse, tau2AR
-from indeca.update_bin import max_thres, scal_lstsq, solve_deconv, solve_deconv_l0
+from indeca.core.AR_kernel import solve_fit_h
+from indeca.core.simulation import ar_pulse, exp_pulse, tau2AR
+from indeca.core.deconv import max_thres
+
+# from indeca.core.deconv import solve_deconv, solve_deconv_l0  # TODO: missing functions
+from indeca.utils.utils import scal_lstsq
 
 uid = 0
 iiter = 0

pyproject.toml

@@ -73,6 +73,11 @@ addopts = "-n auto -v --cov=indeca --cov-report=term-missing"
 markers = [
     "slow: marks tests as slow (deselect with '-m \"not slow\"')",
     "integration: marks tests as integration tests",
+    "unit: marks tests as unit tests",
+    "regression: marks tests as regression tests",
+    "robustness: marks tests as robustness tests",
+    "validation: marks tests as validation tests",
+    "demo: marks tests as demo tests",
 ]
 
 [build-system]

tests/demo/__init__.py

@@ -0,0 +1,76 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+from indeca.core.deconv import DeconvBin
+from tests.conftest import fixt_deconv
+from tests.testing_utils.metrics import compute_f1_metrics, df_assign_metadata
+from tests.testing_utils.plotting import plot_met_ROC_thres
+
+pytestmark = pytest.mark.demo
+
+
+@pytest.mark.slow
+class TestDemoDeconv:
+    @pytest.mark.parametrize("taus", [(6, 1), (10, 3)])
+    @pytest.mark.parametrize("rand_seed", np.arange(15))
+    @pytest.mark.parametrize("upsamp", [1, 2])
+    @pytest.mark.parametrize("ns_lev", [0, 0.2, 0.5])
+    @pytest.mark.parametrize("y_scaling", [False])
+    def test_demo_solve_penal(
+        self, taus, rand_seed, upsamp, ns_lev, y_scaling, test_fig_path_svg, results_bag
+    ):
+        # act
+        deconv, y, c, c_org, s, s_org, scale = fixt_deconv(
+            taus=taus,
+            rand_seed=rand_seed,
+            upsamp=upsamp,
+            ns_lev=ns_lev,
+            y_scaling=y_scaling,
+        )
+        _, _, _, _, intm_free = deconv.solve_thres(
+            scaling=False, amp_constraint=False, return_intm=True
+        )
+        _, _, _, _, intm_nopn = deconv.solve_thres(scaling=True, return_intm=True)
+        _, _, _, _, opt_penal, intm_pn = deconv.solve_penal(
+            scaling=True, return_intm=True
+        )
+        # save results
+        intms = {"CNMF": intm_free, "No Penalty": intm_nopn, "Penalty": intm_pn}
+        metdf = []
+        for grp, cur_intm in intms.items():
+            if grp == "Penalty":
+                cur_svals = []
+                oidx = intm_pn[7]
+                for sv in intm_pn[2]:
+                    s_pad = np.zeros(deconv.T)
+                    s_pad[deconv.nzidx_s] = sv
+                    cur_svals.append(s_pad)
+            else:
+                cur_svals = cur_intm[2]
+            cur_met = compute_f1_metrics(
+                s_org,
+                cur_svals,
+                {
+                    "group": grp,
+                    "thres": cur_intm[1],
+                    "scals": cur_intm[5],
+                    "objs": cur_intm[6],
+                    "penal": opt_penal if grp == "Penalty" else 0,
+                    "opt_idx": cur_intm[7],
+                },
+                tdist_thres=2,
+            )
+            metdf.append(cur_met)
+        metdf = pd.concat(metdf, ignore_index=True)
+        metdf = df_assign_metadata(
+            metdf,
+            {"tau_d": taus[0], "tau_r": taus[1]},
+        )
+        results_bag.data = metdf
+        # plotting
+        fig = plot_met_ROC_thres(metdf, grad_color=False)
+        fig.savefig(test_fig_path_svg)
+        # assertion
+        if ns_lev == 0 and upsamp == 1:
+            assert (cur_svals[oidx][:-1] == s[:-1]).all()