Merge pull request #393 from bayesflow-org/test-examples

Test examples

Author: LarsKue

.github/workflows/tests.yaml

@@ -99,7 +99,13 @@ jobs:
 
       - name: Run Tests
         run: |
-          pytest -x
+          pytest -x -m "not slow"
+
+      - name: Run Slow Tests
+        # run only on manual trigger
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          pytest -x -m "slow"
 
       - name: Create Coverage Report
         run: |

README.md

@@ -54,7 +54,7 @@ For an in-depth exposition, check out our walkthrough notebooks below.
 1. [Linear regression starter example](examples/Linear_Regression_Starter.ipynb)
 2. [From ABC to BayesFlow](examples/From_ABC_to_BayesFlow.ipynb)
 3. [Two moons starter example](examples/Two_Moons_Starter.ipynb)
-4. [Rapid iteration with point estimators](examples/Lotka_Volterra_point_estimation_and_expert_stats.ipynb)
+4. [Rapid iteration with point estimators](examples/Lotka_Volterra_Point_Estimation_and_Expert_Stats.ipynb)
 5. [SIR model with custom summary network](examples/SIR_Posterior_Estimation.ipynb)
 6. [Bayesian experimental design](examples/Bayesian_Experimental_Design.ipynb)
 7. [Simple model comparison example](examples/One_Sample_TTest.ipynb)

examples/Lotka_Volterra_point_estimation_and_expert_stats.ipynb renamed to examples/Lotka_Volterra_Point_Estimation_and_Expert_Stats.ipynb

@@ -37,6 +37,7 @@ all = [
     # dev
     "jupyter",
     "jupyterlab",
+    "nbconvert",
     "pre-commit",
     "ruff",
     "tox",
@@ -71,6 +72,7 @@ docs = [
     "sphinxcontrib-bibtex",
 ]
 test = [
+    "nbconvert",
     "pytest",
     "pytest-cov",
     "pytest-rerunfailures",

pyproject.toml

@@ -6,6 +6,7 @@ markers =
     numpy: mark test as requiring NumPy
     tensorflow: mark test as requiring TensorFlow
     torch: mark test as requiring PyTorch
+    slow: mark test as especially slow, such as running a whole fitting workflow
 
 testpaths =
     tests

pytest.ini

@@ -0,0 +1,8 @@
+
+import pytest
+
+
+@pytest.fixture(scope="session")
+def examples_path():
+    from pathlib import Path
+    return Path(__file__).parents[2] / "examples"

tests/test_examples/__init__.py

@@ -0,0 +1,39 @@
+
+import pytest
+
+from tests.utils import run_notebook
+
+
+@pytest.mark.slow
+def test_bayesian_experimental_design(examples_path):
+    run_notebook(examples_path / "Bayesian_Experimental_Design.ipynb")
+
+
+@pytest.mark.slow
+def test_from_abc_to_bayesflow(examples_path):
+    run_notebook(examples_path / "From_ABC_to_BayesFlow.ipynb")
+
+
+@pytest.mark.slow
+def test_linear_regression_starter(examples_path):
+    run_notebook(examples_path / "Linear_Regression_Starter.ipynb")
+
+
+@pytest.mark.slow
+def test_lotka_volterra_point_estimation_and_expert_stats(examples_path):
+    run_notebook(examples_path / "Lotka_Volterra_Point_Estimation_and_Expert_Stats.ipynb")
+
+
+@pytest.mark.slow
+def test_one_sample_ttest(examples_path):
+    run_notebook(examples_path / "One_Sample_TTest.ipynb")
+
+
+@pytest.mark.slow
+def test_sir_posterior_estimation(examples_path):
+    run_notebook(examples_path / "SIR_Posterior_estimation.ipynb")
+
+
+@pytest.mark.slow
+def test_two_moons_starter(examples_path):
+    run_notebook(examples_path / "Two_Moons_Starter.ipynb")

tests/test_examples/conftest.py

@@ -1,5 +1,6 @@
 from .assertions import *
 from .callbacks import *
-from .ops import *
-from .ecdf import *
 from .check_combinations import *
+from .ecdf import *
+from .jupyter import *
+from .ops import *

tests/test_examples/test_examples.py

@@ -0,0 +1,11 @@
+
+import nbformat
+from nbconvert.preprocessors import ExecutePreprocessor
+
+def run_notebook(path):
+    with open(str(path)) as f:
+        nb = nbformat.read(f, nbformat.NO_CONVERT)
+
+    kernel = ExecutePreprocessor(timeout=600, kernel_name="python3")
+
+    return kernel.preprocess(nb)