Merge pull request #550 from aai-institute/fix-notebook-tests

Fix notebook tests

Author: AnesBenmerzoug

.github/workflows/main.yaml

@@ -85,12 +85,16 @@ jobs:
 
   notebook-tests:
     strategy:
+      fail-fast: false
       matrix:
         python_version: ["3.8", "3.9", "3.10", "3.11"]
-    name: Run Notebook tests -  Python ${{ matrix.python_version }}
+        group_number: [1, 2, 3, 4]
+    name: Run Notebook tests -  Python ${{ matrix.python_version }} - Group ${{ matrix.group_number }}
     uses: ./.github/workflows/run-notebook-tests-workflow.yaml
     with:
       python_version: ${{ matrix.python_version }}
+      split_size: 4
+      group_number: ${{ matrix.group_number }}
     needs: [code-quality]
 
   push-docs-and-release-testpypi:

.github/workflows/run-notebook-tests-workflow.yaml

@@ -3,6 +3,14 @@ name: Run Notebook Tests
 on:
   workflow_call:
     inputs:
+      split_size:
+        description: "Determines the number of groups into which the tests should be split"
+        type: string
+        default: 4
+      group_number:
+        description: "Determines which which group of tests to run. Can be 1, 2, ..., split_size"
+        type: string
+        required: true      
       python_version:
         description: "Determines which Python version to use"
         type: string
@@ -30,5 +38,5 @@ jobs:
     - name: Set up memcached
       uses: niden/actions-memcached@v7
     - name: Test Notebooks
-      run: tox -e notebook-tests
-      continue-on-error: true
+      run: tox -e notebook-tests -- --durations-path .notebook_test_durations --splits ${{ inputs.split_size }} --group ${{ inputs.group_number }}
+

.notebook_test_durations

@@ -0,0 +1,11 @@
+{
+    "notebooks/data_oob.ipynb::": 14.608769827000287,
+    "notebooks/influence_imagenet.ipynb::": 13.570316236000508,
+    "notebooks/influence_sentiment_analysis.ipynb::": 20.546479973001624,
+    "notebooks/influence_synthetic.ipynb::": 5.9324631089984905,
+    "notebooks/influence_wine.ipynb::": 16.114133220999065,
+    "notebooks/least_core_basic.ipynb::": 14.312467472000208,
+    "notebooks/shapley_basic_spotify.ipynb::": 15.608795123000164,
+    "notebooks/shapley_knn_flowers.ipynb::": 3.9430189769991557,
+    "notebooks/shapley_utility_learning.ipynb::": 26.96671833400069
+}

notebooks/influence_sentiment_analysis.ipynb

@@ -1,7 +1,7 @@
 tox<4.0.0
 tox-wheel
 pre-commit==3.1.1
-black[jupyter] == 23.1.0
+black[jupyter] == 24.3.0
 isort == 5.12.0
 pylint==3.1.0
 pylint-json2html==0.5.0

notebooks/influence_wine.ipynb

@@ -1,7 +1,7 @@
 datasets==2.14.6
-distributed==2024.4.0
-pillow==10.0.1
+distributed==2023.4.0
+pillow==10.3.0
 torch==2.0.1
 torchvision==0.15.2
 transformers==4.36.0
-zarr==2.17.1
+zarr==2.16.1

notebooks/least_core_basic.ipynb

@@ -24,7 +24,7 @@ deps =
     {[testenv]deps}
     -r requirements-notebooks.txt
 commands =
-    pytest --nbmake --nbmake-timeout=30 -n 0 --cov "{envsitepackagesdir}/pydvl" notebooks/ {posargs}
+    pytest --nbmake --nbmake-timeout=60 -n 0 --cov "{envsitepackagesdir}/pydvl" notebooks/ {posargs}
 
 [testenv:linting]
 skip_install = true