Cache remote files for tests (#2313)

Cache remote files for tests locally via pooch, and cache the pooch cache for GitHub actions.

Today, I had dozens of workflow failures due to rate-limiting or unavailability of biomodels. This is avoidable.

Author: dweindl

.github/workflows/test_python_cplusplus.yml

@@ -18,6 +18,13 @@ jobs:
         python-version: [ "3.9" ]
 
     steps:
+    - name: Cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cache/pooch
+        key: ${{ runner.os }}-py${{ matrix.python-version }}-${{ github.job }}
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
@@ -265,6 +272,13 @@ jobs:
     runs-on: macos-latest
 
     steps:
+    - name: Cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/Library/Caches/pooch
+        key: ${{ runner.os }}-py${{ matrix.python-version }}-${{ github.job }}
+
     - name: Set up Python
       uses: actions/setup-python@v5
       with:

python/sdist/setup.cfg

@@ -62,6 +62,7 @@ test =
     #  unsupported x86_64 / x86_64h
     antimony!=2.14; platform_system=='Darwin' and platform_machine in 'x86_64h'
     scipy
+    pooch
 vis =
     matplotlib
     seaborn

python/tests/test_conserved_quantities_demartino.py

@@ -155,15 +155,15 @@
 def data_demartino2014():
     """Get tests from DeMartino2014 Suppl. Material"""
     import gzip
-    import io
-    import urllib.request
+    import pooch
 
     # stoichiometric matrix
-    response = urllib.request.urlopen(
-        r"https://github.com/AMICI-dev/AMICI/files/11430971/DeMartinoDe2014_test-ecoli.dat.gz",
-        timeout=10,
+    data = gzip.GzipFile(
+        pooch.retrieve(
+            "https://github.com/AMICI-dev/AMICI/files/11430971/DeMartinoDe2014_test-ecoli.dat.gz",
+            known_hash="md5:899873f8f1c413d13c3f8e94c1496b7e",
+        )
     )
-    data = gzip.GzipFile(fileobj=io.BytesIO(response.read()))
     S = [
         int(item)
         for sl in [
@@ -174,14 +174,13 @@ def data_demartino2014():
     ]
 
     # metabolite / row names
-    response = urllib.request.urlopen(
-        r"https://github.com/AMICI-dev/AMICI/files/11430970/test-ecoli-met.txt",
-        timeout=10,
-    )
-    row_names = [
-        entry.decode("ascii").strip() for entry in io.BytesIO(response.read())
-    ]
-
+    with open(
+        pooch.retrieve(
+            "https://github.com/AMICI-dev/AMICI/files/11430970/test-ecoli-met.txt",
+            known_hash="md5:d71e711a3655311390b38d00dcd6aa7f",
+        )
+    ) as f:
+        row_names = [entry.strip() for entry in f.readlines()]
     return S, row_names
 
 

python/tests/test_sbml_import.py

@@ -3,7 +3,6 @@
 import re
 from numbers import Number
 from pathlib import Path
-from urllib.request import urlopen
 
 import amici
 import libsbml
@@ -543,13 +542,13 @@ def test_sympy_exp_monkeypatch():
     monkeypatching sympy.Pow._eval_derivative in order to be able to compute
     non-nan sensitivities
     """
-    url = (
-        "https://www.ebi.ac.uk/biomodels/model/download/BIOMD0000000529.2?"
-        "filename=BIOMD0000000529_url.xml"
-    )
-    importer = amici.SbmlImporter(
-        urlopen(url, timeout=20).read().decode("utf-8"), from_file=False
+    import pooch
+
+    model_file = pooch.retrieve(
+        url="https://www.ebi.ac.uk/biomodels/model/download/BIOMD0000000529.2?filename=BIOMD0000000529_url.xml",
+        known_hash="md5:c6e0b298397485b93d7acfab80b21fd4",
     )
+    importer = amici.SbmlImporter(model_file)
     module_name = "BIOMD0000000529"
 
     with TemporaryDirectory() as outdir: