Support fetching models from GitHub. Make cache expire. (#30)

- Unzip the compressed data file in model 244262
- Update cache key to include the month
  => cache is never more than a month out of date.
- Support a github key in the YAML configuration that overwrites
  the ModelDB copy of a model with the GitHub copy.

Author: olupton

.github/workflows/nrn-modeldb-ci.yaml

@@ -83,6 +83,7 @@ jobs:
       if: github.event_name == 'pull_request'
 
     - name: Install dependencies and project
+      id: install-deps
       run: |
         set
         # Set up Xvfb
@@ -93,15 +94,16 @@ jobs:
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         #Install project in editable mode
         python -m pip install -e .
+        echo "::set-output name=date::$(date -u "+%Y%m")"
     
     - name: Cache ModelDB models
       id: cache-models
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
         path: |
           cache
           modeldb/modeldb-meta.yaml
-        key: models
+        key: models-${{steps.install-deps.outputs.date}}
 
     - name: Get ModelDB models
       if: steps.cache-models.outputs.cache-hit != 'true'

modeldb/modeldb-run.yaml

@@ -1064,6 +1064,10 @@
 223962:
     skip: true
     comment: takes too long, need to see how to reduce time
+244262:
+    script:
+    - if [[ ! -f Iintra.dat ]]; then unzip Iintra.dat.zip; fi
+    - ls -l Iintra.dat
 149174:
     skip: true
     comment: testing in Python + hardcoded NEURON

modeldb/modeldb.py

@@ -4,14 +4,16 @@
 import base64
 import os
 import requests
+import time
 from .progressbar import ProgressBar
 import yaml
 from .data import Model
 from .config import *
 import traceback
 from pprint import pformat
 
-def download_model(model_id):
+def download_model(arg_tuple):
+    model_id, model_run_info = arg_tuple
     try:
         model_json = requests.get(MDB_MODEL_DOWNLOAD_URL.format(model_id=model_id)).json()
         model = Model(
@@ -31,7 +33,41 @@ def download_model(model_id):
         )
         with open(model_zip_uri, "wb+") as zipfile:
             zipfile.write(base64.standard_b64decode(url["file_content"]))
-    except Exception as e:   #  noqa
+
+        if "github" in model_run_info:
+            # This means we should try to replace the version of the model that
+            # we downloaded from the ModelDB API just above with a version from
+            # GitHub
+            github = model_run_info["github"]
+            if github == "default":
+                suffix = ""
+            elif github.startswith("pull/"):
+                pr_number = int(github[5:])
+                suffix = "/pull/{}/head".format(pr_number)
+            else:
+                raise Exception("Invalid value for github key: {}".format(github))
+            github_url = "https://api.github.com/repos/ModelDBRepository/{model_id}/zipball{suffix}".format(
+                model_id=model_id, suffix=suffix
+            )
+            # Replace the local file `model_zip_uri` with the zip file we
+            # downloaded from `github_url`
+            num_attempts = 3
+            status_codes = []
+            for _ in range(num_attempts):
+                github_response = requests.get(github_url)
+                status_codes.append(github_response.status_code)
+                if github_response.status_code == requests.codes.ok:
+                    break
+                time.sleep(5)
+            else:
+                raise Exception(
+                    "Failed to download {} with status codes {}".format(
+                        github_url, status_codes
+                    )
+                )
+            with open(model_zip_uri, "wb+") as zipfile:
+                zipfile.write(github_response.content)
+    except Exception as e:  #  noqa
         model = e
 
     return model_id, model
@@ -64,7 +100,10 @@ def _download_models(self, model_list=None):
             os.mkdir(MODELS_ZIP_DIR)
         models = requests.get(MDB_NEURON_MODELS_URL).json() if model_list is None else model_list
         pool = multiprocessing.Pool()
-        processed_models = pool.imap_unordered(download_model, models)
+        processed_models = pool.imap_unordered(
+            download_model,
+            [(model_id, self._run_instr.get(model_id, {})) for model_id in models],
+        )
         download_err = {}
         for model_id, model in ProgressBar.iter(processed_models, len(models)):
             if not isinstance(model, Exception):