implemented as functional

Author: CodyCBakerPhD

nwbinspector/nwbinspector.py

@@ -29,7 +29,7 @@
 )
 from .register_checks import InspectorMessage, Importance
 from .tools import get_s3_urls_and_dandi_paths
-from .utils import FilePathType, PathType, OptionalListOfStrings
+from .utils import FilePathType, PathType, OptionalListOfStrings, robust_s3_read
 
 INTERNAL_CONFIGS = dict(dandi=Path(__file__).parent / "internal_configs" / "dandi.inspector_config.yaml")
 
@@ -462,51 +462,30 @@ def inspect_nwb(
     filterwarnings(action="ignore", message="No cached namespaces found in .*")
     filterwarnings(action="ignore", message="Ignoring cached namespace .*")
 
-    def _collect_all_messages(
-        nwbfile_path: FilePathType, checks: list, driver: Optional[str] = None, skip_validate: bool = False
-    ):
-        with pynwb.NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True, driver=driver) as io:
-            if not skip_validate:
-                validation_errors = pynwb.validate(io=io)
-                for validation_error in validation_errors:
-                    yield InspectorMessage(
-                        message=validation_error.reason,
-                        importance=Importance.PYNWB_VALIDATION,
-                        check_function_name=validation_error.name,
-                        location=validation_error.location,
-                        file_path=nwbfile_path,
-                    )
-
-            try:
-                nwbfile = io.read()
-                for inspector_message in run_checks(nwbfile=nwbfile, checks=checks):
-                    inspector_message.file_path = nwbfile_path
-                    yield inspector_message
-            except OSError as ex:
-                raise ex  # propagate outside private
-            except Exception as ex:
+    with pynwb.NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True, driver=driver) as io:
+        if not skip_validate:
+            validation_errors = pynwb.validate(io=io)
+            for validation_error in validation_errors:
                 yield InspectorMessage(
-                    message=traceback.format_exc(),
-                    importance=Importance.ERROR,
-                    check_function_name=f"During io.read() - {type(ex)}: {str(ex)}",
+                    message=validation_error.reason,
+                    importance=Importance.PYNWB_VALIDATION,
+                    check_function_name=validation_error.name,
+                    location=validation_error.location,
                     file_path=nwbfile_path,
                 )
 
-    if driver != "ros3":
-        return _collect_all_messages(
-            nwbfile_path=nwbfile_path, checks=checks, driver=driver, skip_validate=skip_validate
-        )
-    else:
-        retries = 0
-
-        while retries < max_retries:
-            try:
-                retries += 1
-                return _collect_all_messages(
-                    nwbfile_path=nwbfile_path, checks=checks, driver=driver, skip_validate=skip_validate
-                )
-            except OSError:  # Cannot curl request
-                sleep(0.1 * 2**retries)
+        try:
+            nwbfile = robust_s3_read(command=io.read, max_retries=max_retries)
+            for inspector_message in run_checks(nwbfile=nwbfile, checks=checks):
+                inspector_message.file_path = nwbfile_path
+                yield inspector_message
+        except Exception as ex:
+            yield InspectorMessage(
+                message=traceback.format_exc(),
+                importance=Importance.ERROR,
+                check_function_name=f"During io.read() - {type(ex)}: {str(ex)}",
+                file_path=nwbfile_path,
+            )
 
 
 def run_checks(nwbfile: pynwb.NWBFile, checks: list):
@@ -522,7 +501,7 @@ def run_checks(nwbfile: pynwb.NWBFile, checks: list):
         for nwbfile_object in nwbfile.objects.values():
             if check_function.neurodata_type is None or issubclass(type(nwbfile_object), check_function.neurodata_type):
                 try:
-                    output = check_function(nwbfile_object)
+                    output = robust_s3_read(command=check_function, command_args=[nwbfile_object])
                 # if an individual check fails, include it in the report and continue with the inspection
                 except Exception:
                     output = InspectorMessage(

nwbinspector/utils.py

@@ -2,10 +2,11 @@
 import re
 import json
 import numpy as np
-from typing import TypeVar, Optional, List
+from typing import TypeVar, Optional, List, Dict, Callable
 from pathlib import Path
 from importlib import import_module
 from packaging import version
+from time import sleep
 
 PathType = TypeVar("PathType", str, Path)  # For types that can be either files or folders
 FilePathType = TypeVar("FilePathType", str, Path)
@@ -113,3 +114,19 @@ def get_package_version(name: str) -> version.Version:
 
         package_version = get_distribution(name).version
     return version.parse(package_version)
+
+
+def robust_s3_read(
+    command: Callable, max_retries: int = 10, command_args: Optional[list] = None, command_kwargs: Optional[Dict] = None
+):
+    """Attempt the command (usually acting on an S3 IO) up to the number of max_retries using exponential backoff."""
+    command_args = command_args or []
+    command_kwargs = command_kwargs or dict()
+    for retry in range(max_retries):
+        try:
+            return command(*command_args, **command_kwargs)
+        except OSError:  # cannot curl request
+            sleep(0.1 * 2**retry)
+        except Exception as exc:
+            raise exc
+    raise TimeoutError(f"Unable to complete the command ({command.__name__}) after {max_retries} attempts!")

tests/unit_tests/test_time_series.py

@@ -15,7 +15,7 @@
     check_missing_unit,
     check_resolution,
 )
-from nwbinspector.utils import get_package_version
+from nwbinspector.utils import get_package_version, robust_s3_read
 
 try:
     # Test ros3 on sub-YutaMouse54/sub-YutaMouse54_ses-YutaMouse54-160630_behavior+ecephys.nwb from #3
@@ -190,22 +190,17 @@ def test_check_none_matnwb_resolution_pass():
 
     produced with MatNWB, when read with PyNWB~=2.0.1 and HDMF<=3.2.1 contains a resolution value of None.
     """
-    max_retries = 10
-    retries = 0
-
-    while retries < max_retries:
-        try:
-            retries += 1
-            with pynwb.NWBHDF5IO(
-                path="https://dandiarchive.s3.amazonaws.com/blobs/da5/107/da510761-653e-4b81-a330-9cdae4838180",
-                mode="r",
-                load_namespaces=True,
-                driver="ros3",
-            ) as io:
-                nwbfile = io.read()
-                time_series = nwbfile.processing["video_files"]["video"].time_series["20170203_KIB_01_s1.1.h264"]
-        except OSError:  # Cannot curl request
-            sleep(0.1 * 2**retries)
+    with pynwb.NWBHDF5IO(
+        path="https://dandiarchive.s3.amazonaws.com/blobs/da5/107/da510761-653e-4b81-a330-9cdae4838180",
+        mode="r",
+        load_namespaces=True,
+        driver="ros3",
+    ) as io:
+        nwbfile = robust_s3_read(command=io.read)
+        time_series = robust_s3_read(
+            "20170203_KIB_01_s1.1.h264",
+            command=nwbfile.processing["video_files"]["video"].time_series.get,
+        )
     assert check_resolution(time_series) is None