Merge branch 'dev' into add_strain_details

Author: CodyCBakerPhD

nwbinspector/nwbinspector.py

@@ -13,6 +13,7 @@
 from types import FunctionType
 from warnings import filterwarnings, warn
 from distutils.util import strtobool
+from time import sleep
 
 import click
 import pynwb
@@ -28,7 +29,7 @@
 )
 from .register_checks import InspectorMessage, Importance
 from .tools import get_s3_urls_and_dandi_paths
-from .utils import FilePathType, PathType, OptionalListOfStrings
+from .utils import FilePathType, PathType, OptionalListOfStrings, robust_s3_read, calculate_number_of_cpu
 
 INTERNAL_CONFIGS = dict(dandi=Path(__file__).parent / "internal_configs" / "dandi.inspector_config.yaml")
 
@@ -317,6 +318,8 @@ def inspect_all(
         The default is the lowest level, BEST_PRACTICE_SUGGESTION.
     n_jobs : int
         Number of jobs to use in parallel. Set to -1 to use all available resources.
+        This may also be a negative integer x from -2 to -(number_of_cpus - 1) which acts like negative slicing by using
+        all available CPUs minus x.
         Set to 1 (also the default) to disable.
     skip_validate : bool, optional
         Skip the PyNWB validation step. This may be desired for older NWBFiles (< schema version v2.10).
@@ -340,6 +343,7 @@ def inspect_all(
         Importance[importance_threshold] if isinstance(importance_threshold, str) else importance_threshold
     )
     modules = modules or []
+    n_jobs = calculate_number_of_cpu(requested_cpu=n_jobs)
     if progress_bar_options is None:
         progress_bar_options = dict(position=0, leave=False)
         if stream:
@@ -414,8 +418,9 @@ def inspect_nwb(
     ignore: OptionalListOfStrings = None,
     select: OptionalListOfStrings = None,
     importance_threshold: Union[str, Importance] = Importance.BEST_PRACTICE_SUGGESTION,
-    driver: str = None,
+    driver: Optional[str] = None,
     skip_validate: bool = False,
+    max_retries: int = 10,
 ) -> List[InspectorMessage]:
     """
     Inspect a NWBFile object and return suggestions for improvements according to best practices.
@@ -449,6 +454,11 @@ def inspect_nwb(
     skip_validate : bool
         Skip the PyNWB validation step. This may be desired for older NWBFiles (< schema version v2.10).
         The default is False, which is also recommended.
+    max_retries : int, optional
+        When using the ros3 driver to stream data from an s3 path, occasional curl issues can result.
+        AWS suggests using iterative retry with an exponential backoff of 0.1 * 2^retries.
+        This sets a hard bound on the number of times to attempt to retry the collection of messages.
+        Defaults to 10 (corresponds to 102.4s maximum delay on final attempt).
     """
     importance_threshold = (
         Importance[importance_threshold] if isinstance(importance_threshold, str) else importance_threshold
@@ -460,6 +470,7 @@ def inspect_nwb(
     nwbfile_path = str(nwbfile_path)
     filterwarnings(action="ignore", message="No cached namespaces found in .*")
     filterwarnings(action="ignore", message="Ignoring cached namespace .*")
+
     with pynwb.NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True, driver=driver) as io:
         if not skip_validate:
             validation_errors = pynwb.validate(io=io)
@@ -473,7 +484,7 @@ def inspect_nwb(
                 )
 
         try:
-            nwbfile = io.read()
+            nwbfile = robust_s3_read(command=io.read, max_retries=max_retries)
             for inspector_message in run_checks(nwbfile=nwbfile, checks=checks):
                 inspector_message.file_path = nwbfile_path
                 yield inspector_message
@@ -499,7 +510,7 @@ def run_checks(nwbfile: pynwb.NWBFile, checks: list):
         for nwbfile_object in nwbfile.objects.values():
             if check_function.neurodata_type is None or issubclass(type(nwbfile_object), check_function.neurodata_type):
                 try:
-                    output = check_function(nwbfile_object)
+                    output = robust_s3_read(command=check_function, command_args=[nwbfile_object])
                 # if an individual check fails, include it in the report and continue with the inspection
                 except Exception:
                     output = InspectorMessage(

nwbinspector/tools.py

@@ -7,7 +7,7 @@
 
 from pynwb import NWBFile
 
-from .utils import is_module_installed
+from .utils import is_module_installed, calculate_number_of_cpu
 
 
 def make_minimal_nwbfile():
@@ -43,6 +43,7 @@ def get_s3_urls_and_dandi_paths(dandiset_id: str, version_id: Optional[str] = No
     ), "The specified 'path' is not a proper DANDISet ID. It should be a six-digit numeric identifier."
 
     s3_urls_to_dandi_paths = dict()
+    n_jobs = calculate_number_of_cpu(requested_cpu=n_jobs)
     if n_jobs != 1:
         with DandiAPIClient() as client:
             dandiset = client.get_dandiset(dandiset_id=dandiset_id, version_id=version_id)

nwbinspector/utils.py

@@ -1,11 +1,13 @@
 """Commonly reused logic for evaluating conditions; must not have external dependencies."""
+import os
 import re
 import json
 import numpy as np
-from typing import TypeVar, Optional, List
+from typing import TypeVar, Optional, List, Dict, Callable
 from pathlib import Path
 from importlib import import_module
 from packaging import version
+from time import sleep
 
 PathType = TypeVar("PathType", str, Path)  # For types that can be either files or folders
 FilePathType = TypeVar("FilePathType", str, Path)
@@ -113,3 +115,41 @@ def get_package_version(name: str) -> version.Version:
 
         package_version = get_distribution(name).version
     return version.parse(package_version)
+
+
+def robust_s3_read(
+    command: Callable, max_retries: int = 10, command_args: Optional[list] = None, command_kwargs: Optional[Dict] = None
+):
+    """Attempt the command (usually acting on an S3 IO) up to the number of max_retries using exponential backoff."""
+    command_args = command_args or []
+    command_kwargs = command_kwargs or dict()
+    for retry in range(max_retries):
+        try:
+            return command(*command_args, **command_kwargs)
+        except OSError:  # cannot curl request
+            sleep(0.1 * 2**retry)
+        except Exception as exc:
+            raise exc
+    raise TimeoutError(f"Unable to complete the command ({command.__name__}) after {max_retries} attempts!")
+
+
+def calculate_number_of_cpu(requested_cpu: int = 1) -> int:
+    """
+    Calculate the number CPUs to use with respect to negative slicing and check against maximal available resources.
+
+    Parameters
+    ----------
+    requested_cpu : int, optional
+        The desired number of CPUs to use.
+
+        The default is 1.
+    """
+    total_cpu = os.cpu_count()
+    assert requested_cpu <= total_cpu, f"Requested more CPUs ({requested_cpu}) than are available ({total_cpu})!"
+    assert requested_cpu >= -(
+        total_cpu - 1
+    ), f"Requested fewer CPUs ({requested_cpu}) than are available ({total_cpu})!"
+    if requested_cpu > 0:
+        return requested_cpu
+    else:
+        return total_cpu + requested_cpu

setup.py

@@ -5,7 +5,7 @@
     long_description = f.read()
 setup(
     name="nwbinspector",
-    version="0.4.6",
+    version="0.4.7",
     description="Tool to inspect NWB files for best practices compliance.",
     long_description=long_description,
     long_description_content_type="text/markdown",

tests/test_utils.py

@@ -1,9 +1,16 @@
+import os
 from packaging import version
 
 from hdmf.testing import TestCase
 
 from nwbinspector import Importance
-from nwbinspector.utils import format_byte_size, check_regular_series, is_dict_in_string, get_package_version
+from nwbinspector.utils import (
+    format_byte_size,
+    check_regular_series,
+    is_dict_in_string,
+    get_package_version,
+    calculate_number_of_cpu,
+)
 
 
 def test_format_byte_size():
@@ -104,3 +111,27 @@ def test_get_package_version_type():
 
 def test_get_package_version_value():
     assert get_package_version("hdmf") >= version.parse("3.1.1")  # minimum supported PyNWB version
+
+
+class TestCalulcateNumberOfCPU(TestCase):
+    total_cpu = os.cpu_count()
+
+    def test_request_more_than_available_assert(self):
+        requested_cpu = 2500
+        with self.assertRaisesWith(
+            exc_type=AssertionError,
+            exc_msg=f"Requested more CPUs ({requested_cpu}) than are available ({self.total_cpu})!",
+        ):
+            calculate_number_of_cpu(requested_cpu=requested_cpu)
+
+    def test_request_fewer_than_available_assert(self):
+        requested_cpu = -2500
+        with self.assertRaisesWith(
+            exc_type=AssertionError,
+            exc_msg=f"Requested fewer CPUs ({requested_cpu}) than are available ({self.total_cpu})!",
+        ):
+            calculate_number_of_cpu(requested_cpu=requested_cpu)
+
+    def test_calculate_number_of_cpu_negative_value(self):
+        requested_cpu = -1  # CI only has 2 jobs available
+        assert calculate_number_of_cpu(requested_cpu=requested_cpu) == requested_cpu % self.total_cpu

tests/unit_tests/test_time_series.py

@@ -1,4 +1,5 @@
 from packaging import version
+from time import sleep
 
 import numpy as np
 import pynwb
@@ -14,7 +15,7 @@
     check_missing_unit,
     check_resolution,
 )
-from nwbinspector.utils import get_package_version
+from nwbinspector.utils import get_package_version, robust_s3_read
 
 try:
     # Test ros3 on sub-YutaMouse54/sub-YutaMouse54_ses-YutaMouse54-160630_behavior+ecephys.nwb from #3
@@ -195,9 +196,12 @@ def test_check_none_matnwb_resolution_pass():
         load_namespaces=True,
         driver="ros3",
     ) as io:
-        nwbfile = io.read()
-        time_series = nwbfile.processing["video_files"]["video"].time_series["20170203_KIB_01_s1.1.h264"]
-        assert check_resolution(time_series) is None
+        nwbfile = robust_s3_read(command=io.read)
+        time_series = robust_s3_read(
+            "20170203_KIB_01_s1.1.h264",
+            command=nwbfile.processing["video_files"]["video"].time_series.get,
+        )
+    assert check_resolution(time_series) is None
 
 
 def test_check_resolution_fail():