Merge pull request #722 from bioimage-io/dev

add BIOIMAGEIO_HTTP_TIMEOUT env var

Author: FynnBe

changelog.md

@@ -10,6 +10,10 @@ In this file we log both:
 
 This changelog includes implementation details and my reference the [changes to the Resource Description Format](#changes-to-the-resource-description-format), e.g. in entry [bioimageio.spec 0.5.2](#bioimageiospec-052).
 
+#### bioimageio.spec 0.5.5.6 (planned; not yet released)
+
+- Add BIOIMAGEIO_HTTP_TIMEOUT environment variable to adjust timeout for http requests.
+
 #### bioimageio.spec 0.5.5.5
 
 - use OS specific conda command to fix conda subprocess issues

src/bioimageio/spec/_internal/_settings.py

@@ -59,6 +59,9 @@ def _expand_user(cls, value: Path):
         2. Generate a new token at https://bioimage.io/#/api?tab=hypha-rpc
     """
 
+    http_timeout: float = 10.0
+    """Timeout in seconds for http requests."""
+
     id_map: str = (
         "https://uk1s3.embassy.ebi.ac.uk/public-datasets/bioimage.io/id_map.json"
     )

src/bioimageio/spec/_internal/field_validation.py

@@ -77,7 +77,7 @@ def validate_github_user(
         r = httpx.get(
             f"https://api.github.com/users/{username}",
             auth=settings.github_auth,
-            timeout=3,
+            timeout=settings.http_timeout,
         )
     except httpx.TimeoutException:
         issue_warning(

src/bioimageio/spec/_internal/io.py

@@ -760,7 +760,12 @@ def _fetch_url(
     elif settings.CI:
         headers["User-Agent"] = "ci"
 
-    r = httpx.get(str(source), follow_redirects=True, headers=headers)
+    r = httpx.get(
+        str(source),
+        follow_redirects=True,
+        headers=headers,
+        timeout=settings.http_timeout,
+    )
     _ = r.raise_for_status()
 
     # set progressbar.total

src/bioimageio/spec/_internal/io_utils.py

@@ -245,7 +245,9 @@ def _get_id_map_impl(url: str) -> Dict[str, LightHttpFileDescr]:
     if not isinstance(url, str) or "/" not in url:
         logger.opt(depth=1).error("invalid id map url: {}", url)
     try:
-        id_map_raw: Any = httpx.get(url, timeout=10, follow_redirects=True).json()
+        id_map_raw: Any = httpx.get(
+            url, timeout=settings.http_timeout, follow_redirects=True
+        ).json()
     except Exception as e:
         logger.opt(depth=1).error("failed to get {}: {}", url, e)
         return {}

src/bioimageio/spec/_internal/url.py

@@ -8,13 +8,14 @@
 from typing_extensions import Literal, assert_never
 
 from . import warning_levels
+from ._settings import settings
 from .field_warning import issue_warning
 from .root_url import RootHttpUrl
 from .validation_context import get_validation_context
 
 
 def _validate_url(url: Union[str, pydantic.HttpUrl]) -> pydantic.HttpUrl:
-    return _validate_url_impl(url, request_mode="head")
+    return _validate_url_impl(url, request_mode="head", timeout=settings.http_timeout)
 
 
 _KNOWN_VALID_URLS = ("https://zenodo.org/records/3446812/files/unet2d_weights.torch",)
@@ -24,7 +25,7 @@ def _validate_url(url: Union[str, pydantic.HttpUrl]) -> pydantic.HttpUrl:
 def _validate_url_impl(
     url: Union[str, pydantic.HttpUrl],
     request_mode: Literal["head", "get_stream", "get"],
-    timeout: int = 3,
+    timeout: float,
 ) -> pydantic.HttpUrl:
     url = str(url)
     context = get_validation_context()