PR #770 openeo.utils.http finetuning (#441, #764)

Author: soxofaan

openeo/extra/job_management/__init__.py

@@ -29,7 +29,8 @@
 import shapely.errors
 import shapely.geometry.base
 import shapely.wkt
-from requests.adapters import HTTPAdapter, Retry
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
 
 from openeo import BatchJob, Connection
 from openeo.internal.processes.parse import (

openeo/rest/_connection.py

@@ -5,7 +5,7 @@
 from typing import Iterable, Optional, Union
 
 import requests
-import requests.adapters
+import urllib3.util
 from requests import Response
 from requests.auth import AuthBase
 
@@ -33,7 +33,7 @@ def __init__(
         session: Optional[requests.Session] = None,
         default_timeout: Optional[int] = None,
         slow_response_threshold: Optional[float] = None,
-        retry: Union[requests.adapters.Retry, dict, bool, None] = None,
+        retry: Union[urllib3.util.Retry, dict, bool, None] = None,
     ):
         self._root_url = root_url
         self._auth = None

openeo/rest/connection.py

@@ -28,8 +28,8 @@
 )
 
 import requests
-import requests.adapters
 import shapely.geometry.base
+import urllib3.util
 from requests.auth import AuthBase, HTTPBasicAuth
 
 import openeo
@@ -117,7 +117,7 @@ class Connection(RestApiConnection):
         Usage of this parameter is deprecated, use the specific authentication methods instead.
     :param retry: general request retry settings, can be specified as:
 
-        - :py:class:`requests.adapters.Retry` object
+        - :py:class:`urllib3.util.Retry` object
           or a dictionary with corresponding keyword arguments
           (e.g. ``total``, ``backoff_factor``, ``status_forcelist``, ...)
         - ``None`` (default) to use default openEO-oriented retry settings
@@ -141,7 +141,7 @@ def __init__(
         refresh_token_store: Optional[RefreshTokenStore] = None,
         oidc_auth_renewer: Optional[OidcAuthenticator] = None,
         auth: Optional[AuthBase] = None,
-        retry: Union[requests.adapters.Retry, dict, bool, None] = None,
+        retry: Union[urllib3.util.Retry, dict, bool, None] = None,
     ):
         if "://" not in url:
             url = "https://" + url
@@ -1898,7 +1898,7 @@ def connect(
     session: Optional[requests.Session] = None,
     default_timeout: Optional[int] = None,
     auto_validate: bool = True,
-    retry: Union[requests.adapters.Retry, dict, bool, None] = None,
+    retry: Union[urllib3.util.Retry, dict, bool, None] = None,
 ) -> Connection:
     """
     This method is the entry point to OpenEO.
@@ -1920,7 +1920,7 @@ def connect(
     :param auto_validate: toggle to automatically validate process graphs before execution
     :param retry: general request retry settings, can be specified as:
 
-        - :py:class:`requests.adapters.Retry` object
+        - :py:class:`urllib3.util.Retry` object
           or a dictionary with corresponding keyword arguments
           (e.g. ``total``, ``backoff_factor``, ``status_forcelist``, ...)
         - ``None`` (default) to use default openEO-oriented retry settings

openeo/utils/http.py

@@ -6,6 +6,7 @@
 
 import requests
 import requests.adapters
+from urllib3.util import Retry
 
 DEFAULT_RETRIES_TOTAL = 5
 
@@ -28,29 +29,29 @@
 )
 
 
-def retry_adapter(
+def retry_configuration(
     *,
     total: int = DEFAULT_RETRIES_TOTAL,
     backoff_factor: float = DEFAULT_BACKOFF_FACTOR,
     status_forcelist: Collection[int] = DEFAULT_RETRY_FORCELIST,
     **kwargs,
-) -> requests.adapters.Retry:
+) -> Retry:
     """
-    Factory for creating a `requests.adapters.Retry` configuration object with
+    Factory for creating a :py:class:`urllib3.util.retry.Retry` configuration object with
     openEO-oriented retry settings.
 
     :param total: Total number of retries to allow
     :param backoff_factor: scaling factor for sleeps between retries
     :param status_forcelist: A set of integer HTTP status codes that we should force a retry on.
-    :param kwargs: additional kwargs to pass to `requests.adapters.Retry`
+    :param kwargs: additional kwargs to pass to :py:class:`urllib3.util.retry.Retry`
     :return:
 
     Inspiration and references:
     - https://requests.readthedocs.io/en/latest/api/#requests.adapters.HTTPAdapter
     - https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.Retry
     - https://findwork.dev/blog/advanced-usage-python-requests-timeouts-retries-hooks/#retry-on-failure
     """
-    retry = requests.adapters.Retry(
+    retry = Retry(
         total=total,
         backoff_factor=backoff_factor,
         status_forcelist=status_forcelist,
@@ -59,32 +60,28 @@ def retry_adapter(
     return retry
 
 
-def _to_retry(
-    retry: Union[requests.adapters.Retry, dict, None],
-) -> requests.adapters.Retry:
+def _to_retry(retry: Union[Retry, dict, None]) -> Retry:
     """
-    Convert a retry specification to a `requests.adapters.Retry` object.
+    Convert a retry specification to a :py:class:`urllib3.util.retry.Retry` object.
     """
-    if isinstance(retry, requests.adapters.Retry):
-        return retry
+    if isinstance(retry, Retry):
+        pass
     elif isinstance(retry, dict):
-        adapter = retry_adapter(**retry)
+        retry = retry_configuration(**retry)
     elif retry in {None, True}:
-        adapter = retry_adapter()
+        retry = retry_configuration()
     else:
         raise ValueError(f"Invalid retry setting: {retry!r}")
-    return adapter
+    return retry
 
 
-def session_with_retries(
-    retry: Union[requests.adapters.Retry, dict, None] = None,
-) -> requests.Session:
+def session_with_retries(retry: Union[Retry, dict, None] = None) -> requests.Session:
     """
     Factory for a requests session with openEO-oriented retry settings.
 
     :param retry: The retry configuration, can be specified as:
-        - :py:class:`requests.adapters.Retry`
-        - a dictionary with :py:class:`requests.adapters.Retry` arguments,
+        - :py:class:`urllib3.util.retry.Retry`
+        - a dictionary with :py:class:`urllib3.util.retry.Retry` arguments,
           e.g. ``total``, ``backoff_factor``, ``status_forcelist``, ...
         - ``None`` for default openEO-oriented retry settings
     """

setup.py

@@ -72,6 +72,7 @@
     test_suite="tests",
     install_requires=[
         "requests>=2.26.0",
+        "urllib3>=2.0.0",
         "shapely>=1.6.4",
         "numpy>=1.17.0",
         "xarray>=0.12.3,<2025.01.2",  # TODO #721 xarray non-nanosecond support

tests/utils/test_http.py

@@ -1,5 +1,3 @@
-import contextlib
-import logging
 from typing import Iterator
 from unittest import mock