✨ Implement retry strategy (#135)

* update workflow before publishing python package

* fix dependency issue and bump version

* point to website in project description

* fix broken dependency

* improve doc

* add github token to download artifacts

* ensure only read-access @wvangeit

* yet another attempt at downloading artifacts

* make sure to use repo that ran the trigger wf

* another attempt at fixing

* change owner

* allow publishing to testpypi also when pr

* minor change

* revert minor (but breaking) change

* minor fix

* add debug messages

* another debug message

* hopefully the final version

* final fix

* minor fix

* move master and tag to individual jobs

* add debug messages

* dev->post

* add python script for determining semantic version

* minor changes

* minor changes

* improve error handling and add version file to artifacts

* check if release

* minor fix

* ensure to enter venv

* also when tagging

* source venv in publishin workflow

* ensure only master

* add script for testing 'pure' semver

* adapt workflows to new python script

* minor change

* attempt to evaluate expressions correctly

* several fixes to fix tests

* ensure repo is checked out in publish workflow

* several small fixes

* cleanup

* debug

* minor cleanup

* mionr changes

* add debug message

* minor change

* minor change

* yet another try

* minor change

* minor change

* minor change

* mionr change

* minor changes

* correct workflow run id

* cosmetic change

* avoid using gh

* change to a single job for publishing

* minor cleanup

* swap loops in clean up jobs

* correction

* add retry

* improve type hints

* improve retry strategy @pcrespov

* py 3.7 compatible

* py 3.7 compatible

* another attempt

* minor fix

Author: bisgaard-itis

clients/python/client/osparc/_files_api.py

@@ -8,7 +8,7 @@
 from typing import Any, Iterator, List, Optional, Tuple, Union
 
 import httpx
-from httpx import AsyncClient, Response
+from httpx import Response
 from osparc_client import (
     BodyCompleteMultipartUploadV0FilesFileIdCompletePost,
     ClientFile,
@@ -114,7 +114,9 @@ async def upload_file_async(
 
             uploaded_parts: list[UploadedPart] = []
             print("- uploading chunks...")
-            async with AsyncHttpClient(timeout=timeout_seconds) as session:
+            async with AsyncHttpClient(
+                configuration=self.api_client.configuration, timeout=timeout_seconds
+            ) as session:
                 async for chunck, size in tqdm(
                     file_chunk_generator(file, chunk_size), total=n_urls
                 ):
@@ -130,6 +132,7 @@ async def upload_file_async(
                     )
 
                 async with AsyncHttpClient(
+                    configuration=self.api_client.configuration,
                     request_type="post",
                     url=links.abort_upload,
                     base_url=self.api_client.configuration.host,
@@ -148,7 +151,7 @@ async def upload_file_async(
 
         async def _complete_multipart_upload(
             self,
-            http_client: AsyncClient,
+            http_client: AsyncHttpClient,
             complete_link: str,
             client_file: ClientFile,
             uploaded_parts: List[UploadedPart],
@@ -167,7 +170,7 @@ async def _complete_multipart_upload(
 
         async def _upload_chunck(
             self,
-            http_client: AsyncClient,
+            http_client: AsyncHttpClient,
             chunck: bytes,
             chunck_size: int,
             upload_link: str,

clients/python/client/osparc/_http_client.py

@@ -1,7 +1,12 @@
-from typing import Optional
+from datetime import datetime
+from email.utils import parsedate_to_datetime
+from typing import Any, Awaitable, Callable, Optional, Set
 
 import httpx
 import tenacity
+from osparc_client import Configuration
+
+_RETRY_AFTER_STATUS_CODES: Set[int] = {429, 503}
 
 
 class AsyncHttpClient:
@@ -10,16 +15,18 @@ class AsyncHttpClient:
     def __init__(
         self,
         *,
+        configuration: Configuration,
         request_type: Optional[str] = None,
         url: Optional[str] = None,
-        **httpx_async_client_kwargs
+        **httpx_async_client_kwargs,
     ):
+        self.configuration = configuration
         self._client = httpx.AsyncClient(**httpx_async_client_kwargs)
         self._callback = getattr(self._client, request_type) if request_type else None
         self._url = url
 
-    async def __aenter__(self) -> httpx.AsyncClient:
-        return self._client
+    async def __aenter__(self) -> "AsyncHttpClient":
+        return self
 
     async def __aexit__(self, exc_type, exc_value, traceback) -> None:
         if exc_value is None:
@@ -41,3 +48,57 @@ async def __aexit__(self, exc_type, exc_value, traceback) -> None:
                     raise err from exc_value
             await self._client.aclose()
             raise exc_value
+
+    async def _request(
+        self, method: Callable[[Any], Awaitable[httpx.Response]], *args, **kwargs
+    ) -> httpx.Response:
+        n_attempts = self.configuration.retries.total
+        assert isinstance(n_attempts, int)
+
+        @tenacity.retry(
+            reraise=True,
+            wait=self._wait_callback,
+            stop=tenacity.stop_after_attempt(n_attempts),
+            retry=tenacity.retry_if_exception_type(httpx.HTTPStatusError),
+        )
+        async def _():
+            response: httpx.Response = await method(*args, **kwargs)
+            if response.status_code in self.configuration.retries.status_forcelist:
+                response.raise_for_status()
+            return response
+
+        return await _()
+
+    async def put(self, *args, **kwargs) -> httpx.Response:
+        return await self._request(self._client.put, *args, **kwargs)
+
+    async def post(self, *args, **kwargs) -> httpx.Response:
+        return await self._request(self._client.post, *args, **kwargs)
+
+    async def delete(self, *args, **kwargs) -> httpx.Response:
+        return await self._request(self._client.delete, *args, **kwargs)
+
+    async def patch(self, *args, **kwargs) -> httpx.Response:
+        return await self._request(self._client.patch, *args, **kwargs)
+
+    def _wait_callback(self, retry_state: tenacity.RetryCallState) -> int:
+        assert retry_state.outcome is not None
+        response: httpx.Response = retry_state.outcome.exception().response
+        if response.status_code in _RETRY_AFTER_STATUS_CODES:
+            retry_after = response.headers.get("Retry-After")
+            if retry_after is not None:
+                try:
+                    next_try = parsedate_to_datetime(retry_after)
+                    return int(
+                        (next_try - datetime.now(tz=next_try.tzinfo)).total_seconds()
+                    )
+                except (ValueError, TypeError):
+                    pass
+                try:
+                    return int(retry_after)
+                except ValueError:
+                    pass
+        # https://urllib3.readthedocs.io/en/stable/reference/urllib3.util.html#utilities
+        return self.configuration.retries.backoff_factor * (
+            2**retry_state.attempt_number
+        )

clients/python/test/test_osparc/conftest.py

@@ -0,0 +1,13 @@
+from typing import Iterator
+
+import osparc
+import pytest
+
+
+@pytest.fixture
+def cfg() -> Iterator[osparc.Configuration]:
+    yield osparc.Configuration(
+        host="https://1a52d8d7-9f9f-48e5-b2f0-a226e6b25f0b.com",
+        username="askjdg",
+        password="asdjbaskjdb",
+    )

clients/python/test/test_osparc/test_AsyncHttpClient.py

@@ -0,0 +1,52 @@
+import httpx
+import osparc
+import pytest
+from osparc._http_client import AsyncHttpClient
+
+
+@pytest.fixture
+def fake_retry_state():
+    def _(r: httpx.Response):
+        class Exc:
+            response = r
+
+        class Outcome:
+            def exception(self):
+                return Exc()
+
+        class FakeRetryCallState:
+            outcome = Outcome()
+
+        return FakeRetryCallState()
+
+    yield _
+
+
+def test_retry_strategy(cfg: osparc.Configuration, fake_retry_state):
+    async_client = AsyncHttpClient(
+        configuration=cfg,
+        request_type="get",
+        url="79ae41cc-0d89-4714-ac9d-c23ee1b110ce",
+    )
+    assert (
+        async_client._wait_callback(
+            fake_retry_state(
+                httpx.Response(
+                    status_code=503,
+                    headers={"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"},
+                )
+            )
+        )
+        < 0
+    )
+    assert (
+        async_client._wait_callback(
+            fake_retry_state(
+                httpx.Response(
+                    status_code=503,
+                    headers={"Retry-After": "15"},
+                )
+            )
+        )
+        == 15
+    )