Skip to content

Commit 96ff977

Browse files
bisgaard-itisbisgaard-itis
authored
🐛 fix abort multipart upload (#142)
* update workflow before publishing python package * fix dependency issue and bump version * point to website in project description * fix broken dependency * improve doc * add github token to download artifacts * ensure only read-access @wvangeit * yet another attempt at downloading artifacts * make sure to use repo that ran the trigger wf * another attempt at fixing * change owner * allow publishing to testpypi also when pr * minor change * revert minor (but breaking) change * minor fix * add debug messages * another debug message * hopefully the final version * final fix * minor fix * move master and tag to individual jobs * add debug messages * dev->post * add python script for determining semantic version * minor changes * minor changes * improve error handling and add version file to artifacts * check if release * minor fix * ensure to enter venv * also when tagging * source venv in publishin workflow * ensure only master * add script for testing 'pure' semver * adapt workflows to new python script * minor change * attempt to evaluate expressions correctly * several fixes to fix tests * ensure repo is checked out in publish workflow * several small fixes * cleanup * debug * minor cleanup * mionr changes * add debug message * minor change * minor change * yet another try * minor change * minor change * minor change * mionr change * minor changes * correct workflow run id * cosmetic change * avoid using gh * change to a single job for publishing * minor cleanup * swap loops in clean up jobs * correction * update server compatibility to new url * minor change to trigger ci * fix abort multipart upload * start adding test * minor corrections * minor change * add test * assert mock is called * minor change * @pcrespov mention file in logs * @pcrespov nosec
1 parent 13a8586 commit 96ff977

File tree

5 files changed

+79
-23
lines changed

5 files changed

+79
-23
lines changed

‎clients/python/client/osparc/_files_api.py‎

Lines changed: 29 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import httpx
1212
from httpx import Response
1313
from osparc_client import (
14+
BodyAbortMultipartUploadV0FilesFileIdAbortPost,
1415
BodyCompleteMultipartUploadV0FilesFileIdCompletePost,
1516
ClientFile,
1617
ClientFileUploadData,
@@ -104,7 +105,7 @@ async def upload_file_async(
104105
)
105106
client_upload_schema: ClientFileUploadData = self._super.get_upload_links(
106107
client_file=client_file, _request_timeout=timeout_seconds
107-
)
108+
) # type: ignore
108109
chunk_size: int = client_upload_schema.upload_schema.chunk_size
109110
links: FileUploadData = client_upload_schema.upload_schema.links
110111
url_iter: Iterator[Tuple[int, str]] = enumerate(
@@ -121,12 +122,12 @@ async def upload_file_async(
121122
configuration=self.api_client.configuration, timeout=timeout_seconds
122123
) as session:
123124
with logging_redirect_tqdm():
124-
_logger.info("Uploading %i chunks", n_urls)
125+
_logger.info("Uploading %s in %i chunks", file.name, n_urls)
125126
async for chunck, size in tqdm(
126127
file_chunk_generator(file, chunk_size),
127128
total=n_urls,
128129
disable=(not _logger.isEnabledFor(logging.INFO)),
129-
):
130+
): # type: ignore
130131
index, url = next(url_iter)
131132
uploaded_parts.append(
132133
await self._upload_chunck(
@@ -138,23 +139,31 @@ async def upload_file_async(
138139
)
139140
)
140141

141-
async with AsyncHttpClient(
142-
configuration=self.api_client.configuration,
143-
request_type="post",
144-
url=links.abort_upload,
145-
base_url=self.api_client.configuration.host,
146-
follow_redirects=True,
147-
auth=self._auth,
148-
timeout=timeout_seconds,
149-
) as session:
150-
_logger.info(
151-
"Completing upload (this might take a couple of minutes)..."
152-
)
153-
server_file: File = await self._complete_multipart_upload(
154-
session, links.complete_upload, client_file, uploaded_parts
155-
)
156-
_logger.info("File upload complete")
157-
return server_file
142+
abort_body = BodyAbortMultipartUploadV0FilesFileIdAbortPost(
143+
client_file=client_file
144+
)
145+
async with AsyncHttpClient(
146+
configuration=self.api_client.configuration,
147+
request_type="post",
148+
url=links.abort_upload,
149+
body=abort_body.to_dict(),
150+
base_url=self.api_client.configuration.host,
151+
follow_redirects=True,
152+
auth=self._auth,
153+
timeout=timeout_seconds,
154+
) as session:
155+
_logger.info(
156+
(
157+
"Completing upload of %s "
158+
"(this might take a couple of minutes)..."
159+
),
160+
file.name,
161+
)
162+
server_file: File = await self._complete_multipart_upload(
163+
session, links.complete_upload, client_file, uploaded_parts
164+
)
165+
_logger.info("File upload complete: %s", file.name)
166+
return server_file
158167

159168
async def _complete_multipart_upload(
160169
self,

‎clients/python/client/osparc/_http_client.py‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
from datetime import datetime
22
from email.utils import parsedate_to_datetime
3-
from typing import Any, Awaitable, Callable, Optional, Set
3+
from typing import Any, Awaitable, Callable, Dict, Optional, Set
44

55
import httpx
66
import tenacity
@@ -18,12 +18,17 @@ def __init__(
1818
configuration: Configuration,
1919
request_type: Optional[str] = None,
2020
url: Optional[str] = None,
21+
body: Optional[Dict] = None,
2122
**httpx_async_client_kwargs,
2223
):
2324
self.configuration = configuration
2425
self._client = httpx.AsyncClient(**httpx_async_client_kwargs)
2526
self._callback = getattr(self._client, request_type) if request_type else None
2627
self._url = url
28+
self._body = body
29+
if self._callback is not None:
30+
assert self._url is not None # nosec
31+
assert self._body is not None # nosec
2732

2833
async def __aenter__(self) -> "AsyncHttpClient":
2934
return self
@@ -41,9 +46,11 @@ async def __aexit__(self, exc_type, exc_value, traceback) -> None:
4146
retry=tenacity.retry_if_exception_type(httpx.RequestError),
4247
):
4348
with attempt:
44-
response = await self._callback(self._url)
49+
response = await self._callback(
50+
self._url, json={} if self._body is None else self._body
51+
)
4552
response.raise_for_status()
46-
except Exception as err:
53+
except httpx.HTTPError as err:
4754
await self._client.aclose()
4855
raise err from exc_value
4956
await self._client.aclose()

‎clients/python/requirements/unit-test.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
-r ../../../requirements.txt
2+
faker
23
pipdeptree
34
pipreqs
45
pytest
6+
pytest-asyncio
7+
respx

‎clients/python/test/test_osparc/pytest.ini‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[pytest]
2+
required_plugins = pytest-asyncio
3+
asyncio_mode = auto

‎clients/python/test/test_osparc/test_AsyncHttpClient.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
1+
import json
2+
13
import httpx
24
import osparc
35
import pytest
6+
import respx
7+
from faker import Faker
48
from osparc._http_client import AsyncHttpClient
59

610

@@ -27,6 +31,7 @@ def test_retry_strategy(cfg: osparc.Configuration, fake_retry_state):
2731
configuration=cfg,
2832
request_type="get",
2933
url="79ae41cc-0d89-4714-ac9d-c23ee1b110ce",
34+
body={},
3035
)
3136
assert (
3237
async_client._wait_callback(
@@ -50,3 +55,32 @@ def test_retry_strategy(cfg: osparc.Configuration, fake_retry_state):
5055
)
5156
== 15
5257
)
58+
59+
60+
async def test_aexit(
61+
cfg: osparc.Configuration, respx_mock: respx.MockRouter, faker: Faker
62+
):
63+
_call_url: str = "https://5b0c5cb6-5e88-479c-a54d-2d5fa39aa97b"
64+
_exit_url: str = "https://43c2fdfc-690e-4ba9-9ae7-55a911c159d0"
65+
_body = {"msg": faker.text()}
66+
67+
def _side_effect(request: httpx.Request):
68+
msg = json.loads(request.content.decode()).get("msg")
69+
assert msg
70+
assert _body["msg"] == msg
71+
return httpx.Response(status_code=200)
72+
73+
exit_mock = respx_mock.post(_exit_url).mock(side_effect=_side_effect)
74+
respx_mock.put(_call_url).mock(return_value=httpx.Response(500))
75+
76+
with pytest.raises(httpx.HTTPError):
77+
async with AsyncHttpClient(
78+
configuration=cfg,
79+
request_type="post",
80+
url=_exit_url,
81+
body=_body,
82+
) as session:
83+
response = await session.put(_call_url)
84+
response.raise_for_status()
85+
86+
assert exit_mock.called

0 commit comments

Comments
 (0)