Add custom timeout in multipart upload (#132)

* update workflow before publishing python package

* fix dependency issue and bump version

* point to website in project description

* fix broken dependency

* improve doc

* add github token to download artifacts

* ensure only read-access @wvangeit

* yet another attempt at downloading artifacts

* make sure to use repo that ran the trigger wf

* another attempt at fixing

* change owner

* allow publishing to testpypi also when pr

* minor change

* revert minor (but breaking) change

* minor fix

* add debug messages

* another debug message

* hopefully the final version

* final fix

* minor fix

* move master and tag to individual jobs

* add debug messages

* dev->post

* add python script for determining semantic version

* minor changes

* minor changes

* improve error handling and add version file to artifacts

* check if release

* minor fix

* ensure to enter venv

* also when tagging

* source venv in publishin workflow

* ensure only master

* add script for testing 'pure' semver

* adapt workflows to new python script

* minor change

* attempt to evaluate expressions correctly

* several fixes to fix tests

* ensure repo is checked out in publish workflow

* several small fixes

* cleanup

* debug

* minor cleanup

* mionr changes

* add debug message

* minor change

* minor change

* yet another try

* minor change

* minor change

* minor change

* mionr change

* minor changes

* correct workflow run id

* cosmetic change

* avoid using gh

* change to a single job for publishing

* minor cleanup

* swap loops in clean up jobs

* correction

* add timeouts

Author: bisgaard-itis

clients/python/client/osparc/_files_api.py

@@ -21,6 +21,7 @@
 from . import ApiClient, File
 from ._http_client import AsyncHttpClient
 from ._utils import (
+    DEFAULT_TIMEOUT_SECONDS,
     PaginationGenerator,
     compute_sha256,
     dev_features_enabled,
@@ -70,16 +71,24 @@ def download_file(
                 downloaded_file = dest_file
             return str(downloaded_file.resolve())
 
-        def upload_file(self, file: Union[str, Path]):
-            return asyncio.run(self.upload_file_async(file=file))
+        def upload_file(
+            self, file: Union[str, Path], timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS
+        ):
+            return asyncio.run(
+                self.upload_file_async(file=file, timeout_seconds=timeout_seconds)
+            )
 
-        async def upload_file_async(self, file: Union[str, Path]) -> File:
+        async def upload_file_async(
+            self, file: Union[str, Path], timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS
+        ) -> File:
             if isinstance(file, str):
                 file = Path(file)
             if not file.is_file():
                 raise RuntimeError(f"{file} is not a file")
             checksum: str = compute_sha256(file)
-            for file_result in self._search_files(sha256_checksum=checksum):
+            for file_result in self._search_files(
+                sha256_checksum=checksum, timeout_seconds=timeout_seconds
+            ):
                 if file_result.filename == file.name:
                     # if a file has the same sha256 checksum
                     # and name they are considered equal
@@ -90,7 +99,7 @@ async def upload_file_async(self, file: Union[str, Path]) -> File:
                 sha256_checksum=checksum,
             )
             client_upload_schema: ClientFileUploadData = self._super.get_upload_links(
-                client_file=client_file
+                client_file=client_file, _request_timeout=timeout_seconds
             )
             chunk_size: int = client_upload_schema.upload_schema.chunk_size
             links: FileUploadData = client_upload_schema.upload_schema.links
@@ -105,7 +114,7 @@ async def upload_file_async(self, file: Union[str, Path]) -> File:
 
             uploaded_parts: list[UploadedPart] = []
             print("- uploading chunks...")
-            async with AsyncHttpClient() as session:
+            async with AsyncHttpClient(timeout=timeout_seconds) as session:
                 async for chunck, size in tqdm(
                     file_chunk_generator(file, chunk_size), total=n_urls
                 ):
@@ -126,15 +135,16 @@ async def upload_file_async(self, file: Union[str, Path]) -> File:
                     base_url=self.api_client.configuration.host,
                     follow_redirects=True,
                     auth=self._auth,
+                    timeout=timeout_seconds,
                 ) as session:
                     print(
                         "- completing upload (this might take a couple of minutes)..."
                     )
-                    file: File = await self._complete_multipart_upload(
+                    server_file: File = await self._complete_multipart_upload(
                         session, links.complete_upload, client_file, uploaded_parts
                     )
                     print("- file upload complete")
-                    return file
+                    return server_file
 
         async def _complete_multipart_upload(
             self,
@@ -175,11 +185,16 @@ async def _upload_chunck(
             return UploadedPart(number=index, e_tag=etag)
 
         def _search_files(
-            self, file_id: Optional[str] = None, sha256_checksum: Optional[str] = None
+            self,
+            file_id: Optional[str] = None,
+            sha256_checksum: Optional[str] = None,
+            timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS,
         ) -> PaginationGenerator:
             def pagination_method():
                 return super(FilesApi, self).search_files_page(
-                    file_id=file_id, sha256_checksum=sha256_checksum
+                    file_id=file_id,
+                    sha256_checksum=sha256_checksum,
+                    _request_timeout=timeout_seconds,
                 )
 
             return PaginationGenerator(

clients/python/client/osparc/_utils.py

@@ -23,6 +23,8 @@
 _MB = _KB * 1024  # in bytes
 _GB = _MB * 1024  # in bytes
 
+DEFAULT_TIMEOUT_SECONDS: int = 30 * 60
+
 Page = Union[PageJob, PageFile, PageSolver, PageStudy]
 T = TypeVar("T", Job, File, Solver, Study)