allow to also upload files to programs

Author: bisgaard-itis

api/openapi.json

@@ -451,13 +451,13 @@
               "schema": {
                 "anyOf": [
                   {
-                    "$ref": "#/components/schemas/ClientFileInProgramJob"
+                    "$ref": "#/components/schemas/ClientFileToProgramJob"
                   },
                   {
                     "$ref": "#/components/schemas/ClientFile"
                   }
                 ],
-                "title": "Client File"
+                "title": "User File"
               }
             }
           }
@@ -6045,28 +6045,44 @@
     "schemas": {
       "Body_abort_multipart_upload_v0_files__file_id__abort_post": {
         "properties": {
-          "client_file": {
-            "$ref": "#/components/schemas/ClientFile"
+          "user_file": {
+            "anyOf": [
+              {
+                "$ref": "#/components/schemas/ClientFileToProgramJob"
+              },
+              {
+                "$ref": "#/components/schemas/ClientFile"
+              }
+            ],
+            "title": "User File"
           }
         },
         "type": "object",
         "required": [
-          "client_file"
+          "user_file"
         ],
         "title": "Body_abort_multipart_upload_v0_files__file_id__abort_post"
       },
       "Body_complete_multipart_upload_v0_files__file_id__complete_post": {
         "properties": {
-          "client_file": {
-            "$ref": "#/components/schemas/ClientFile"
+          "user_file": {
+            "anyOf": [
+              {
+                "$ref": "#/components/schemas/ClientFileToProgramJob"
+              },
+              {
+                "$ref": "#/components/schemas/ClientFile"
+              }
+            ],
+            "title": "User File"
           },
           "uploaded_parts": {
             "$ref": "#/components/schemas/FileUploadCompletionBody"
           }
         },
         "type": "object",
         "required": [
-          "client_file",
+          "user_file",
           "uploaded_parts"
         ],
         "title": "Body_complete_multipart_upload_v0_files__file_id__complete_post"
@@ -6114,7 +6130,7 @@
         "title": "ClientFile",
         "description": "Represents a file stored on the client side"
       },
-      "ClientFileInProgramJob": {
+      "ClientFileToProgramJob": {
         "properties": {
           "filename": {
             "type": "string",
@@ -6170,7 +6186,7 @@
           "job_id",
           "workspace_path"
         ],
-        "title": "ClientFileInProgramJob"
+        "title": "ClientFileToProgramJob"
       },
       "ClientFileUploadData": {
         "properties": {

clients/python/src/osparc/_api_files_api.py

@@ -25,7 +25,8 @@
     FileUploadCompletionBody,
     FileUploadData,
     UploadedPart,
-    ClientFileInProgramJob,
+    ClientFileToProgramJob,
+    UserFile,
 )
 from urllib.parse import urljoin
 import aiofiles
@@ -116,23 +117,31 @@ async def download_file_async(
             )  # aiofiles doesnt seem to have an async variant of this
         return dest_file
 
-    def upload_file(
+    def upload_file_to_program_job(
         self,
         file: Union[str, Path],
+        program_key: str,
+        program_version: str,
+        job_id: str,
+        workspace_path: str,
         timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS,
         max_concurrent_uploads: int = _MAX_CONCURRENT_UPLOADS,
         **kwargs,
     ):
         return asyncio.run(
-            self.upload_file_async(
+            self.upload_file_to_program_job_async(
                 file=file,
+                program_key=program_key,
+                program_version=program_version,
+                job_id=job_id,
+                workspace_path=workspace_path,
                 timeout_seconds=timeout_seconds,
                 max_concurrent_uploads=max_concurrent_uploads,
                 **kwargs,
             )
         )
 
-    async def upload_file_async(
+    async def upload_file_to_program_job_async(
         self,
         file: Union[str, Path],
         program_key: str,
@@ -155,13 +164,8 @@ async def upload_file_async(
                 # if a file has the same sha256 checksum
                 # and name they are considered equal
                 return file_result
-        # ClientFileInProgramJob(
-        #     filename=file.name,
-        #     filesize=file.stat().st_size,
-        #     sha256_checksum=checksum,
-        # )
-        client_file = ClientFile(
-            ClientFileInProgramJob(
+        user_file = UserFile(
+            ClientFileToProgramJob(
                 filename=file.name,
                 filesize=file.stat().st_size,
                 sha256_checksum=checksum,
@@ -171,23 +175,80 @@ async def upload_file_async(
                 workspace_path=workspace_path,
             )
         )
+        return await self._upload_user_file(
+            user_file, file, timeout_seconds, max_concurrent_uploads, **kwargs
+        )
+
+    def upload_file(
+        self,
+        file: Union[str, Path],
+        timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS,
+        max_concurrent_uploads: int = _MAX_CONCURRENT_UPLOADS,
+        **kwargs,
+    ) -> File:
+        return asyncio.run(
+            self.upload_file_async(
+                file=file,
+                timeout_seconds=timeout_seconds,
+                max_concurrent_uploads=max_concurrent_uploads,
+                **kwargs,
+            )
+        )
+
+    async def upload_file_async(
+        self,
+        file: Union[str, Path],
+        timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS,
+        max_concurrent_uploads: int = _MAX_CONCURRENT_UPLOADS,
+        **kwargs,
+    ) -> File:
+        if isinstance(file, str):
+            file = Path(file)
+        if not file.is_file():
+            raise RuntimeError(f"{file} is not a file")
+        checksum: str = await compute_sha256(file)
+        for file_result in self._search_files(
+            sha256_checksum=checksum, timeout_seconds=timeout_seconds
+        ):
+            if file_result.filename == file.name:
+                # if a file has the same sha256 checksum
+                # and name they are considered equal
+                return file_result
+        user_file = UserFile(
+            ClientFile(
+                filename=file.name,
+                filesize=file.stat().st_size,
+                sha256_checksum=checksum,
+            )
+        )
+        return await self._upload_user_file(
+            user_file, file, timeout_seconds, max_concurrent_uploads, **kwargs
+        )
+
+    async def _upload_user_file(
+        self,
+        user_file: UserFile,
+        file: Path,
+        timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS,
+        max_concurrent_uploads: int = _MAX_CONCURRENT_UPLOADS,
+        **kwargs,
+    ) -> File:
+        assert file.is_file()  # nosec
         client_upload_schema: ClientFileUploadData = super().get_upload_links(
-            client_file=client_file, _request_timeout=timeout_seconds, **kwargs
+            user_file=user_file, _request_timeout=timeout_seconds, **kwargs
         )
         chunk_size: int = client_upload_schema.upload_schema.chunk_size
         links: FileUploadData = client_upload_schema.upload_schema.links
         url_iter: Iterator[Tuple[int, str]] = enumerate(
             iter(client_upload_schema.upload_schema.urls), start=1
         )
         n_urls: int = len(client_upload_schema.upload_schema.urls)
-        if n_urls < math.ceil(file.stat().st_size / chunk_size):
+        if n_urls < math.ceil(user_file.actual_instance.filesize / chunk_size):
             raise RuntimeError(
                 "Did not receive sufficient number of upload URLs from the server."
             )
 
-        abort_body = BodyAbortMultipartUploadV0FilesFileIdAbortPost(
-            client_file=client_file
-        )
+        abort_body = BodyAbortMultipartUploadV0FilesFileIdAbortPost(user_file=user_file)
         upload_tasks: Set[asyncio.Task] = set()
         uploaded_parts: List[UploadedPart] = []
 
@@ -240,7 +301,7 @@ async def upload_file_async(
             server_file: File = await self._complete_multipart_upload(
                 api_server_session,
                 links.complete_upload,  # type: ignore
-                client_file,
+                user_file,
                 uploaded_parts,
             )
             _logger.debug("File upload complete: %s", file.name)
@@ -250,11 +311,11 @@ async def _complete_multipart_upload(
         self,
         http_client: AsyncHttpClient,
         complete_link: str,
-        client_file: ClientFile,
+        user_file: UserFile,
         uploaded_parts: List[UploadedPart],
     ) -> File:
         complete_payload = BodyCompleteMultipartUploadV0FilesFileIdCompletePost(
-            client_file=client_file,
+            user_file=user_file,
             uploaded_parts=FileUploadCompletionBody(parts=uploaded_parts),
         )
         response: Response = await http_client.post(

clients/python/src/osparc/models.py

@@ -13,9 +13,10 @@
     BodyCompleteMultipartUploadV0FilesFileIdCompletePost as BodyCompleteMultipartUploadV0FilesFileIdCompletePost,
 )
 from osparc_client.models.client_file import ClientFile as ClientFile
-from osparc_client.models.client_file_in_program_job import (
-    ClientFileInProgramJob as ClientFileInProgramJob,
+from osparc_client.models.client_file_to_program_job import (
+    ClientFileToProgramJob as ClientFileToProgramJob,
 )
+from osparc_client.models.user_file import UserFile as UserFile
 from osparc_client.models.client_file_upload_data import (
     ClientFileUploadData as ClientFileUploadData,
 )

programs_example.py

@@ -48,7 +48,7 @@ async def run(program_key: str, version: str) -> osparc.JobStatus:
             # Upload the input file
             tmp_file = Path(tmp_dir) / "tmpfile"
             tmp_file.write_text("Hi oSPARC 👋")
-            await files_api.upload_file_async(
+            await files_api.upload_file_to_program_job_async(
                 file=tmp_file,
                 program_key=program.id,
                 program_version=program.version,