✨Allow computational sidecar to understand S3 links for download/upload (ITISFoundation#2997)

Author: sanderegg

packages/dask-task-models-library/src/dask_task_models_library/container_tasks/events.py

@@ -8,7 +8,7 @@
 
 class BaseTaskEvent(BaseModel, ABC):
     job_id: str
-    msg: Optional[str]
+    msg: Optional[str] = None
 
     @staticmethod
     @abstractmethod

packages/pytest-simcore/src/pytest_simcore/minio_service.py

@@ -6,13 +6,15 @@
 from typing import Dict, Iterator
 
 import pytest
-import tenacity
 from _pytest.monkeypatch import MonkeyPatch
 from minio import Minio
 from minio.datatypes import Object
 from minio.deleteobjects import DeleteError, DeleteObject
 from pydantic import parse_obj_as
 from tenacity import Retrying
+from tenacity.before_sleep import before_sleep_log
+from tenacity.stop import stop_after_attempt
+from tenacity.wait import wait_fixed
 
 from .helpers.utils_docker import get_localhost_ip, get_service_published_port
 
@@ -71,9 +73,9 @@ def minio_service(minio_config: Dict[str, str]) -> Iterator[Minio]:
     client = Minio(**minio_config["client"])
 
     for attempt in Retrying(
-        wait=tenacity.wait_fixed(5),
-        stop=tenacity.stop_after_attempt(60),
-        before_sleep=tenacity.before_sleep_log(log, logging.WARNING),
+        wait=wait_fixed(5),
+        stop=stop_after_attempt(60),
+        before_sleep=before_sleep_log(log, logging.WARNING),
         reraise=True,
     ):
         with attempt:

packages/settings-library/src/settings_library/s3.py

@@ -1,9 +1,12 @@
+from typing import Optional
+
 from .base import BaseCustomSettings
 
 
 class S3Settings(BaseCustomSettings):
     S3_ENDPOINT: str = "minio:9000"
     S3_ACCESS_KEY: str = "12345678"
     S3_SECRET_KEY: str = "12345678"
+    S3_ACCESS_TOKEN: Optional[str] = None
     S3_BUCKET_NAME: str = "simcore"
     S3_SECURE: bool = False

packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py

@@ -33,6 +33,7 @@ async def _push_file(
     await filemanager.upload_file(
         user_id=user_id,
         store_id=store_id,
+        store_name=None,
         s3_object=s3_object,
         local_file_path=file_path,
     )
@@ -79,6 +80,7 @@ async def _pull_file(
     downloaded_file = await filemanager.download_file_from_s3(
         user_id=user_id,
         store_id="0",
+        store_name=None,
         s3_object=s3_object,
         local_folder=destination_path.parent,
     )

packages/simcore-sdk/src/simcore_sdk/node_ports/_item.py

@@ -177,6 +177,7 @@ async def set(self, value: ItemConcreteValue):
             )
             store_id, _ = await filemanager.upload_file(
                 user_id=self._user_id,
+                store_id=None,
                 store_name=config.STORE,
                 s3_object=s3_object,
                 local_file_path=file_path,
@@ -216,6 +217,7 @@ async def __get_value_from_store(self, user_id: int, value: StoreLink) -> Path:
         downloaded_file = await filemanager.download_file_from_s3(
             user_id=user_id,
             store_id=store_id,
+            store_name=None,
             s3_object=s3_path,
             local_folder=local_path,
         )

packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py

@@ -137,8 +137,8 @@ async def file_sender(file_name: Path):
 async def get_download_link_from_s3(
     *,
     user_id: int,
-    store_name: str = None,
-    store_id: str = None,
+    store_name: Optional[str],
+    store_id: Optional[str],
     s3_object: str,
     client_session: Optional[ClientSession] = None,
 ) -> Optional[URL]:
@@ -157,8 +157,8 @@ async def get_download_link_from_s3(
 async def get_upload_link_from_s3(
     *,
     user_id: int,
-    store_name: str = None,
-    store_id: str = None,
+    store_name: Optional[str],
+    store_id: Optional[str],
     s3_object: str,
     client_session: Optional[ClientSession] = None,
 ) -> Tuple[str, URL]:
@@ -180,8 +180,8 @@ async def get_upload_link_from_s3(
 async def download_file_from_s3(
     *,
     user_id: int,
-    store_name: str = None,
-    store_id: str = None,
+    store_name: Optional[str],
+    store_id: Optional[str],
     s3_object: str,
     local_folder: Path,
     client_session: Optional[ClientSession] = None,
@@ -246,8 +246,8 @@ async def download_file_from_link(
 async def upload_file(
     *,
     user_id: int,
-    store_id: Optional[str] = None,
-    store_name: Optional[str] = None,
+    store_id: Optional[str],
+    store_name: Optional[str],
     s3_object: str,
     local_file_path: Path,
     client_session: Optional[ClientSession] = None,

packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py

@@ -70,6 +70,7 @@ async def get_download_link_from_storage(
     link = await filemanager.get_download_link_from_s3(
         user_id=user_id,
         store_id=f"{value.store}",
+        store_name=None,
         s3_object=value.path,
     )
     return parse_obj_as(AnyUrl, f"{link}") if link else None
@@ -85,6 +86,7 @@ async def get_upload_link_from_storage(
     s3_object = data_items_utils.encode_file_id(Path(file_name), project_id, node_id)
     _, link = await filemanager.get_upload_link_from_s3(
         user_id=user_id,
+        store_id=None,
         store_name=config.STORE,
         s3_object=s3_object,
     )
@@ -125,6 +127,7 @@ async def pull_file_from_store(
     downloaded_file = await filemanager.download_file_from_s3(
         user_id=user_id,
         store_id=f"{value.store}",
+        store_name=None,
         s3_object=value.path,
         local_folder=local_path,
     )
@@ -150,6 +153,7 @@ async def push_file_to_store(
     s3_object = data_items_utils.encode_file_id(file, project_id, node_id)
     store_id, e_tag = await filemanager.upload_file(
         user_id=user_id,
+        store_id=None,
         store_name=config.STORE,
         s3_object=s3_object,
         local_file_path=file,
@@ -198,6 +202,7 @@ async def get_file_link_from_url(
     node_id: str,
 ) -> FileLink:
     log.debug("url %s will now be converted to a file link", new_value)
+    assert new_value.path  # nosec
     s3_object = data_items_utils.encode_file_id(
         Path(new_value.path), project_id, node_id
     )

packages/simcore-sdk/tests/integration/test_node_ports_common_filemanager.py

@@ -37,6 +37,7 @@ async def test_valid_upload_download(
     store_id, e_tag = await filemanager.upload_file(
         user_id=user_id,
         store_id=s3_simcore_location,
+        store_name=None,
         s3_object=file_id,
         local_file_path=file_path,
     )
@@ -53,6 +54,7 @@ async def test_valid_upload_download(
     download_file_path = await filemanager.download_file_from_s3(
         user_id=user_id,
         store_id=s3_simcore_location,
+        store_name=None,
         s3_object=file_id,
         local_folder=download_folder,
     )
@@ -79,6 +81,7 @@ async def test_invalid_file_path(
         await filemanager.upload_file(
             user_id=user_id,
             store_id=store,
+            store_name=None,
             s3_object=file_id,
             local_file_path=Path(tmpdir) / "some other file.txt",
         )
@@ -88,6 +91,7 @@ async def test_invalid_file_path(
         await filemanager.download_file_from_s3(
             user_id=user_id,
             store_id=store,
+            store_name=None,
             s3_object=file_id,
             local_folder=download_folder,
         )
@@ -108,27 +112,37 @@ async def test_errors_upon_invalid_file_identifiers(
     store = s3_simcore_location
     with pytest.raises(exceptions.StorageInvalidCall):
         await filemanager.upload_file(
-            user_id=user_id, store_id=store, s3_object="", local_file_path=file_path
+            user_id=user_id,
+            store_id=store,
+            store_name=None,
+            s3_object="",
+            local_file_path=file_path,
         )
 
     with pytest.raises(exceptions.StorageInvalidCall):
         await filemanager.upload_file(
             user_id=user_id,
             store_id=store,
+            store_name=None,
             s3_object="file_id",
             local_file_path=file_path,
         )
 
     download_folder = Path(tmpdir) / "downloads"
     with pytest.raises(exceptions.StorageInvalidCall):
         await filemanager.download_file_from_s3(
-            user_id=user_id, store_id=store, s3_object="", local_folder=download_folder
+            user_id=user_id,
+            store_id=store,
+            store_name=None,
+            s3_object="",
+            local_folder=download_folder,
         )
 
     with pytest.raises(exceptions.StorageInvalidCall):
         await filemanager.download_file_from_s3(
             user_id=user_id,
             store_id=store,
+            store_name=None,
             s3_object=np_helpers.file_uuid(
                 Path("invisible.txt"), project_id, f"{uuid4()}"
             ),
@@ -153,6 +167,7 @@ async def test_invalid_store(
     with pytest.raises(exceptions.S3InvalidStore):
         await filemanager.upload_file(
             user_id=user_id,
+            store_id=None,
             store_name=store,
             s3_object=file_id,
             local_file_path=file_path,
@@ -162,6 +177,7 @@ async def test_invalid_store(
     with pytest.raises(exceptions.S3InvalidStore):
         await filemanager.download_file_from_s3(
             user_id=user_id,
+            store_id=None,
             store_name=store,
             s3_object=file_id,
             local_folder=download_folder,
@@ -194,6 +210,7 @@ async def test_valid_metadata(
     store_id, e_tag = await filemanager.upload_file(
         user_id=user_id,
         store_id=s3_simcore_location,
+        store_name=None,
         s3_object=file_id,
         local_file_path=file_path,
     )
@@ -252,6 +269,7 @@ async def test_delete_File(
     store_id, e_tag = await filemanager.upload_file(
         user_id=user_id,
         store_id=s3_simcore_location,
+        store_name=None,
         s3_object=file_id,
         local_file_path=file_path,
     )

packages/simcore-sdk/tests/unit/test_node_data_data_manager.py

@@ -5,14 +5,14 @@
 from filecmp import cmpfiles
 from pathlib import Path
 from shutil import copy, make_archive, unpack_archive
-from typing import Callable, List
+from typing import Callable, Iterator, List
 
 import pytest
 from simcore_sdk.node_data import data_manager
 
 
 @pytest.fixture
-def create_files() -> Callable:
+def create_files() -> Iterator[Callable[..., List[Path]]]:
     created_files = []
 
     def _create_files(number: int, folder: Path) -> List[Path]:
@@ -76,6 +76,7 @@ async def test_push_folder(
         local_file_path=(test_compression_folder / "{}.zip".format(test_folder.stem)),
         s3_object=f"{project_id}/{node_uuid}/{test_folder.stem}.zip",
         store_id="0",
+        store_name=None,
         user_id=user_id,
     )
 
@@ -121,6 +122,7 @@ async def test_push_file(
         local_file_path=file_path,
         s3_object=f"{project_id}/{node_uuid}/{file_path.name}",
         store_id="0",
+        store_name=None,
         user_id=user_id,
     )
     mock_filemanager.reset_mock()
@@ -152,7 +154,7 @@ async def test_pull_folder(
     create_files(files_number, test_control_folder)
     compressed_file_name = test_compression_folder / test_folder.stem
     archive_file = make_archive(
-        compressed_file_name, "zip", root_dir=test_control_folder
+        f"{compressed_file_name}", "zip", root_dir=test_control_folder
     )
     assert Path(archive_file).exists()
     # create mock downloaded folder
@@ -179,6 +181,7 @@ async def test_pull_folder(
         local_folder=test_compression_folder,
         s3_object=f"{project_id}/{node_uuid}/{test_folder.stem}.zip",
         store_id="0",
+        store_name=None,
         user_id=user_id,
     )
 
@@ -223,5 +226,6 @@ async def test_pull_file(
         local_folder=file_path.parent,
         s3_object=f"{project_id}/{node_uuid}/{file_path.name}",
         store_id="0",
+        store_name=None,
         user_id=user_id,
     )

services/dask-sidecar/requirements/_base.in

@@ -24,7 +24,8 @@ requests
 dask[complete]
 dask-gateway
 pydantic[email,dotenv]
-
+# s3 access
+s3fs
 # compression
 blosc
 lz4