refactored

Author: sanderegg

services/datcore-adapter/src/simcore_service_datcore_adapter/cli.py

@@ -1,9 +1,9 @@
 import logging
 
 import typer
-from settings_library.utils_cli import create_settings_command
+from settings_library.utils_cli import create_settings_command, create_version_callback
 
-from ._meta import PROJECT_NAME
+from ._meta import PROJECT_NAME, __version__
 from .core.settings import ApplicationSettings
 
 log = logging.getLogger(__name__)
@@ -12,13 +12,14 @@
 main = typer.Typer(name=PROJECT_NAME)
 
 main.command()(create_settings_command(settings_cls=ApplicationSettings, logger=log))
+main.callback()(create_version_callback(__version__))
 
 
 @main.command()
-def run():
+def run() -> None:
     """Runs application"""
     typer.secho("Sorry, this entrypoint is intentionally disabled. Use instead")
     typer.secho(
-        "$ uvicorn simcore_service_datcore_adapter.main:the_app",
+        f"$ uvicorn {PROJECT_NAME}.main:the_app",
         fg=typer.colors.BLUE,
     )

services/datcore-adapter/src/simcore_service_datcore_adapter/main.py

@@ -1,7 +1,22 @@
-"""Main application to be deployed in for example uvicorn
-"""
+"""Main application to be deployed in for example uvicorn"""
+
+import logging
+
 from fastapi import FastAPI
+from servicelib.logging_utils import config_all_loggers
 from simcore_service_datcore_adapter.core.application import create_app
+from simcore_service_datcore_adapter.core.settings import ApplicationSettings
+
+_the_settings = ApplicationSettings.create_from_envs()
+
+# SEE https://github.com/ITISFoundation/osparc-simcore/issues/3148
+logging.basicConfig(level=_the_settings.log_level)  # NOSONAR
+logging.root.setLevel(_the_settings.log_level)
+config_all_loggers(
+    log_format_local_dev_enabled=_the_settings.DATCORE_ADAPTER_LOG_FORMAT_LOCAL_DEV_ENABLED,
+    logger_filter_mapping=_the_settings.DATCORE_ADAPTER_LOG_FILTER_MAPPING,
+    tracing_settings=_the_settings.DATCORE_ADAPTER_TRACING,
+)
 
 # SINGLETON FastAPI app
-the_app: FastAPI = create_app()
+the_app: FastAPI = create_app(_the_settings)

services/datcore-adapter/src/simcore_service_datcore_adapter/modules/pennsieve.py

@@ -9,6 +9,11 @@
 import boto3
 from aiocache import SimpleMemoryCache  # type: ignore[import-untyped]
 from fastapi.applications import FastAPI
+from models_library.api_schemas_datcore_adapter.datasets import (
+    DatasetMetaData,
+    DataType,
+    FileMetaData,
+)
 from servicelib.logging_utils import log_context
 from servicelib.utils import logged_gather
 from starlette import status
@@ -19,8 +24,8 @@
 from tenacity.stop import stop_after_attempt
 
 from ..core.settings import PennsieveSettings
-from ..models.domains.user import Profile
-from ..models.schemas.datasets import DatasetMetaData, FileMetaData
+from ..models.files import DatCorePackageMetaData
+from ..models.user import Profile
 from ..utils.client_base import BaseServiceClientApi, setup_client_instance
 
 logger = logging.getLogger(__name__)
@@ -29,6 +34,36 @@
 _GATHER_MAX_CONCURRENCY = 10
 
 
+def _to_file_meta_data(
+    package: dict[str, Any], files: list[DatCorePackageMetaData], base_path: Path
+) -> FileMetaData:
+    """creates a FileMetaData from a pennsieve data structure."""
+    pck_name: str = package["content"]["name"]
+    if "extension" in package and not pck_name.endswith(package["extension"]):
+        pck_name += ".".join((pck_name, package["extension"]))
+
+    file_size = 0
+    if package["content"]["packageType"] != "Collection" and files:
+        file_size = files[0].size
+
+    return FileMetaData(
+        dataset_id=package["content"]["datasetNodeId"],
+        package_id=package["content"]["nodeId"],
+        id=f"{package['content']['id']}",
+        name=pck_name,
+        path=base_path / pck_name,
+        type=package["content"]["packageType"],
+        size=file_size,
+        created_at=package["content"]["createdAt"],
+        last_modified_at=package["content"]["updatedAt"],
+        data_type=(
+            DataType.FOLDER
+            if package["content"]["packageType"] == "Collection"
+            else DataType.FILE
+        ),
+    )
+
+
 def _compute_file_path(
     all_packages: dict[str, dict[str, Any]], pck: dict[str, Any]
 ) -> Path:
@@ -215,27 +250,66 @@ async def _get_package(
         )
 
     async def get_package_files(
-        self, api_key: str, api_secret: str, package_id: str, limit: int, offset: int
-    ) -> list[dict[str, Any]]:
-        return cast(
-            list[dict[str, Any]],
-            await self._request(
-                api_key,
-                api_secret,
-                "GET",
-                f"/packages/{package_id}/files",
-                params={"limit": limit, "offset": offset},
-            ),
+        self,
+        *,
+        api_key: str,
+        api_secret: str,
+        package_id: str,
+        limit: int,
+        offset: int,
+        fill_path: bool,
+    ) -> list[DatCorePackageMetaData]:
+        raw_data = await self._request(
+            api_key,
+            api_secret,
+            "GET",
+            f"/packages/{package_id}/files",
+            params={"limit": limit, "offset": offset},
         )
+        path = display_path = Path()
+        if fill_path:
+            package_info = await self._get_package(api_key, api_secret, package_id)
+            dataset_id = package_info["content"]["datasetId"]
+            dataset = await self._get_dataset(api_key, api_secret, dataset_id)
+
+            path = (
+                Path(dataset_id)
+                / Path(
+                    "/".join(
+                        ancestor["content"]["id"]
+                        for ancestor in package_info.get("ancestors", [])
+                    )
+                )
+                / Path(package_info["content"]["name"])
+            )
+            display_path = (
+                Path(dataset["content"]["name"])
+                / Path(
+                    "/".join(
+                        ancestor["content"]["name"]
+                        for ancestor in package_info.get("ancestors", [])
+                    )
+                )
+                / Path(package_info["content"]["name"])
+            )
+
+        return [
+            DatCorePackageMetaData(**_["content"], path=path, display_path=display_path)
+            for _ in raw_data
+        ]
 
     async def _get_pck_id_files(
         self, api_key: str, api_secret: str, pck_id: str, pck: dict[str, Any]
-    ) -> tuple[str, list[dict[str, Any]]]:
-
+    ) -> tuple[str, list[DatCorePackageMetaData]]:
         return (
             pck_id,
             await self.get_package_files(
-                api_key, api_secret, pck["content"]["nodeId"], limit=1, offset=0
+                api_key=api_key,
+                api_secret=api_secret,
+                package_id=pck["content"]["nodeId"],
+                limit=1,
+                offset=0,
+                fill_path=False,
             ),
         )
 
@@ -293,7 +367,7 @@ async def list_packages_in_dataset(
             for pck in islice(dataset_pck["children"], offset, offset + limit)
             if pck["content"]["packageType"] != "Collection"
         ]
-        package_files = dict(
+        package_files: dict[str, list[DatCorePackageMetaData]] = dict(
             await logged_gather(
                 *package_files_tasks,
                 log=logger,
@@ -302,7 +376,7 @@ async def list_packages_in_dataset(
         )
         return (
             [
-                FileMetaData.from_pennsieve_package(
+                _to_file_meta_data(
                     pck,
                     (
                         package_files[pck["content"]["id"]]
@@ -353,7 +427,7 @@ async def list_packages_in_collection(
 
         return (
             [
-                FileMetaData.from_pennsieve_package(
+                _to_file_meta_data(
                     pck,
                     (
                         package_files[pck["content"]["id"]]
@@ -433,7 +507,7 @@ async def list_all_dataset_files(
                 file_path = base_path / _compute_file_path(all_packages, package)
 
                 file_meta_data.append(
-                    FileMetaData.from_pennsieve_package(
+                    _to_file_meta_data(
                         package, package_files[package_id], file_path.parent
                     )
                 )
@@ -445,11 +519,16 @@ async def get_presigned_download_link(
     ) -> URL:
         """returns the presigned download link of the first file in the package"""
         files = await self.get_package_files(
-            api_key, api_secret, package_id, limit=1, offset=0
+            api_key=api_key,
+            api_secret=api_secret,
+            package_id=package_id,
+            limit=1,
+            offset=0,
+            fill_path=False,
         )
         # NOTE: this was done like this in the original dsm. we might encounter a problem when there are more than one files
         assert len(files) == 1  # nosec
-        file_id = files[0]["content"]["id"]
+        file_id = files[0].id
         file_link = cast(
             dict[str, Any],
             await self._request(

services/datcore-adapter/tests/unit/conftest.py

@@ -15,6 +15,7 @@
 import simcore_service_datcore_adapter
 from asgi_lifespan import LifespanManager
 from fastapi.applications import FastAPI
+from models_library.utils.fastapi_encoders import jsonable_encoder
 from pytest_mock import MockFixture
 from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict
 from simcore_service_datcore_adapter.modules.pennsieve import (
@@ -24,6 +25,7 @@
 from starlette.testclient import TestClient
 
 pytest_plugins = [
+    "pytest_simcore.cli_runner",
     "pytest_simcore.environment_configs",
     "pytest_simcore.repository_paths",
     "pytest_simcore.pytest_global_environs",
@@ -317,15 +319,40 @@ async def pennsieve_subsystem_mock(
 
             # get collection packages
             mock.get(
-                f"https://api.pennsieve.io/packages/{pennsieve_collection_id}"
+                rf"https://api.pennsieve.io/packages/{pennsieve_collection_id}"
             ).respond(
                 status.HTTP_200_OK,
                 json={
                     "content": {"name": "this package name is also awesome"},
                     "children": pennsieve_mock_dataset_packages["packages"],
                     "ancestors": [
-                        {"content": {"name": "Bigger guy"}},
-                        {"content": {"name": "Big guy"}},
+                        {
+                            "content": {
+                                "name": "Bigger guy",
+                            }
+                        },
+                        {
+                            "content": {
+                                "name": "Big guy",
+                            }
+                        },
+                    ],
+                },
+            )
+            # get package ancestry
+            mock.get(
+                url__regex=rf"https://api.pennsieve.io/packages/{pennsieve_file_id}\?includeAncestors=(?P<include>.+)$"
+            ).respond(
+                status.HTTP_200_OK,
+                json={
+                    "content": {
+                        "datasetId": pennsieve_dataset_id,
+                        "name": pennsieve_file_id,
+                    },
+                    "ancestors": [
+                        {"content": {"id": faker.pystr(), "name": faker.name()}},
+                        {"content": {"id": faker.pystr(), "name": faker.name()}},
+                        {"content": {"id": faker.pystr(), "name": faker.name()}},
                     ],
                 },
             )
@@ -334,7 +361,22 @@ async def pennsieve_subsystem_mock(
                 url__regex=r"https://api.pennsieve.io/packages/.+/files\?limit=1&offset=0$"
             ).respond(
                 status.HTTP_200_OK,
-                json=[{"content": {"size": 12345, "id": "fake_file_id"}}],
+                json=[
+                    jsonable_encoder(
+                        {
+                            "content": {
+                                "size": 12345,
+                                "id": faker.pyint(),
+                                "packageId": "N:package:475beff2-03c8-4dca-a221-d1d02e17f064",
+                                "name": faker.file_name(),
+                                "filename": faker.file_name(),
+                                "s3bucket": faker.pystr(),
+                                "createdAt": faker.date_time(),
+                                "updatedAt": faker.date_time(),
+                            }
+                        }
+                    )
+                ],
             )
 
             # download file

services/datcore-adapter/tests/unit/test_cli.py

@@ -0,0 +1,35 @@
+# pylint:disable=unused-variable
+# pylint:disable=unused-argument
+# pylint:disable=redefined-outer-name
+
+import os
+
+from pytest_simcore.helpers.typing_env import EnvVarsDict
+from simcore_service_datcore_adapter._meta import API_VERSION
+from simcore_service_datcore_adapter.cli import main
+from simcore_service_datcore_adapter.core.settings import ApplicationSettings
+from typer.testing import CliRunner
+
+
+def test_cli_help_and_version(cli_runner: CliRunner):
+    result = cli_runner.invoke(main, "--help")
+    assert result.exit_code == os.EX_OK, result.output
+
+    result = cli_runner.invoke(main, "--version")
+    assert result.exit_code == os.EX_OK, result.output
+    assert result.stdout.strip() == API_VERSION
+
+
+def test_settings(cli_runner: CliRunner, app_environment: EnvVarsDict):
+    result = cli_runner.invoke(main, ["settings", "--show-secrets", "--as-json"])
+    assert result.exit_code == os.EX_OK
+
+    print(result.output)
+    settings = ApplicationSettings(result.output)
+    assert settings.model_dump() == ApplicationSettings.create_from_envs().model_dump()
+
+
+def test_run(cli_runner: CliRunner):
+    result = cli_runner.invoke(main, ["run"])
+    assert result.exit_code == 0
+    assert "disabled" in result.stdout