⬆️ Maintenance: Datcore-adapter grand update (#5600)

Author: sanderegg

services/datcore-adapter/requirements/_base.txt

@@ -1,44 +1,46 @@
-anyio==3.6.2
-    # via httpcore
+anyio==4.3.0
+    # via httpx
 asgi-lifespan==2.1.0
-certifi==2023.7.22
+certifi==2024.2.2
     # via
     #   httpcore
     #   httpx
     #   requests
-charset-normalizer==3.1.0
+charset-normalizer==3.3.2
     # via requests
-coverage==7.3.2
+coverage==7.4.4
     # via pytest-cov
-exceptiongroup==1.1.3
-    # via pytest
+exceptiongroup==1.2.0
+    # via
+    #   anyio
+    #   pytest
 execnet==2.0.2
     # via pytest-xdist
-faker==19.13.0
-h11==0.12.0
+faker==24.4.0
+h11==0.14.0
     # via httpcore
-httpcore==0.15.0
+httpcore==1.0.5
     # via httpx
-httpx==0.24.0
+httpx==0.27.0
     # via respx
 icdiff==2.0.7
     # via pytest-icdiff
-idna==3.4
+idna==3.6
     # via
     #   anyio
     #   httpx
     #   requests
 iniconfig==2.0.0
     # via pytest
-packaging==23.2
+packaging==24.0
     # via
     #   pytest
     #   pytest-sugar
-pluggy==1.3.0
+pluggy==1.4.0
     # via pytest
 pprintpp==0.4.0
     # via pytest-icdiff
-pytest==7.4.3
+pytest==8.1.1
     # via
     #   pytest-asyncio
     #   pytest-cov
@@ -48,30 +50,31 @@ pytest==7.4.3
     #   pytest-sugar
     #   pytest-xdist
 pytest-asyncio==0.21.1
-pytest-cov==4.1.0
-pytest-icdiff==0.8
+pytest-cov==5.0.0
+pytest-icdiff==0.9
 pytest-instafail==0.5.0
-pytest-mock==3.12.0
-pytest-runner==6.0.0
-pytest-sugar==0.9.7
-pytest-xdist==3.3.1
-python-dateutil==2.8.2
+pytest-mock==3.14.0
+pytest-runner==6.0.1
+pytest-sugar==1.0.0
+pytest-xdist==3.5.0
+python-dateutil==2.9.0.post0
     # via faker
 requests==2.31.0
-respx==0.20.2
+respx==0.21.1
 six==1.16.0
     # via python-dateutil
-sniffio==1.3.0
+sniffio==1.3.1
     # via
     #   anyio
     #   asgi-lifespan
-    #   httpcore
     #   httpx
-termcolor==2.3.0
+termcolor==2.4.0
     # via pytest-sugar
 tomli==2.0.1
     # via
     #   coverage
     #   pytest
-urllib3==1.26.12
+typing-extensions==4.10.0
+    # via anyio
+urllib3==2.2.1
     # via requests

services/datcore-adapter/requirements/_test.txt

@@ -1,74 +1,56 @@
-aiohttp==3.8.5
-    # via black
-aiosignal==1.3.1
-    # via aiohttp
-astroid==3.0.2
+astroid==3.1.0
     # via pylint
-async-timeout==4.0.2
-    # via aiohttp
-attrs==21.4.0
-    # via aiohttp
-black==23.12.0
-build==1.0.3
+black==24.3.0
+build==1.2.1
     # via pip-tools
 bump2version==1.0.1
 cfgv==3.4.0
     # via pre-commit
-charset-normalizer==3.1.0
-    # via aiohttp
-click==8.1.3
+click==8.1.7
     # via
     #   black
     #   pip-tools
-dill==0.3.7
+dill==0.3.8
     # via pylint
 distlib==0.3.8
     # via virtualenv
-filelock==3.13.1
+filelock==3.13.3
     # via virtualenv
-frozenlist==1.3.3
-    # via
-    #   aiohttp
-    #   aiosignal
-identify==2.5.33
+identify==2.5.35
     # via pre-commit
-idna==3.4
-    # via yarl
 isort==5.13.2
     # via pylint
 mccabe==0.7.0
     # via pylint
-multidict==6.0.3
-    # via
-    #   aiohttp
-    #   yarl
 mypy-extensions==1.0.0
     # via black
 nodeenv==1.8.0
     # via pre-commit
-packaging==23.2
+packaging==24.0
     # via
     #   black
     #   build
 pathspec==0.12.1
     # via black
 pip==24.0
     # via pip-tools
-pip-tools==7.3.0
-platformdirs==4.1.0
+pip-tools==7.4.1
+platformdirs==4.2.0
     # via
     #   black
     #   pylint
     #   virtualenv
-pre-commit==3.6.0
-pylint==3.0.3
+pre-commit==3.7.0
+pylint==3.1.0
 pyproject-hooks==1.0.0
-    # via build
+    # via
+    #   build
+    #   pip-tools
 pyyaml==6.0.1
     # via
     #   pre-commit
     #   watchdog
-ruff==0.1.8
+ruff==0.3.5
 setuptools==69.2.0
     # via
     #   nodeenv
@@ -80,16 +62,14 @@ tomli==2.0.1
     #   pip-tools
     #   pylint
     #   pyproject-hooks
-tomlkit==0.12.3
+tomlkit==0.12.4
     # via pylint
-typing-extensions==4.4.0
+typing-extensions==4.10.0
     # via
     #   astroid
     #   black
-virtualenv==20.25.0
+virtualenv==20.25.1
     # via pre-commit
-watchdog==3.0.0
-wheel==0.42.0
+watchdog==4.0.0
+wheel==0.43.0
     # via pip-tools
-yarl==1.9.2
-    # via aiohttp

services/datcore-adapter/requirements/_tools.txt

@@ -1,5 +1,5 @@
 import logging
-from typing import Final
+from typing import Annotated, Final
 
 from aiocache import cached
 from fastapi import APIRouter, Depends, Header, Request
@@ -35,10 +35,10 @@
 )
 async def list_datasets(
     request: Request,
-    x_datcore_api_key: str = Header(..., description="Datcore API Key"),
-    x_datcore_api_secret: str = Header(..., description="Datcore API Secret"),
-    pennsieve_client: PennsieveApiClient = Depends(get_pennsieve_api_client),
-    params: Params = Depends(),
+    x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
+    x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
+    pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
+    params: Annotated[Params, Depends()],
 ) -> Page[DatasetsOut]:
     assert request  # nosec
     raw_params: RawParams = resolve_params(params).to_raw_params()
@@ -48,7 +48,7 @@ async def list_datasets(
         limit=raw_params.limit,
         offset=raw_params.offset,
     )
-    return create_page(items=datasets, total=total, params=params)
+    return create_page(datasets, total=total, params=params)
 
 
 @router.get(
@@ -65,10 +65,10 @@ async def list_datasets(
 async def list_dataset_top_level_files(
     request: Request,
     dataset_id: str,
-    x_datcore_api_key: str = Header(..., description="Datcore API Key"),
-    x_datcore_api_secret: str = Header(..., description="Datcore API Secret"),
-    pennsieve_client: PennsieveApiClient = Depends(get_pennsieve_api_client),
-    params: Params = Depends(),
+    x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
+    x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
+    pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
+    params: Annotated[Params, Depends()],
 ) -> Page[FileMetaDataOut]:
     assert request  # nosec
     raw_params: RawParams = resolve_params(params).to_raw_params()
@@ -80,7 +80,7 @@ async def list_dataset_top_level_files(
         limit=raw_params.limit,
         offset=raw_params.offset,
     )
-    return create_page(items=file_metas, total=total, params=params)
+    return create_page(file_metas, total=total, params=params)
 
 
 @router.get(
@@ -98,10 +98,10 @@ async def list_dataset_collection_files(
     request: Request,
     dataset_id: str,
     collection_id: str,
-    x_datcore_api_key: str = Header(..., description="Datcore API Key"),
-    x_datcore_api_secret: str = Header(..., description="Datcore API Secret"),
-    pennsieve_client: PennsieveApiClient = Depends(get_pennsieve_api_client),
-    params: Params = Depends(),
+    x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
+    x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
+    pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
+    params: Annotated[Params, Depends()],
 ) -> Page[FileMetaDataOut]:
     assert request  # nosec
     raw_params: RawParams = resolve_params(params).to_raw_params()
@@ -114,7 +114,7 @@ async def list_dataset_collection_files(
         dataset_id=dataset_id,
         collection_id=collection_id,
     )
-    return create_page(items=file_metas, total=total, params=params)
+    return create_page(file_metas, total=total, params=params)
 
 
 @router.get(
@@ -131,14 +131,13 @@ async def list_dataset_collection_files(
 async def list_dataset_files_legacy(
     request: Request,
     dataset_id: str,
-    x_datcore_api_key: str = Header(..., description="Datcore API Key"),
-    x_datcore_api_secret: str = Header(..., description="Datcore API Secret"),
-    pennsieve_client: PennsieveApiClient = Depends(get_pennsieve_api_client),
+    x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
+    x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
+    pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
 ) -> list[FileMetaDataOut]:
     assert request  # nosec
-    file_metas = await pennsieve_client.list_all_dataset_files(
+    return await pennsieve_client.list_all_dataset_files(
         api_key=x_datcore_api_key,
         api_secret=x_datcore_api_secret,
         dataset_id=dataset_id,
     )
-    return file_metas

services/datcore-adapter/src/simcore_service_datcore_adapter/api/routes/datasets.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Any
+from typing import Annotated, Any
 
 from fastapi import APIRouter, Depends, Header, Request
 from pydantic import AnyUrl, parse_obj_as
@@ -24,9 +24,9 @@
 async def download_file(
     request: Request,
     file_id: str,
-    x_datcore_api_key: str = Header(..., description="Datcore API Key"),
-    x_datcore_api_secret: str = Header(..., description="Datcore API Secret"),
-    pennsieve_client: PennsieveApiClient = Depends(get_pennsieve_api_client),
+    x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
+    x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
+    pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
 ) -> FileDownloadOut:
     assert request  # nosec
     presigned_download_link = await pennsieve_client.get_presigned_download_link(
@@ -44,9 +44,9 @@ async def download_file(
 async def delete_file(
     request: Request,
     file_id: str,
-    x_datcore_api_key: str = Header(..., description="Datcore API Key"),
-    x_datcore_api_secret: str = Header(..., description="Datcore API Secret"),
-    pennsieve_client: PennsieveApiClient = Depends(get_pennsieve_api_client),
+    x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
+    x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
+    pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
 ):
     assert request  # nosec
     await pennsieve_client.delete_object(
@@ -66,9 +66,9 @@ async def delete_file(
 async def get_package(
     request: Request,
     package_id: str,
-    x_datcore_api_key: str = Header(..., description="Datcore API Key"),
-    x_datcore_api_secret: str = Header(..., description="Datcore API Secret"),
-    pennsieve_client: PennsieveApiClient = Depends(get_pennsieve_api_client),
+    x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
+    x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
+    pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
 ) -> list[dict[str, Any]]:
     assert request  # nosec
     return await pennsieve_client.get_package_files(

services/datcore-adapter/src/simcore_service_datcore_adapter/api/routes/files.py

@@ -1,6 +1,7 @@
 import logging
-from datetime import datetime
-from typing import Callable
+from collections.abc import Callable
+from datetime import datetime, timezone
+from typing import Annotated
 
 from fastapi import APIRouter, Depends
 from models_library.app_diagnostics import AppStatusCheck
@@ -23,13 +24,13 @@
     status_code=status.HTTP_200_OK,
 )
 async def get_service_alive():
-    return f"{__name__}@{datetime.utcnow().isoformat()}"
+    return f"{__name__}@{datetime.now(timezone.utc).isoformat()}"
 
 
 @router.get("/ready", status_code=status.HTTP_200_OK, response_model=AppStatusCheck)
 async def get_service_ready(
-    pennsieve_client: PennsieveApiClient = Depends(get_pennsieve_api_client),
-    url_for: Callable = Depends(get_reverse_url_mapper),
+    pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
+    url_for: Annotated[Callable, Depends(get_reverse_url_mapper)],
 ):
     pennsieve_health_ok = await pennsieve_client.is_responsive()
     return AppStatusCheck(