♻️Dask sidecar: Glob error tests + autoscaling monitoring fix (#7650)

Author: sanderegg

packages/pytest-simcore/src/pytest_simcore/aws_s3_service.py

@@ -25,7 +25,7 @@ async def s3_client(s3_settings: S3Settings) -> typing.AsyncIterator[S3Client]:
     exit_stack = contextlib.AsyncExitStack()
     session_client = session.client(
         "s3",
-        endpoint_url=f"{s3_settings.S3_ENDPOINT}",
+        endpoint_url=f"{s3_settings.S3_ENDPOINT}" if s3_settings.S3_ENDPOINT else None,
         aws_access_key_id=s3_settings.S3_ACCESS_KEY,
         aws_secret_access_key=s3_settings.S3_SECRET_KEY,
         region_name=s3_settings.S3_REGION,

packages/service-library/tests/aiohttp/test_tracing.py

@@ -3,8 +3,8 @@
 # pylint: disable=unused-variable
 
 import importlib
-from collections.abc import Callable
-from typing import Any, Iterator
+from collections.abc import Callable, Iterator
+from typing import Any
 
 import pip
 import pytest
@@ -92,6 +92,9 @@ def manage_package(request):
     uninstall_package(package)
 
 
+@pytest.mark.skip(
+    reason="this test installs always the latest version of the package which creates conflicts."
+)
 @pytest.mark.parametrize(
     "tracing_settings_in, manage_package",
     [

packages/service-library/tests/fastapi/test_tracing.py

@@ -4,8 +4,8 @@
 import importlib
 import random
 import string
-from collections.abc import Callable
-from typing import Any, Iterator
+from collections.abc import Callable, Iterator
+from typing import Any
 
 import pip
 import pytest
@@ -115,6 +115,9 @@ def manage_package(request):
     uninstall_package(package)
 
 
+@pytest.mark.skip(
+    reason="this test installs always the latest version of the package which creates conflicts."
+)
 @pytest.mark.parametrize(
     "tracing_settings_in, manage_package",
     [

scripts/maintenance/computational-clusters/autoscaled_monitor/dask.py

@@ -116,7 +116,7 @@ def _list_tasks(
         list_of_tasks = await client.run_on_scheduler(_list_tasks)  # type: ignore
     except TypeError:
         rich.print(
-            f"ERROR while recoverring unrunnable tasks using {dask_client=}. Defaulting to empty list of tasks!!"
+            "ERROR while recoverring unrunnable tasks . Defaulting to empty list of tasks!!"
         )
     return list_of_tasks
 
@@ -126,12 +126,16 @@ async def get_scheduler_details(state: AppState, instance: Instance):
     datasets_on_cluster = ()
     processing_jobs = {}
     all_tasks = {}
-    with contextlib.suppress(TimeoutError, OSError):
+    try:
         async with dask_client(state, instance) as client:
             scheduler_info = client.scheduler_info()
             datasets_on_cluster = await _wrap_dask_async_call(client.list_datasets())
             processing_jobs = await _wrap_dask_async_call(client.processing())
             all_tasks = await _list_all_tasks(client)
+    except (TimeoutError, OSError, TypeError):
+        rich.print(
+            "ERROR while recoverring scheduler details !! no scheduler info found!!"
+        )
 
     return scheduler_info, datasets_on_cluster, processing_jobs, all_tasks
 

services/dask-sidecar/requirements/_test.in

@@ -7,7 +7,7 @@
 #
 --constraint _base.txt
 
-
+aioboto3
 coverage
 docker
 faker
@@ -22,4 +22,6 @@ pytest-mock
 pytest-sugar
 python-dotenv
 # mypy
+types-aioboto3
+types-aiobotocore[s3]
 types-aiofiles

services/dask-sidecar/requirements/_test.txt

@@ -1,3 +1,30 @@
+aioboto3==14.1.0
+    # via -r requirements/_test.in
+aiobotocore==2.21.1
+    # via
+    #   -c requirements/_base.txt
+    #   aioboto3
+aiofiles==24.1.0
+    # via
+    #   -c requirements/_base.txt
+    #   aioboto3
+aiohappyeyeballs==2.4.4
+    # via
+    #   -c requirements/_base.txt
+    #   aiohttp
+aiohttp==3.11.18
+    # via
+    #   -c requirements/../../../requirements/constraints.txt
+    #   -c requirements/_base.txt
+    #   aiobotocore
+aioitertools==0.12.0
+    # via
+    #   -c requirements/_base.txt
+    #   aiobotocore
+aiosignal==1.3.1
+    # via
+    #   -c requirements/_base.txt
+    #   aiohttp
 annotated-types==0.7.0
     # via
     #   -c requirements/_base.txt
@@ -7,6 +34,7 @@ antlr4-python3-runtime==4.13.2
 attrs==24.2.0
     # via
     #   -c requirements/_base.txt
+    #   aiohttp
     #   jsonschema
     #   referencing
 aws-sam-translator==1.95.0
@@ -17,15 +45,21 @@ blinker==1.9.0
     # via flask
 boto3==1.37.1
     # via
+    #   aiobotocore
     #   aws-sam-translator
     #   moto
 botocore==1.37.1
     # via
     #   -c requirements/_base.txt
+    #   aiobotocore
     #   aws-xray-sdk
     #   boto3
     #   moto
     #   s3transfer
+botocore-stubs==1.38.11
+    # via
+    #   types-aioboto3
+    #   types-aiobotocore
 certifi==2024.8.30
     # via
     #   -c requirements/../../../requirements/constraints.txt
@@ -65,6 +99,11 @@ flask==3.1.0
     #   moto
 flask-cors==5.0.1
     # via moto
+frozenlist==1.5.0
+    # via
+    #   -c requirements/_base.txt
+    #   aiohttp
+    #   aiosignal
 graphql-core==3.2.6
     # via moto
 icdiff==2.0.7
@@ -73,6 +112,7 @@ idna==3.10
     # via
     #   -c requirements/_base.txt
     #   requests
+    #   yarl
 iniconfig==2.0.0
     # via pytest
 itsdangerous==2.2.0
@@ -86,6 +126,7 @@ jinja2==3.1.4
 jmespath==1.0.1
     # via
     #   -c requirements/_base.txt
+    #   aiobotocore
     #   boto3
     #   botocore
 joserfc==1.0.4
@@ -120,6 +161,12 @@ moto==5.1.4
     # via -r requirements/_test.in
 mpmath==1.3.0
     # via sympy
+multidict==6.1.0
+    # via
+    #   -c requirements/_base.txt
+    #   aiobotocore
+    #   aiohttp
+    #   yarl
 networkx==3.4.2
     # via cfn-lint
 openapi-schema-validator==0.6.3
@@ -139,6 +186,11 @@ ply==3.11
     # via jsonpath-ng
 pprintpp==0.4.0
     # via pytest-icdiff
+propcache==0.2.1
+    # via
+    #   -c requirements/_base.txt
+    #   aiohttp
+    #   yarl
 py-partiql-parser==0.6.1
     # via moto
 pycparser==2.22
@@ -187,6 +239,7 @@ pytest-sugar==1.0.0
 python-dateutil==2.9.0.post0
     # via
     #   -c requirements/_base.txt
+    #   aiobotocore
     #   botocore
     #   moto
 python-dotenv==1.0.1
@@ -239,8 +292,20 @@ sympy==1.13.3
     # via cfn-lint
 termcolor==2.5.0
     # via pytest-sugar
+types-aioboto3==14.3.0
+    # via -r requirements/_test.in
+types-aiobotocore==2.22.0
+    # via
+    #   -r requirements/_test.in
+    #   types-aioboto3
+types-aiobotocore-s3==2.22.0
+    # via types-aiobotocore
 types-aiofiles==24.1.0.20241221
     # via -r requirements/_test.in
+types-awscrt==0.26.1
+    # via botocore-stubs
+types-s3transfer==0.12.0
+    # via types-aioboto3
 typing-extensions==4.12.2
     # via
     #   -c requirements/_base.txt
@@ -249,6 +314,9 @@ typing-extensions==4.12.2
     #   pydantic
     #   pydantic-core
     #   pyopenssl
+    #   types-aioboto3
+    #   types-aiobotocore
+    #   types-aiobotocore-s3
 tzdata==2024.2
     # via
     #   -c requirements/_base.txt
@@ -269,6 +337,11 @@ werkzeug==3.1.3
 wrapt==1.17.0
     # via
     #   -c requirements/_base.txt
+    #   aiobotocore
     #   aws-xray-sdk
 xmltodict==0.14.2
     # via moto
+yarl==1.18.3
+    # via
+    #   -c requirements/_base.txt
+    #   aiohttp

services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/docker_utils.py

@@ -3,7 +3,13 @@
 import logging
 import re
 import socket
-from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable
+from collections.abc import (
+    AsyncGenerator,
+    AsyncIterator,
+    Awaitable,
+    Callable,
+    Coroutine,
+)
 from pathlib import Path
 from pprint import pformat
 from typing import Any, Final, cast
@@ -51,7 +57,7 @@
 from .task_shared_volume import TaskSharedVolumes
 
 logger = logging.getLogger(__name__)
-LogPublishingCB = Callable[[LogMessageStr, LogLevelInt], Awaitable[None]]
+LogPublishingCB = Callable[[LogMessageStr, LogLevelInt], Coroutine[Any, Any, None]]
 
 
 async def create_container_config(

services/dask-sidecar/src/simcore_service_dask_sidecar/utils/files.py

@@ -4,7 +4,7 @@
 import mimetypes
 import time
 import zipfile
-from collections.abc import Awaitable, Callable
+from collections.abc import Callable, Coroutine
 from io import IOBase
 from pathlib import Path
 from typing import Any, Final, TypedDict, cast
@@ -25,7 +25,7 @@
 S3_FILE_SYSTEM_SCHEMES: Final = ["s3", "s3a"]
 
 
-LogPublishingCB = Callable[[LogMessageStr, LogLevelInt], Awaitable[None]]
+LogPublishingCB = Callable[[LogMessageStr, LogLevelInt], Coroutine[Any, Any, None]]
 
 
 def _file_progress_cb(

services/dask-sidecar/tests/unit/conftest.py

@@ -14,7 +14,6 @@
 import fsspec
 import pytest
 import simcore_service_dask_sidecar
-from aiobotocore.session import AioBaseClient, get_session
 from common_library.json_serialization import json_dumps
 from common_library.serialization import model_dump_with_secrets
 from dask_task_models_library.container_tasks.protocol import TaskOwner
@@ -36,6 +35,7 @@
 
 pytest_plugins = [
     "pytest_simcore.aws_server",
+    "pytest_simcore.aws_s3_service",
     "pytest_simcore.cli_runner",
     "pytest_simcore.docker_compose",
     "pytest_simcore.docker_registry",
@@ -179,45 +179,6 @@ def s3_settings(mocked_s3_server_envs: None) -> S3Settings:
     return S3Settings.create_from_envs()
 
 
-@pytest.fixture
-def s3_endpoint_url(s3_settings: S3Settings) -> AnyUrl:
-    assert s3_settings.S3_ENDPOINT
-    return TypeAdapter(AnyUrl).validate_python(
-        f"{s3_settings.S3_ENDPOINT}",
-    )
-
-
-@pytest.fixture
-async def aiobotocore_s3_client(
-    s3_settings: S3Settings, s3_endpoint_url: AnyUrl
-) -> AsyncIterator[AioBaseClient]:
-    session = get_session()
-    async with session.create_client(
-        "s3",
-        endpoint_url=f"{s3_endpoint_url}",
-        aws_secret_access_key="xxx",  # noqa: S106
-        aws_access_key_id="xxx",
-    ) as client:
-        yield client
-
-
-@pytest.fixture
-async def bucket(
-    aiobotocore_s3_client: AioBaseClient, s3_settings: S3Settings
-) -> AsyncIterator[str]:
-    response = await aiobotocore_s3_client.create_bucket(
-        Bucket=s3_settings.S3_BUCKET_NAME
-    )
-    assert "ResponseMetadata" in response
-    assert "HTTPStatusCode" in response["ResponseMetadata"]
-    assert response["ResponseMetadata"]["HTTPStatusCode"] == 200
-
-    response = await aiobotocore_s3_client.list_buckets()
-    assert response["Buckets"]
-    assert len(response["Buckets"]) == 1
-    return response["Buckets"][0]["Name"]
-
-
 @pytest.fixture
 def s3_remote_file_url(s3_settings: S3Settings, faker: Faker) -> Callable[..., AnyUrl]:
     def creator(file_path: Path | None = None) -> AnyUrl: