Merge branch 'master' into 7548-monitor-api-servers-log-streaming-queues

Author: bisgaard-itis

.env-devel

@@ -86,10 +86,10 @@ DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS=null
 DIRECTOR_TRACING=null
 
 DOCKER_API_PROXY_HOST=docker-api-proxy
-DOCKER_API_PROXY_PASSWORD=null
+DOCKER_API_PROXY_PASSWORD=admin
 DOCKER_API_PROXY_PORT=8888
 DOCKER_API_PROXY_SECURE=False
-DOCKER_API_PROXY_USER=null
+DOCKER_API_PROXY_USER=admin
 
 EFS_USER_ID=8006
 EFS_USER_NAME=efs

.github/workflows/ci-testing-deploy.yml

@@ -573,7 +573,7 @@ jobs:
       - name: install uv
         uses: astral-sh/setup-uv@v6
         with:
-          version: "0.5.x"
+          version: "0.6.x"
           enable-cache: false
           cache-dependency-glob: "**/notifications/requirements/ci.txt"
       - name: show system version

packages/models-library/src/models_library/rpc/webserver/projects.py

@@ -29,7 +29,7 @@ class ProjectJobRpcGet(BaseModel):
     workbench: NodesDict
 
     # timestamps
-    creation_at: datetime
+    created_at: datetime
     modified_at: datetime
 
     # Specific to jobs
@@ -43,12 +43,30 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
                 "examples": [
                     {
                         "uuid": "12345678-1234-5678-1234-123456789012",
-                        "name": "My project",
-                        "description": "My project description",
+                        "name": "A solver job",
+                        "description": "A description of a solver job with a single node",
                         "workbench": {f"{uuid4()}": n for n in nodes_examples[2:3]},
-                        "creation_at": "2023-01-01T00:00:00Z",
+                        "created_at": "2023-01-01T00:00:00Z",
                         "modified_at": "2023-01-01T00:00:00Z",
-                        "job_parent_resource_name": "solvers/foo/release/1.2.3",
+                        "job_parent_resource_name": "solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.0.2",
+                    },
+                    {
+                        "uuid": "00000000-1234-5678-1234-123456789012",
+                        "name": "A study job",
+                        "description": "A description of a study job with many node",
+                        "workbench": {f"{uuid4()}": n for n in nodes_examples},
+                        "created_at": "2023-02-01T00:00:00Z",
+                        "modified_at": "2023-02-01T00:00:00Z",
+                        "job_parent_resource_name": "studies/96642f2a-a72c-11ef-8776-02420a00087d",
+                    },
+                    {
+                        "uuid": "00000000-0000-5678-1234-123456789012",
+                        "name": "A program job",
+                        "description": "A program of a solver job with a single node",
+                        "workbench": {f"{uuid4()}": n for n in nodes_examples[2:3]},
+                        "created_at": "2023-03-01T00:00:00Z",
+                        "modified_at": "2023-03-01T00:00:00Z",
+                        "job_parent_resource_name": "program/simcore%2Fservices%2Fdynamic%2Fjupyter/releases/5.0.2",
                     },
                 ]
             }

packages/models-library/src/models_library/utils/common_validators.py

@@ -1,4 +1,4 @@
-""" Reusable validators
+"""Reusable validators
 
     Example:
 
@@ -22,10 +22,19 @@ class MyModel(BaseModel):
 
 from common_library.json_serialization import json_loads
 from orjson import JSONDecodeError
-from pydantic import BaseModel
+from pydantic import BaseModel, BeforeValidator
 from pydantic.alias_generators import to_camel
 
 
+def trim_string_before(max_length: int) -> BeforeValidator:
+    def _trim(value: str):
+        if isinstance(value, str):
+            return value[:max_length]
+        return value
+
+    return BeforeValidator(_trim)
+
+
 def empty_str_to_none_pre_validator(value: Any):
     if isinstance(value, str) and value.strip() == "":
         return None

packages/models-library/tests/test_utils_common_validators.py

@@ -1,13 +1,15 @@
 from enum import Enum
+from typing import Annotated
 
 import pytest
 from models_library.utils.common_validators import (
     create_enums_pre_validator,
     empty_str_to_none_pre_validator,
     none_to_empty_str_pre_validator,
     null_or_none_str_to_none_validator,
+    trim_string_before,
 )
-from pydantic import BaseModel, ValidationError, field_validator
+from pydantic import BaseModel, StringConstraints, ValidationError, field_validator
 
 
 def test_enums_pre_validator():
@@ -89,3 +91,67 @@ class Model(BaseModel):
 
     model = Model.model_validate({"message": ""})
     assert model == Model.model_validate({"message": ""})
+
+
+def test_trim_string_before():
+    max_length = 10
+
+    class ModelWithTrim(BaseModel):
+        text: Annotated[str, trim_string_before(max_length=max_length)]
+
+    # Test with string shorter than max_length
+    short_text = "Short"
+    model = ModelWithTrim(text=short_text)
+    assert model.text == short_text
+
+    # Test with string equal to max_length
+    exact_text = "1234567890"  # 10 characters
+    model = ModelWithTrim(text=exact_text)
+    assert model.text == exact_text
+
+    # Test with string longer than max_length
+    long_text = "This is a very long text that should be trimmed"
+    model = ModelWithTrim(text=long_text)
+    assert model.text == long_text[:max_length]
+    assert len(model.text) == max_length
+
+    # Test with non-string value (should be left unchanged)
+    class ModelWithTrimOptional(BaseModel):
+        text: Annotated[str | None, trim_string_before(max_length=max_length)]
+
+    model = ModelWithTrimOptional(text=None)
+    assert model.text is None
+
+
+def test_trim_string_before_with_string_constraints():
+    max_length = 10
+
+    class ModelWithTrimAndConstraints(BaseModel):
+        text: Annotated[
+            str | None,
+            StringConstraints(
+                max_length=max_length
+            ),  # NOTE: order does not matter for validation but has an effect in the openapi schema
+            trim_string_before(max_length=max_length),
+        ]
+
+    # Check that the OpenAPI schema contains the string constraint
+    schema = ModelWithTrimAndConstraints.model_json_schema()
+    assert schema["properties"]["text"] == {
+        "anyOf": [{"maxLength": max_length, "type": "string"}, {"type": "null"}],
+        "title": "Text",
+    }
+
+    # Test with string longer than max_length
+    # This should pass because trim_string_before runs first and trims the input
+    # before StringConstraints validation happens
+    long_text = "This is a very long text that should be trimmed"
+    model = ModelWithTrimAndConstraints(text=long_text)
+    assert model.text is not None
+    assert model.text == long_text[:max_length]
+    assert len(model.text) == max_length
+
+    # Test with string exactly at max_length
+    exact_text = "1234567890"  # 10 characters
+    model = ModelWithTrimAndConstraints(text=exact_text)
+    assert model.text == exact_text

packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py

@@ -1,7 +1,7 @@
 import logging
 
 import pytest
-from aiohttp import ClientSession, ClientTimeout
+from aiohttp import BasicAuth, ClientSession, ClientTimeout
 from pydantic import TypeAdapter
 from settings_library.docker_api_proxy import DockerApiProxysettings
 from tenacity import before_sleep_log, retry, stop_after_delay, wait_fixed
@@ -22,7 +22,13 @@
 async def _wait_till_docker_api_proxy_is_responsive(
     settings: DockerApiProxysettings,
 ) -> None:
-    async with ClientSession(timeout=ClientTimeout(1, 1, 1, 1, 1)) as client:
+    async with ClientSession(
+        timeout=ClientTimeout(total=1),
+        auth=BasicAuth(
+            settings.DOCKER_API_PROXY_USER,
+            settings.DOCKER_API_PROXY_PASSWORD.get_secret_value(),
+        ),
+    ) as client:
         response = await client.get(f"{settings.base_url}/version")
         assert response.status == 200, await response.text()
 
@@ -44,6 +50,12 @@ async def docker_api_proxy_settings(
         {
             "DOCKER_API_PROXY_HOST": get_localhost_ip(),
             "DOCKER_API_PROXY_PORT": published_port,
+            "DOCKER_API_PROXY_USER": env_vars_for_docker_compose[
+                "DOCKER_API_PROXY_USER"
+            ],
+            "DOCKER_API_PROXY_PASSWORD": env_vars_for_docker_compose[
+                "DOCKER_API_PROXY_PASSWORD"
+            ],
         }
     )
 

packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py

@@ -8,7 +8,10 @@
 from models_library.products import ProductName
 from models_library.projects import ProjectID
 from models_library.rest_pagination import PageOffsetInt
-from models_library.rpc.webserver.projects import PageRpcProjectJobRpcGet
+from models_library.rpc.webserver.projects import (
+    PageRpcProjectJobRpcGet,
+    ProjectJobRpcGet,
+)
 from models_library.rpc_pagination import (
     DEFAULT_NUMBER_OF_ITEMS_PER_PAGE,
     PageLimitInt,
@@ -54,19 +57,28 @@ async def list_projects_marked_as_jobs(
         offset: PageOffsetInt = 0,
         limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE,
         # filters
-        job_parent_resource_name_filter: str | None = None,
+        job_parent_resource_name_prefix: str | None = None,
     ) -> PageRpcProjectJobRpcGet:
         assert rpc_client
         assert product_name
         assert user_id
 
-        if job_parent_resource_name_filter:
-            assert not job_parent_resource_name_filter.startswith("/")
+        if job_parent_resource_name_prefix:
+            assert not job_parent_resource_name_prefix.startswith("/")
+            assert not job_parent_resource_name_prefix.endswith("%")
+            assert not job_parent_resource_name_prefix.startswith("%")
 
-        items = PageRpcProjectJobRpcGet.model_json_schema()["examples"]
+        items = [
+            item
+            for item in ProjectJobRpcGet.model_json_schema()["examples"]
+            if job_parent_resource_name_prefix is None
+            or item.get("job_parent_resource_name").startswith(
+                job_parent_resource_name_prefix
+            )
+        ]
 
         return PageRpcProjectJobRpcGet.create(
-            items[offset, : offset + limit],
+            items[offset : offset + limit],
             total=len(items),
             limit=limit,
             offset=offset,

packages/pytest-simcore/src/pytest_simcore/simcore_services.py

@@ -9,6 +9,7 @@
 from collections.abc import Iterator
 from dataclasses import dataclass
 from io import StringIO
+from typing import Final
 
 import aiohttp
 import pytest
@@ -27,7 +28,7 @@
 log = logging.getLogger(__name__)
 
 
-_SERVICES_TO_SKIP = {
+_SERVICES_TO_SKIP: Final[set[str]] = {
     "agent",  # global mode deploy (NO exposed ports, has http API)
     "dask-sidecar",  # global mode deploy (NO exposed ports, **NO** http API)
     "migration",
@@ -41,9 +42,8 @@
     "sto-worker-cpu-bound",
 }
 # TODO: unify healthcheck policies see  https://github.com/ITISFoundation/osparc-simcore/pull/2281
-SERVICE_PUBLISHED_PORT = {}
-DEFAULT_SERVICE_HEALTHCHECK_ENTRYPOINT = "/v0/"
-MAP_SERVICE_HEALTHCHECK_ENTRYPOINT = {
+DEFAULT_SERVICE_HEALTHCHECK_ENTRYPOINT: Final[str] = "/v0/"
+MAP_SERVICE_HEALTHCHECK_ENTRYPOINT: Final[dict[str, str]] = {
     "autoscaling": "/",
     "clusters-keeper": "/",
     "dask-scheduler": "/health",
@@ -57,16 +57,23 @@
     "resource-usage-tracker": "/",
     "docker-api-proxy": "/version",
 }
-AIOHTTP_BASED_SERVICE_PORT: int = 8080
-FASTAPI_BASED_SERVICE_PORT: int = 8000
-DASK_SCHEDULER_SERVICE_PORT: int = 8787
-DOCKER_API_PROXY_SERVICE_PORT: int = 8888
 
-_SERVICE_NAME_REPLACEMENTS: dict[str, str] = {
+# some services require authentication to access their health-check endpoints
+_BASE_AUTH_ENV_VARS: Final[dict[str, tuple[str, str]]] = {
+    "docker-api-proxy": ("DOCKER_API_PROXY_USER", "DOCKER_API_PROXY_PASSWORD"),
+}
+
+_SERVICE_NAME_REPLACEMENTS: Final[dict[str, str]] = {
     "dynamic-scheduler": "dynamic-schdlr",
 }
 
-_ONE_SEC_TIMEOUT = ClientTimeout(total=1)  # type: ignore
+
+AIOHTTP_BASED_SERVICE_PORT: Final[int] = 8080
+FASTAPI_BASED_SERVICE_PORT: Final[int] = 8000
+DASK_SCHEDULER_SERVICE_PORT: Final[int] = 8787
+DOCKER_API_PROXY_SERVICE_PORT: Final[int] = 8888
+
+_ONE_SEC_TIMEOUT: Final[ClientTimeout] = ClientTimeout(total=1)  # type: ignore
 
 
 async def wait_till_service_healthy(service_name: str, endpoint: URL):
@@ -108,13 +115,12 @@ class ServiceHealthcheckEndpoint:
     @classmethod
     def create(cls, service_name: str, baseurl):
         # TODO: unify healthcheck policies see  https://github.com/ITISFoundation/osparc-simcore/pull/2281
-        obj = cls(
+        return cls(
             name=service_name,
             url=URL(
                 f"{baseurl}{MAP_SERVICE_HEALTHCHECK_ENTRYPOINT.get(service_name, DEFAULT_SERVICE_HEALTHCHECK_ENTRYPOINT)}"
             ),
         )
-        return obj
 
 
 @pytest.fixture(scope="module")
@@ -140,9 +146,17 @@ def services_endpoint(
                 DASK_SCHEDULER_SERVICE_PORT,
                 DOCKER_API_PROXY_SERVICE_PORT,
             ]
-            endpoint = URL(
-                f"http://{get_localhost_ip()}:{get_service_published_port(full_service_name, target_ports)}"
-            )
+            if service in _BASE_AUTH_ENV_VARS:
+                user_env, password_env = _BASE_AUTH_ENV_VARS[service]
+                user = env_vars_for_docker_compose[user_env]
+                password = env_vars_for_docker_compose[password_env]
+                endpoint = URL(
+                    f"http://{user}:{password}@{get_localhost_ip()}:{get_service_published_port(full_service_name, target_ports)}"
+                )
+            else:
+                endpoint = URL(
+                    f"http://{get_localhost_ip()}:{get_service_published_port(full_service_name, target_ports)}"
+                )
             services_endpoint[service] = endpoint
         else:
             print(f"Collecting service endpoints: '{service}' skipped")

packages/service-integration/Dockerfile

@@ -4,8 +4,15 @@
 ARG PYTHON_VERSION="3.11.9"
 ARG UV_VERSION="0.6"
 FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build
-# we docker image is built based on debian
-FROM python:${PYTHON_VERSION}-slim-bookworm AS base
+
+FROM python:${PYTHON_VERSION}-slim-bookworm AS base-arm64
+# These environment variables are necessary because of https://github.com/astral-sh/uv/issues/6105
+# and until https://gitlab.com/qemu-project/qemu/-/issues/2846 gets fixed
+ENV UV_CONCURRENT_INSTALLS=1
+
+FROM python:${PYTHON_VERSION}-slim-bookworm AS base-amd64
+
+FROM base-${TARGETARCH} AS base
 
 LABEL maintainer=pcrespov
 
@@ -66,13 +73,7 @@ COPY --from=uv_build /uv /uvx /bin/
 # packages may be moved to production image easily by copying the venv
 RUN uv venv "${VIRTUAL_ENV}"
 
-# Set UV_CONCURRENT_INSTALLS=1 for ARM64 to improve build performance
-# These environment variables are necessary because of https://github.com/astral-sh/uv/issues/6105
-# and until https://gitlab.com/qemu-project/qemu/-/issues/2846 gets fixed
-RUN if [ "$TARGETARCH" = "arm64" ]; then \
-  echo "Setting UV_CONCURRENT_INSTALLS=1 for ARM64 architecture"; \
-  export UV_CONCURRENT_INSTALLS=1; \
-  fi
+
 
 RUN --mount=type=cache,target=/root/.cache/uv \
   uv pip install --upgrade \