Merge branch 'master' into enh/search-by-tags

Author: odeimaiz

packages/models-library/src/models_library/api_schemas_dynamic_sidecar/__init__.py

@@ -0,0 +1,9 @@
+from typing import Final
+
+from pydantic import parse_obj_as
+
+from ..rabbitmq_basic_types import RPCNamespace
+
+DYNAMIC_SIDECAR_RPC_NAMESPACE: Final[RPCNamespace] = parse_obj_as(
+    RPCNamespace, "dynamic-sidecar"
+)

packages/models-library/src/models_library/api_schemas_dynamic_sidecar/telemetry.py

@@ -1,9 +1,28 @@
 from abc import abstractmethod
-from pathlib import Path
-from typing import Protocol
+from enum import auto
+from typing import Any, Final, Protocol
 
-from models_library.projects_nodes_io import NodeID
-from pydantic import BaseModel, ByteSize, Field
+from pydantic import (
+    BaseModel,
+    ByteSize,
+    Field,
+    NonNegativeFloat,
+    NonNegativeInt,
+    root_validator,
+    validator,
+)
+
+from ..projects_nodes_io import NodeID
+from ..utils.enums import StrAutoEnum
+
+_EPSILON: Final[NonNegativeFloat] = 1e-16
+
+
+class MountPathCategory(StrAutoEnum):
+    HOST = auto()
+    STATES_VOLUMES = auto()
+    INPUTS_VOLUMES = auto()
+    OUTPUTS_VOLUMES = auto()
 
 
 class SDiskUsageProtocol(Protocol):
@@ -28,31 +47,71 @@ def percent(self) -> float:
         ...
 
 
+def _get_percent(used: float, total: float) -> float:
+    return round(used * 100 / (total + _EPSILON), 2)
+
+
 class DiskUsage(BaseModel):
     used: ByteSize = Field(description="used space")
     free: ByteSize = Field(description="remaining space")
 
     total: ByteSize = Field(description="total space = free + used")
-    used_percent: float = Field(
+    used_percent: NonNegativeFloat = Field(
         gte=0.00,
         lte=100.00,
         description="Percent of used space relative to the total space",
     )
 
+    @validator("free")
     @classmethod
-    def from_ps_util_disk_usage(
-        cls, ps_util_disk_usage: SDiskUsageProtocol
+    def _free_positive(cls, v: float) -> float:
+        if v < 0:
+            msg = f"free={v} cannot be a negative value"
+            raise ValueError(msg)
+        return v
+
+    @validator("used")
+    @classmethod
+    def _used_positive(cls, v: float) -> float:
+        if v < 0:
+            msg = f"used={v} cannot be a negative value"
+            raise ValueError(msg)
+        return v
+
+    @root_validator(pre=True)
+    @classmethod
+    def _check_total(cls, values: dict[str, Any]) -> dict[str, Any]:
+        total = values["total"]
+        free = values["free"]
+        used = values["used"]
+        if total != free + used:
+            msg = f"{total=} is different than the sum of {free=}+{used=} => sum={free+used}"
+            raise ValueError(msg)
+        return values
+
+    @classmethod
+    def from_efs_guardian(
+        cls, used: NonNegativeInt, total: NonNegativeInt
     ) -> "DiskUsage":
-        total = ps_util_disk_usage.free + ps_util_disk_usage.used
-        used_percent = round(ps_util_disk_usage.used * 100 / total, 2)
+        free = total - used
         return cls(
-            used=ByteSize(ps_util_disk_usage.used),
-            free=ByteSize(ps_util_disk_usage.free),
+            used=ByteSize(used),
+            free=ByteSize(free),
             total=ByteSize(total),
-            used_percent=used_percent,
+            used_percent=_get_percent(used, total),
         )
 
+    @classmethod
+    def from_ps_util_disk_usage(
+        cls, ps_util_disk_usage: SDiskUsageProtocol
+    ) -> "DiskUsage":
+        total = ps_util_disk_usage.free + ps_util_disk_usage.used
+        return cls.from_efs_guardian(ps_util_disk_usage.used, total)
+
+    def __hash__(self):
+        return hash((self.used, self.free, self.total, self.used_percent))
+
 
 class ServiceDiskUsage(BaseModel):
     node_id: NodeID
-    usage: dict[Path, DiskUsage]
+    usage: dict[MountPathCategory, DiskUsage]

packages/models-library/tests/test_api_schemas_dynamic_sidecar_telemetry.py

@@ -2,6 +2,7 @@
 import pytest
 from models_library.api_schemas_dynamic_sidecar.telemetry import DiskUsage
 from psutil._common import sdiskusage
+from pydantic import ByteSize, ValidationError
 
 
 def _assert_same_value(ps_util_disk_usage: sdiskusage) -> None:
@@ -27,3 +28,35 @@ def test_disk_usage_regression_cases(ps_util_disk_usage: sdiskusage):
 def test_disk_usage():
     ps_util_disk_usage = psutil.disk_usage("/")
     _assert_same_value(ps_util_disk_usage)
+
+
+def test_from_efs_guardian_constructor():
+    result = DiskUsage.from_efs_guardian(10, 100)
+    assert result.used == ByteSize(10)
+    assert result.free == ByteSize(90)
+    assert result.total == ByteSize(100)
+    assert result.used_percent == 10
+
+
+def test_failing_validation():
+    with pytest.raises(ValidationError) as exc:
+        assert DiskUsage.from_efs_guardian(100, 10)
+
+    assert "free=" in f"{exc.value}"
+    assert "negative value" in f"{exc.value}"
+
+    with pytest.raises(ValidationError) as exc:
+        assert DiskUsage(
+            used=-10,  # type: ignore
+            free=ByteSize(10),
+            total=ByteSize(0),
+            used_percent=-10,
+        )
+    assert "used=" in f"{exc.value}"
+    assert "negative value" in f"{exc.value}"
+
+    with pytest.raises(ValidationError) as exc:
+        DiskUsage(
+            used=ByteSize(10), free=ByteSize(10), total=ByteSize(21), used_percent=0
+        )
+    assert "is different than the sum of" in f"{exc.value}"

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/__init__.py

@@ -0,0 +1,22 @@
+import logging
+
+from models_library.api_schemas_dynamic_sidecar import DYNAMIC_SIDECAR_RPC_NAMESPACE
+from models_library.api_schemas_dynamic_sidecar.telemetry import DiskUsage
+from models_library.rabbitmq_basic_types import RPCMethodName
+from pydantic import parse_obj_as
+from servicelib.logging_utils import log_decorator
+from servicelib.rabbitmq import RabbitMQRPCClient
+
+_logger = logging.getLogger(__name__)
+
+
+@log_decorator(_logger, level=logging.DEBUG)
+async def update_disk_usage(
+    rabbitmq_rpc_client: RabbitMQRPCClient, *, usage: dict[str, DiskUsage]
+) -> None:
+    result = await rabbitmq_rpc_client.request(
+        DYNAMIC_SIDECAR_RPC_NAMESPACE,
+        parse_obj_as(RPCMethodName, "update_disk_usage"),
+        usage=usage,
+    )
+    assert result is None  # nosec

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk_usage.py

@@ -34,6 +34,7 @@
 )
 
 JobID: TypeAlias = UUID
+assert JobID == ProjectID
 
 # ArgumentTypes are types used in the job inputs (see ResultsTypes)
 ArgumentTypes: TypeAlias = (

services/api-server/src/simcore_service_api_server/models/schemas/jobs.py

@@ -8,7 +8,10 @@
 import pytest
 import simcore_service_api_server
 from dotenv import dotenv_values
+from models_library.projects import ProjectID
+from pydantic import parse_obj_as
 from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict
+from simcore_service_api_server.models.schemas.jobs import JobID
 
 CURRENT_DIR = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent
 
@@ -96,3 +99,11 @@ def tests_utils_dir(project_tests_dir: Path) -> Path:
     utils_dir = (project_tests_dir / "utils").resolve()
     assert utils_dir.exists()
     return utils_dir
+
+
+## BASIC IDENTIFIERS ---
+
+
+@pytest.fixture
+def job_id(project_id: ProjectID) -> JobID:
+    return parse_obj_as(JobID, project_id)

services/api-server/tests/conftest.py

@@ -3,14 +3,23 @@
 # pylint: disable=unused-variable
 
 import random
+import textwrap
 import urllib.parse
 from copy import deepcopy
 from uuid import uuid4
 
 import pytest
+from faker import Faker
 from fastapi import FastAPI
+from models_library.api_schemas_webserver.projects_metadata import ProjectMetadataGet
+from models_library.generics import Envelope
 from simcore_service_api_server._meta import API_VTAG
-from simcore_service_api_server.models.schemas.jobs import Job, JobInputs
+from simcore_service_api_server.models.schemas.jobs import (
+    Job,
+    JobID,
+    JobInputs,
+    JobMetadata,
+)
 from simcore_service_api_server.models.schemas.solvers import Solver
 
 
@@ -70,3 +79,48 @@ def test_job_resouce_names_has_associated_url(app: FastAPI):
     )
 
     assert url_path == f"/{API_VTAG}/{urllib.parse.unquote_plus(job_name)}"
+
+
+@pytest.mark.acceptance_test(
+    "Fixing https://github.com/ITISFoundation/osparc-simcore/issues/6556"
+)
+def test_parsing_job_custom_metadata(job_id: JobID, faker: Faker):
+    job_name = faker.name()
+
+    got = Envelope[ProjectMetadataGet].parse_raw(
+        textwrap.dedent(
+            f"""
+        {{
+            "data": {{
+            "projectUuid": "{job_id}",
+            "custom": {{
+                "number": 3.14,
+                "string": "foo",
+                "boolean": true,
+                "integer": 42,
+                "job_id": "{job_id}",
+                "job_name": "{job_name}"
+                }}
+            }}
+        }}
+        """
+        )
+    )
+
+    assert got.data
+    assert got.data.custom == {
+        "number": 3.14,
+        "string": "foo",
+        "boolean": True,
+        "integer": 42,
+        "job_id": f"{job_id}",
+        "job_name": job_name,
+    }
+
+    j = JobMetadata(
+        job_id=job_id,
+        metadata=got.data.custom or {},
+        url=faker.url(),
+    )
+
+    assert j.metadata == got.data.custom

services/api-server/tests/unit/test_models_schemas_jobs.py

@@ -37,6 +37,7 @@
 from models_library.utils.fastapi_encoders import jsonable_encoder
 from pydantic import AnyHttpUrl, parse_obj_as
 from servicelib.async_utils import run_sequentially_in_context
+from servicelib.logging_utils import log_decorator
 from servicelib.rabbitmq import RabbitMQRPCClient
 from simcore_postgres_database.utils_projects_metadata import DBProjectNotFoundError
 from starlette import status
@@ -150,6 +151,7 @@ async def _check_pipeline_startable(
 _UNKNOWN_NODE: Final[str] = "unknown node"
 
 
+@log_decorator(_logger)
 async def _get_project_metadata(
     project_id: ProjectID,
     project_repo: ProjectsRepository,
@@ -160,7 +162,7 @@ async def _get_project_metadata(
             project_id
         )
         if project_ancestors.parent_project_uuid is None:
-            # no parents here
+            _logger.debug("no parent found for project %s", project_id)
             return {}
 
         assert project_ancestors.parent_node_id is not None  # nosec

services/director-v2/src/simcore_service_director_v2/api/routes/computations.py

@@ -50,7 +50,7 @@ async def get(
             )
             row: RowProxy | None = await result.first()
             if not row:
-                raise ComputationalRunNotFoundError()
+                raise ComputationalRunNotFoundError
             return CompRunsAtDB.from_orm(row)
 
     async def list(
@@ -80,7 +80,7 @@ async def create(
         project_id: ProjectID,
         cluster_id: ClusterID,
         iteration: PositiveInt | None = None,
-        metadata: RunMetadataDict | None,
+        metadata: RunMetadataDict,
         use_on_demand_clusters: bool,
     ) -> CompRunsAtDB:
         try:
@@ -102,13 +102,13 @@ async def create(
                     .values(
                         user_id=user_id,
                         project_uuid=f"{project_id}",
-                        cluster_id=cluster_id
-                        if cluster_id != DEFAULT_CLUSTER_ID
-                        else None,
+                        cluster_id=(
+                            cluster_id if cluster_id != DEFAULT_CLUSTER_ID else None
+                        ),
                         iteration=iteration,
                         result=RUNNING_STATE_TO_DB[RunningState.PUBLISHED],
-                        started=datetime.datetime.now(tz=datetime.timezone.utc),
-                        metadata=jsonable_encoder(metadata) if metadata else None,
+                        started=datetime.datetime.now(tz=datetime.UTC),
+                        metadata=jsonable_encoder(metadata),
                         use_on_demand_clusters=use_on_demand_clusters,
                     )
                     .returning(literal_column("*"))
@@ -146,7 +146,7 @@ async def set_run_result(
     ) -> CompRunsAtDB | None:
         values: dict[str, Any] = {"result": RUNNING_STATE_TO_DB[result_state]}
         if final_state:
-            values.update({"ended": datetime.datetime.now(tz=datetime.timezone.utc)})
+            values.update({"ended": datetime.datetime.now(tz=datetime.UTC)})
         return await self.update(
             user_id,
             project_id,