Merge remote-tracking branch 'upstream/master' into is5646/use-project-nodes-table-instead-of-workbench

Author: giancarloromeo

packages/models-library/src/models_library/api_schemas_webserver/projects.py

@@ -274,6 +274,7 @@ class ProjectPatch(InputSchema):
     ] = None
     quality: dict[str, Any] | None = None
     template_type: ProjectTemplateType | None = None
+    hidden: bool | None = None
 
     def to_domain_model(self) -> dict[str, Any]:
         return self.model_dump(exclude_unset=True, by_alias=False)

packages/models-library/src/models_library/rpc/webserver/projects.py

@@ -90,6 +90,7 @@ class ProjectJobRpcGet(BaseModel):
 
     # Specific to jobs
     job_parent_resource_name: str
+    storage_assets_deleted: bool
 
     @staticmethod
     def _update_json_schema_extra(schema: JsonDict) -> None:
@@ -105,6 +106,7 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
                         "created_at": "2023-01-01T00:00:00Z",
                         "modified_at": "2023-01-01T00:00:00Z",
                         "job_parent_resource_name": "solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.0.2",
+                        "storage_assets_deleted": "false",
                     },
                     {
                         "uuid": "00000000-1234-5678-1234-123456789012",
@@ -114,6 +116,7 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
                         "created_at": "2023-02-01T00:00:00Z",
                         "modified_at": "2023-02-01T00:00:00Z",
                         "job_parent_resource_name": "studies/96642f2a-a72c-11ef-8776-02420a00087d",
+                        "storage_assets_deleted": "true",
                     },
                     {
                         "uuid": "00000000-0000-5678-1234-123456789012",
@@ -123,6 +126,7 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
                         "created_at": "2023-03-01T00:00:00Z",
                         "modified_at": "2023-03-01T00:00:00Z",
                         "job_parent_resource_name": "program/simcore%2Fservices%2Fdynamic%2Fjupyter/releases/5.0.2",
+                        "storage_assets_deleted": "false",
                     },
                 ]
             }

packages/postgres-database/src/simcore_postgres_database/migration/versions/5b998370916a_introduce_data_deleted_in_projects_to_.py

@@ -0,0 +1,40 @@
+"""introduce data_deleted in projects_to_jobs table
+
+Revision ID: 5b998370916a
+Revises: 5679165336c8
+Create Date: 2025-08-11 13:58:38.424398+00:00
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "5b998370916a"
+down_revision = "5679165336c8"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "projects_to_jobs",
+        sa.Column("storage_assets_deleted", sa.Boolean(), nullable=True),
+    )
+
+    op.execute("UPDATE projects_to_jobs SET storage_assets_deleted = false")
+
+    op.alter_column(
+        "projects_to_jobs",
+        "storage_assets_deleted",
+        existing_type=sa.BOOLEAN(),
+        nullable=False,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("projects_to_jobs", "storage_assets_deleted")
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py

@@ -28,6 +28,12 @@
         "the relative resource name is shelves/shelf1/jobs/job2, "
         "the parent resource name is shelves/shelf1.",
     ),
+    sa.Column(
+        "storage_assets_deleted",
+        sa.Boolean,
+        nullable=False,
+        doc="Indicates whether the job's S3 assets have been actively deleted.",
+    ),
     # Composite key (project_uuid, job_parent_resource_name) uniquely identifies very row
     sa.UniqueConstraint(
         "project_uuid",

packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py

@@ -26,6 +26,14 @@
 class WebserverRpcSideEffects:
     # pylint: disable=no-self-use
 
+    def __init__(
+        self,
+        project_job_rpc_get: ProjectJobRpcGet = ProjectJobRpcGet.model_validate(
+            ProjectJobRpcGet.model_json_schema()["examples"][0]
+        ),
+    ):
+        self.project_job_rpc_get = project_job_rpc_get
+
     @validate_call(config={"arbitrary_types_allowed": True})
     async def mark_project_as_job(
         self,
@@ -35,12 +43,14 @@ async def mark_project_as_job(
         user_id: UserID,
         project_uuid: ProjectID,
         job_parent_resource_name: str,
+        storage_assets_deleted: bool,
     ) -> None:
         assert rpc_client
 
         assert not job_parent_resource_name.startswith("/")  # nosec
         assert "/" in job_parent_resource_name  # nosec
         assert not job_parent_resource_name.endswith("/")  # nosec
+        assert isinstance(storage_assets_deleted, bool)
 
         assert product_name
         assert user_id
@@ -84,3 +94,25 @@ async def list_projects_marked_as_jobs(
             limit=limit,
             offset=offset,
         )
+
+    @validate_call(config={"arbitrary_types_allowed": True})
+    async def get_project_marked_as_job(
+        self,
+        rpc_client: RabbitMQRPCClient | MockType,
+        *,
+        product_name: ProductName,
+        user_id: UserID,
+        project_uuid: ProjectID,
+        job_parent_resource_name: str,
+    ) -> ProjectJobRpcGet:
+        assert rpc_client
+        assert product_name
+        assert user_id
+        assert project_uuid
+        assert job_parent_resource_name
+
+        # Return a valid example from the schema
+        _data = self.project_job_rpc_get.model_dump()
+        _data["uuid"] = str(project_uuid)
+        _data["job_parent_resource_name"] = job_parent_resource_name
+        return ProjectJobRpcGet.model_validate(_data)

packages/service-library/src/servicelib/aiohttp/db_asyncpg_engine.py

@@ -38,23 +38,22 @@ def get_async_engine(app: web.Application) -> AsyncEngine:
     return engine
 
 
-async def connect_to_db(app: web.Application, settings: PostgresSettings) -> None:
+async def connect_to_db(
+    app: web.Application, settings: PostgresSettings, application_name: str
+) -> None:
     """
     - db services up, data migrated and ready to use
     - sets an engine in app state (use `get_async_engine(app)` to retrieve)
     """
-    if settings.POSTGRES_CLIENT_NAME:
-        settings = settings.model_copy(
-            update={"POSTGRES_CLIENT_NAME": settings.POSTGRES_CLIENT_NAME + "-asyncpg"}
-        )
-
     with log_context(
         _logger,
         logging.INFO,
         "Connecting app[APP_DB_ASYNC_ENGINE_KEY] to postgres with %s",
         f"{settings=}",
     ):
-        engine = await create_async_engine_and_database_ready(settings)
+        engine = await create_async_engine_and_database_ready(
+            settings, application_name
+        )
         _set_async_engine_to_app_state(app, engine)
 
     _logger.info(

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/container_utils.py renamed to packages/service-library/src/servicelib/container_utils.py

@@ -1,19 +1,35 @@
 import asyncio
 import logging
 from collections.abc import Sequence
-from typing import Any
+from typing import Any, Final
 
 from aiodocker import Docker, DockerError
 from aiodocker.execs import Exec
 from aiodocker.stream import Stream
+from common_library.errors_classes import OsparcErrorMixin
 from pydantic import NonNegativeFloat
-from starlette import status
 
-from ..core.errors import (
-    ContainerExecCommandFailedError,
-    ContainerExecContainerNotFoundError,
-    ContainerExecTimeoutError,
-)
+
+class BaseContainerUtilsError(OsparcErrorMixin, Exception):
+    pass
+
+
+class ContainerExecContainerNotFoundError(BaseContainerUtilsError):
+    msg_template = "Container '{container_name}' was not found"
+
+
+class ContainerExecTimeoutError(BaseContainerUtilsError):
+    msg_template = "Timed out after {timeout} while executing: '{command}'"
+
+
+class ContainerExecCommandFailedError(BaseContainerUtilsError):
+    msg_template = (
+        "Command '{command}' exited with code '{exit_code}'"
+        "and output: '{command_result}'"
+    )
+
+
+_HTTP_404_NOT_FOUND: Final[int] = 404
 
 _logger = logging.getLogger(__name__)
 
@@ -77,10 +93,10 @@ async def run_command_in_container(
             _execute_command(container_name, command), timeout
         )
     except DockerError as e:
-        if e.status == status.HTTP_404_NOT_FOUND:
+        if e.status == _HTTP_404_NOT_FOUND:
             raise ContainerExecContainerNotFoundError(
                 container_name=container_name
             ) from e
         raise
-    except asyncio.TimeoutError as e:
+    except TimeoutError as e:
         raise ContainerExecTimeoutError(timeout=timeout, command=command) from e

packages/service-library/src/servicelib/db_asyncpg_utils.py

@@ -18,7 +18,7 @@
 
 @retry(**PostgresRetryPolicyUponInitialization(_logger).kwargs)
 async def create_async_engine_and_database_ready(
-    settings: PostgresSettings,
+    settings: PostgresSettings, application_name: str
 ) -> AsyncEngine:
     """
     - creates asyncio engine
@@ -31,15 +31,11 @@ async def create_async_engine_and_database_ready(
     )
 
     server_settings = {
-        "jit": "off"
-    }  # see https://docs.sqlalchemy.org/en/20/dialects/postgresql.html#disabling-the-postgresql-jit-to-improve-enum-datatype-handling
-    if settings.POSTGRES_CLIENT_NAME:
-        assert isinstance(settings.POSTGRES_CLIENT_NAME, str)  # nosec
-        server_settings.update(
-            {
-                "application_name": settings.POSTGRES_CLIENT_NAME,
-            }
-        )
+        "jit": "off",
+        "application_name": settings.client_name(
+            f"{application_name}", suffix="asyncpg"
+        ),
+    }
 
     engine = create_async_engine(
         settings.dsn_with_async_sqlalchemy,
@@ -75,7 +71,7 @@ async def check_postgres_liveness(engine: AsyncEngine) -> LivenessResult:
 
 @contextlib.asynccontextmanager
 async def with_async_pg_engine(
-    settings: PostgresSettings,
+    settings: PostgresSettings, *, application_name: str
 ) -> AsyncIterator[AsyncEngine]:
     """
     Creates an asyncpg engine and ensures it is properly closed after use.
@@ -86,9 +82,11 @@ async def with_async_pg_engine(
             logging.DEBUG,
             f"connection to db {settings.dsn_with_async_sqlalchemy}",
         ):
-            server_settings = None
-            if settings.POSTGRES_CLIENT_NAME:
-                assert isinstance(settings.POSTGRES_CLIENT_NAME, str)
+            server_settings = {
+                "application_name": settings.client_name(
+                    application_name, suffix="asyncpg"
+                ),
+            }
 
             engine = create_async_engine(
                 settings.dsn_with_async_sqlalchemy,

packages/service-library/src/servicelib/docker_utils.py

@@ -22,6 +22,7 @@
 )
 from settings_library.docker_registry import RegistrySettings
 from tenacity import (
+    before_sleep_log,
     retry,
     retry_if_exception_type,
     stop_after_attempt,
@@ -275,6 +276,7 @@ def _reset_progress_from_previous_attempt() -> None:
             stop=stop_after_attempt(retry_upon_error_count),
             reraise=True,
             retry=retry_if_exception_type(asyncio.TimeoutError),
+            before_sleep=before_sleep_log(_logger, logging.WARNING),
         )
         async def _pull_image_with_retry() -> None:
             nonlocal attempt

packages/service-library/src/servicelib/fastapi/db_asyncpg_engine.py

@@ -14,7 +14,9 @@
 _logger = logging.getLogger(__name__)
 
 
-async def connect_to_db(app: FastAPI, settings: PostgresSettings) -> None:
+async def connect_to_db(
+    app: FastAPI, settings: PostgresSettings, application_name: str
+) -> None:
     warnings.warn(
         "The 'connect_to_db' function is deprecated and will be removed in a future release. "
         "Please use 'postgres_lifespan' instead for managing the database connection lifecycle.",
@@ -27,7 +29,9 @@ async def connect_to_db(app: FastAPI, settings: PostgresSettings) -> None:
         logging.DEBUG,
         f"Connecting and migraging {settings.dsn_with_async_sqlalchemy}",
     ):
-        engine = await create_async_engine_and_database_ready(settings)
+        engine = await create_async_engine_and_database_ready(
+            settings, application_name
+        )
 
     app.state.engine = engine
     _logger.debug(