👽️ Ensure assets cannot be cleaned when computation running (#8681)

Author: bisgaard-itis

services/api-server/openapi.json

@@ -2880,6 +2880,9 @@
               }
             }
           },
+          "409": {
+            "description": "Job not finished yet"
+          },
           "422": {
             "description": "Validation Error",
             "content": {
@@ -12251,7 +12254,7 @@
             "type": "string",
             "format": "uuid",
             "title": "Key name",
-            "description": "port identifier name.Correponds to the UUID of the parameter/probe node in the study"
+            "description": "port identifier name.Corresponds to the UUID of the parameter/probe node in the study"
           },
           "kind": {
             "type": "string",

services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py

@@ -12,7 +12,10 @@
 from pydantic.types import PositiveInt
 
 from ..._service_jobs import JobService, compose_solver_job_resource_name
-from ...exceptions.backend_errors import ProjectAlreadyStartedError
+from ...exceptions.backend_errors import (
+    ProjectAlreadyStartedError,
+    SolverJobNotStoppedYetError,
+)
 from ...exceptions.service_errors_utils import DEFAULT_BACKEND_SERVICE_STATUS_CODES
 from ...models.basic_types import VersionStr
 from ...models.schemas.errors import ErrorGet
@@ -138,7 +141,8 @@ async def delete_job(
 @router.delete(
     "/{solver_key:path}/releases/{version}/jobs/{job_id:uuid}/assets",
     status_code=status.HTTP_204_NO_CONTENT,
-    responses=JOBS_STATUS_CODES,
+    responses=JOBS_STATUS_CODES
+    | {status.HTTP_409_CONFLICT: {"description": "Job not finished yet"}},
     description=create_route_description(
         base="Deletes assets associated with an existing solver job. N.B. this renders the solver job un-startable",
         changelog=[
@@ -159,6 +163,11 @@ async def delete_job_assets(
         job_parent_resource_name=job_parent_resource_name, job_id=job_id
     )
     assert project_job_rpc_get.uuid == job_id  # nosec
+    job_status = await job_service.inspect_solver_job(
+        solver_key=solver_key, version=version, job_id=job_id
+    )
+    if job_status.stopped_at is None:
+        raise SolverJobNotStoppedYetError(job_id=job_id, state=job_status.state)
 
     await job_service.delete_job_assets(
         job_parent_resource_name=job_parent_resource_name, job_id=job_id

services/api-server/src/simcore_service_api_server/exceptions/backend_errors.py

@@ -1,4 +1,5 @@
 import parse  # type: ignore[import-untyped]
+from common_library.user_messages import user_message
 from fastapi import status
 
 from ._base import ApiServerBaseError
@@ -8,7 +9,9 @@ class BaseBackEndError(ApiServerBaseError):
     """status_code: the default return status which will be returned to the client calling the
     api-server (in case this exception is raised)"""
 
-    msg_template = "The api-server encountered an error when contacting the backend"
+    msg_template = user_message(
+        "An error occurred when contacting the backend service.", _version=1
+    )
     status_code = status.HTTP_502_BAD_GATEWAY
 
     @classmethod
@@ -19,145 +22,190 @@ def named_fields(cls) -> set[str]:
 
 
 class BackendTimeoutError(BaseBackEndError):
-    msg_template = "Backend request timed out"
+    msg_template = user_message("The backend request timed out.", _version=1)
     status_code = status.HTTP_504_GATEWAY_TIMEOUT
 
 
 class InvalidInputError(BaseBackEndError):
-    msg_template = "Invalid input"
+    msg_template = user_message("The provided input is not valid.", _version=1)
     status_code = status.HTTP_422_UNPROCESSABLE_ENTITY
 
 
 class ListSolversOrStudiesError(BaseBackEndError):
-    msg_template = "Cannot list solvers/studies"
+    msg_template = user_message(
+        "Unable to retrieve the list of solvers and projects.", _version=1
+    )
     status_code = status.HTTP_404_NOT_FOUND
 
 
 class ListJobsError(BaseBackEndError):
-    msg_template = "Cannot list jobs"
+    msg_template = user_message("Unable to retrieve the list of jobs.", _version=1)
     status_code = status.HTTP_404_NOT_FOUND
 
 
 class PaymentRequiredError(BaseBackEndError):
-    msg_template = "Payment required"
+    msg_template = user_message("Payment is required to proceed.", _version=1)
     status_code = status.HTTP_402_PAYMENT_REQUIRED
 
 
 class ProfileNotFoundError(BaseBackEndError):
-    msg_template = "Profile not found"
+    msg_template = user_message("The requested profile could not be found.", _version=1)
     status_code = status.HTTP_404_NOT_FOUND
 
 
 class ProgramOrSolverOrStudyNotFoundError(BaseBackEndError):
-    msg_template = "Could not get program/solver/study {name}:{version}"
+    msg_template = user_message(
+        "The program, solver, or project '{name}' version {version} could not be found.",
+        _version=1,
+    )
     status_code = status.HTTP_404_NOT_FOUND
 
 
 class ServiceForbiddenAccessError(BaseBackEndError):
-    msg_template = "Forbidden access to program/solver/study {name}:{version}"
+    msg_template = user_message(
+        "You do not have permission to access the program, solver, or project '{name}' version {version}.",
+        _version=1,
+    )
     status_code = status.HTTP_403_FORBIDDEN
 
 
 class JobForbiddenAccessError(BaseBackEndError):
-    msg_template = "Forbidden access to job {project_id}"
+    msg_template = user_message(
+        "You do not have permission to access job {project_id}.", _version=1
+    )
     status_code = status.HTTP_403_FORBIDDEN
 
 
 class JobNotFoundError(BaseBackEndError):
-    msg_template = "Could not get solver/study job {project_id}"
+    msg_template = user_message(
+        "The solver or project job {project_id} could not be found.", _version=1
+    )
     status_code = status.HTTP_404_NOT_FOUND
 
 
 class LogFileNotFoundError(BaseBackEndError):
-    msg_template = "Could not get logfile for solver/study job {project_id}"
+    msg_template = user_message(
+        "The log file for the solver or project job {project_id} could not be found.",
+        _version=1,
+    )
     status_code = status.HTTP_404_NOT_FOUND
 
 
 class SolverOutputNotFoundError(BaseBackEndError):
-    msg_template = "Solver output of project {project_id} not found"
+    msg_template = user_message(
+        "The output for project {project_id} could not be found.", _version=1
+    )
     status_code = status.HTTP_404_NOT_FOUND
 
 
 class ClusterNotFoundError(BaseBackEndError):
-    msg_template = "Cluster not found"
+    msg_template = user_message("The requested cluster could not be found.", _version=1)
     status_code = status.HTTP_406_NOT_ACCEPTABLE
 
 
 class ConfigurationError(BaseBackEndError):
-    msg_template = "Configuration error"
+    msg_template = user_message("A configuration error occurred.", _version=1)
     status_code = status.HTTP_422_UNPROCESSABLE_ENTITY
 
 
 class ProductPriceNotFoundError(BaseBackEndError):
-    msg_template = "Product price not found"
+    msg_template = user_message("The product price could not be found.", _version=1)
     status_code = status.HTTP_404_NOT_FOUND
 
 
 class WalletNotFoundError(BaseBackEndError):
-    msg_template = "Wallet not found"
+    msg_template = user_message("The requested wallet could not be found.", _version=1)
     status_code = status.HTTP_404_NOT_FOUND
 
 
 class ForbiddenWalletError(BaseBackEndError):
-    msg_template = "User does not have access to wallet"
+    msg_template = user_message(
+        "You do not have permission to access this wallet.", _version=1
+    )
     status_code = status.HTTP_403_FORBIDDEN
 
 
 class ProjectPortsNotFoundError(BaseBackEndError):
-    msg_template = "The ports for the job/study {project_id} could not be found"
+    msg_template = user_message(
+        "The ports for job or project {project_id} could not be found.", _version=1
+    )
     status_code = status.HTTP_404_NOT_FOUND
 
 
 class ProjectMetadataNotFoundError(BaseBackEndError):
-    msg_template = "The metadata for the job/study {project_id} could not be found"
+    msg_template = user_message(
+        "The metadata for job or project {project_id} could not be found.", _version=1
+    )
     status_code = status.HTTP_404_NOT_FOUND
 
 
 class PricingUnitNotFoundError(BaseBackEndError):
-    msg_template = "The pricing unit could not be found"
+    msg_template = user_message("The pricing unit could not be found.", _version=1)
     status_code = status.HTTP_404_NOT_FOUND
 
 
 class PricingPlanNotFoundError(BaseBackEndError):
-    msg_template = "The pricing plan could not be found"
+    msg_template = user_message("The pricing plan could not be found.", _version=1)
     status_code = status.HTTP_404_NOT_FOUND
 
 
 class ProjectAlreadyStartedError(BaseBackEndError):
-    msg_template = "Project already started"
+    msg_template = user_message("The project has already been started.", _version=1)
     status_code = status.HTTP_200_OK
 
 
 class InsufficientNumberOfSeatsError(BaseBackEndError):
-    msg_template = "Not enough available seats for license item {licensed_item_id}"
+    msg_template = user_message(
+        "Not enough available seats for license item {licensed_item_id}.", _version=1
+    )
     status_code = status.HTTP_409_CONFLICT
 
 
 class CanNotCheckoutServiceIsNotRunningError(BaseBackEndError):
-    msg_template = "Can not checkout license item {licensed_item_id} as dynamic service is not running. Current service id: {service_run_id}"
+    msg_template = user_message(
+        "Unable to check out license item {licensed_item_id} because the dynamic service is not running. Current service ID: {service_run_id}.",
+        _version=1,
+    )
     status_code = status.HTTP_422_UNPROCESSABLE_ENTITY
 
 
 class LicensedItemCheckoutNotFoundError(BaseBackEndError):
-    msg_template = "Licensed item checkout {licensed_item_checkout_id} not found."
+    msg_template = user_message(
+        "The license item checkout {licensed_item_checkout_id} could not be found.",
+        _version=1,
+    )
     status_code = status.HTTP_404_NOT_FOUND
 
 
 class JobAssetsMissingError(BaseBackEndError):
-    msg_template = "Job assets missing for job {job_id}"
+    msg_template = user_message("The assets for job {job_id} are missing.", _version=1)
     status_code = status.HTTP_409_CONFLICT
 
 
 class CeleryTaskNotFoundError(BaseBackEndError):
-    msg_template = "Task {task_uuid} not found"
+    msg_template = user_message("Task {task_uuid} could not be found.", _version=1)
     status_code = status.HTTP_404_NOT_FOUND
 
 
 class SolverJobOutputRequestButNotSucceededError(BaseBackEndError):
-    msg_template = "Solver job '{job_id}' not succeeded, when output is requested. Current state: {state}"
+    msg_template = user_message(
+        "Cannot retrieve output for solver job '{job_id}' because it has not completed successfully. Current state: {state}.",
+        _version=1,
+    )
+    status_code = status.HTTP_409_CONFLICT
+
+
+class SolverJobNotStoppedYetError(BaseBackEndError):
+    msg_template = user_message(
+        "Solver job '{job_id}' has not stopped yet. Current status: {state}.",
+        _version=1,
+    )
     status_code = status.HTTP_409_CONFLICT
 
 
 class StudyJobOutputRequestButNotSucceededError(BaseBackEndError):
-    msg_template = "Study job '{job_id}' not succeeded, when output is requested. Current state: {state}"
+    msg_template = user_message(
+        "Cannot retrieve output for project job '{job_id}' because it has not completed successfully. Current state: {state}.",
+        _version=1,
+    )
     status_code = status.HTTP_409_CONFLICT

services/api-server/src/simcore_service_api_server/exceptions/custom_errors.py

@@ -1,3 +1,5 @@
+from common_library.user_messages import user_message
+
 from ._base import ApiServerBaseError
 
 
@@ -7,11 +9,16 @@ class CustomBaseError(ApiServerBaseError):
 
 class InsufficientCreditsError(CustomBaseError):
     # NOTE: Same message as WalletNotEnoughCreditsError
-    msg_template = "Wallet '{wallet_name}' has {wallet_credit_amount} credits. Please add some before requesting solver ouputs"
+    msg_template = user_message(
+        "Wallet '{wallet_name}' has {wallet_credit_amount} credits. Please add credits before requesting solver outputs.",
+        _version=1,
+    )
 
 
 class MissingWalletError(CustomBaseError):
-    msg_template = "Job {job_id} does not have an associated wallet."
+    msg_template = user_message(
+        "Job {job_id} does not have an associated wallet.", _version=1
+    )
 
 
 class ApplicationSetupError(CustomBaseError):
@@ -26,4 +33,7 @@ class SolverServiceListJobsFiltersError(
     ServiceConfigurationError
 ):  # pylint: disable=too-many-ancestors
     service_cls_name = "SolverService"
-    detail_msg = "solver_version is set but solver_id is not. Please provide both or none of them"
+    detail_msg = user_message(
+        "The solver_version parameter is set but solver_id is not. Please provide both parameters or neither.",
+        _version=1,
+    )

services/api-server/src/simcore_service_api_server/exceptions/function_errors.py

@@ -1,3 +1,4 @@
+from common_library.user_messages import user_message
 from fastapi import status
 
 from .backend_errors import BaseBackEndError
@@ -8,10 +9,10 @@ class BaseFunctionBackendError(BaseBackEndError):
 
 
 class FunctionJobCacheNotFoundError(BaseBackEndError):
-    msg_template: str = "No cached function job found."
+    msg_template: str = user_message("No cached function job was found.", _version=1)
     status_code: int = 404  # Not Found
 
 
 class FunctionJobProjectMissingError(BaseBackEndError):
-    msg_template: str = "Could not process function job"
+    msg_template: str = user_message("Unable to process the function job.", _version=1)
     status_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR  # Not Found

services/api-server/src/simcore_service_api_server/exceptions/log_streaming_errors.py

@@ -1,3 +1,5 @@
+from common_library.user_messages import user_message
+
 from ._base import ApiServerBaseError
 
 
@@ -10,4 +12,7 @@ class LogStreamerNotRegisteredError(LogStreamingBaseError):
 
 
 class LogStreamerRegistrationConflictError(LogStreamingBaseError):
-    msg_template = "A stream was already connected to {job_id}. Only a single stream can be connected at the time"
+    msg_template = user_message(
+        "A stream is already connected to {job_id}. Only one stream can be connected at a time.",
+        _version=1,
+    )