From 988004bbc02ba271504d027d1b41179707d1192d Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Mon, 13 Oct 2025 11:34:47 +0200 Subject: [PATCH 01/63] batch cache endpoint --- .../src/models_library/functions.py | 2 +- .../functions/functions_rpc_interface.py | 11 +-- .../rpc_interfaces/webserver/v1/functions.py | 9 +- .../services_rpc/wb_api_server.py | 7 +- .../functions/_controller/_functions_rpc.py | 12 ++- .../functions/_function_jobs_repository.py | 69 +++++++++------ .../functions/_functions_service.py | 86 +++++++++---------- .../test_function_jobs_controller_rpc.py | 85 ++++++++++++++++-- 8 files changed, 189 insertions(+), 92 deletions(-) diff --git a/packages/models-library/src/models_library/functions.py b/packages/models-library/src/models_library/functions.py index f8a0f68d9255..cb300abb117a 100644 --- a/packages/models-library/src/models_library/functions.py +++ b/packages/models-library/src/models_library/functions.py @@ -80,7 +80,7 @@ class FunctionClass(str, Enum): FunctionInputsList: TypeAlias = Annotated[ list[FunctionInputs], - Field(max_length=50), + Field(max_length=50, min_length=1), ] FunctionOutputs: TypeAlias = dict[str, Any] | None diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py index ad4a7295351b..c43610bddb83 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py @@ -21,6 +21,7 @@ from models_library.functions import ( FunctionClass, FunctionGroupAccessRights, + FunctionInputsList, FunctionJobStatus, FunctionOutputs, FunctionUserAccessRights, @@ -512,20 +513,20 @@ async def find_cached_function_jobs( user_id: UserID, product_name: ProductName, function_id: FunctionID, - inputs: FunctionInputs, -) -> list[RegisteredFunctionJob] | None: + inputs: FunctionInputsList, + status_filter: list[FunctionJobStatus] | None = None, +) -> list[RegisteredFunctionJob | None]: result = await rabbitmq_rpc_client.request( DEFAULT_WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("find_cached_function_jobs"), function_id=function_id, inputs=inputs, + status_filter=status_filter, user_id=user_id, product_name=product_name, timeout_s=_FUNCTION_RPC_TIMEOUT_SEC, ) - if result is None: - return None - return TypeAdapter(list[RegisteredFunctionJob]).validate_python(result) + return TypeAdapter(list[RegisteredFunctionJob | None]).validate_python(result) @log_decorator(_logger, level=logging.DEBUG) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py index f96c6fb1952e..ca426f8cb758 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py @@ -20,6 +20,7 @@ from models_library.functions import ( FunctionClass, FunctionGroupAccessRights, + FunctionInputsList, FunctionJobStatus, FunctionOutputs, FunctionUserAccessRights, @@ -462,16 +463,18 @@ async def find_cached_function_jobs( product_name: ProductName, user_id: UserID, function_id: FunctionID, - inputs: FunctionInputs, - ) -> list[RegisteredFunctionJob] | None: + inputs: FunctionInputsList, + status_filter: list[FunctionJobStatus] | None = None, + ) -> list[RegisteredFunctionJob | None]: """Find cached function jobs.""" - return TypeAdapter(list[RegisteredFunctionJob] | None).validate_python( + return TypeAdapter(list[RegisteredFunctionJob | None]).validate_python( await self._request( "find_cached_function_jobs", product_name=product_name, user_id=user_id, function_id=function_id, inputs=inputs, + status_filter=status_filter, ), ) diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index 006419c238b4..345e9c529723 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -24,6 +24,7 @@ ) from models_library.api_schemas_webserver.licensed_items import LicensedItemRpcGetPage from models_library.functions import ( + FunctionInputsList, FunctionJobStatus, FunctionOutputs, FunctionUserAccessRights, @@ -588,13 +589,15 @@ async def find_cached_function_jobs( user_id: UserID, product_name: ProductName, function_id: FunctionID, - inputs: FunctionInputs, - ) -> list[RegisteredFunctionJob] | None: + inputs: FunctionInputsList, + status_filter: list[FunctionJobStatus] | None, + ) -> list[RegisteredFunctionJob | None]: return await self._rpc_client.functions.find_cached_function_jobs( user_id=user_id, product_name=product_name, function_id=function_id, inputs=inputs, + status_filter=status_filter, ) async def get_function_job_collection( diff --git a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py index e5abbd51f39e..8300ec55dc48 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py @@ -7,8 +7,8 @@ FunctionClass, FunctionGroupAccessRights, FunctionID, - FunctionInputs, FunctionInputSchema, + FunctionInputsList, FunctionJob, FunctionJobCollection, FunctionJobCollectionID, @@ -445,15 +445,19 @@ async def find_cached_function_jobs( user_id: UserID, product_name: ProductName, function_id: FunctionID, - inputs: FunctionInputs, -) -> list[RegisteredFunctionJob] | None: - return await _functions_service.find_cached_function_jobs( + inputs: FunctionInputsList, + status_filter: list[FunctionJobStatus] | None, +) -> list[RegisteredFunctionJob | None]: + jobs = await _functions_service.find_cached_function_jobs( app=app, user_id=user_id, product_name=product_name, function_id=function_id, inputs=inputs, + status_filter=status_filter, ) + assert len(jobs) == len(inputs) # nosec + return jobs @router.expose(reraise_if_error_type=(FunctionIDNotFoundError,)) diff --git a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py index 7aeff55cba42..4664a0ee348e 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py @@ -8,6 +8,7 @@ FunctionClass, FunctionID, FunctionInputs, + FunctionInputsList, FunctionJobClassSpecificData, FunctionJobCollectionID, FunctionJobID, @@ -19,7 +20,6 @@ ) from models_library.functions_errors import ( FunctionJobIDNotFoundError, - FunctionJobReadAccessDeniedError, ) from models_library.products import ProductName from models_library.rest_pagination import PageMetaInfoLimitOffset @@ -291,36 +291,55 @@ async def find_cached_function_jobs( user_id: UserID, function_id: FunctionID, product_name: ProductName, - inputs: FunctionInputs, -) -> list[RegisteredFunctionJobDB] | None: + inputs: FunctionInputsList, + status_filter: list[FunctionJobStatus] | None = None, +) -> list[RegisteredFunctionJobDB | None]: async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: - jobs: list[RegisteredFunctionJobDB] = [] - async for row in await conn.stream( - function_jobs_table.select().where( - function_jobs_table.c.function_uuid == function_id, - cast(function_jobs_table.c.inputs, Text) == json.dumps(inputs), + # Get user groups for access check + user_groups = await list_all_user_groups_ids(app, user_id=user_id) + + # Create access subquery + access_subquery = ( + function_jobs_access_rights_table.select() + .with_only_columns(function_jobs_access_rights_table.c.function_job_uuid) + .where( + function_jobs_access_rights_table.c.group_id.in_(user_groups), + function_jobs_access_rights_table.c.product_name == product_name, + function_jobs_access_rights_table.c.read, ) - ): - job = RegisteredFunctionJobDB.model_validate(row) - try: - await check_user_permissions( - app, - connection=conn, - user_id=user_id, - product_name=product_name, - object_id=job.uuid, - object_type="function_job", - permissions=["read"], + ) + + # Create list of JSON dumped inputs for comparison + json_inputs = [json.dumps(inp) for inp in inputs] + + # Build filter conditions + filter_conditions = sqlalchemy.and_( + function_jobs_table.c.function_uuid == function_id, + cast(function_jobs_table.c.inputs, Text).in_(json_inputs), + function_jobs_table.c.uuid.in_(access_subquery), + ( + function_jobs_table.c.status.in_( + [status.status for status in status_filter] ) - except FunctionJobReadAccessDeniedError: - continue + if status_filter is not None + else sqlalchemy.sql.true() + ), + ) - jobs.append(job) + # Single query to find all jobs matching any of the inputs with access check + results = await conn.execute( + function_jobs_table.select().where(filter_conditions) + ) - if len(jobs) > 0: - return jobs + # Create a mapping from JSON inputs to jobs + _ensure_str = lambda x: x if isinstance(x, str) else json.dumps(x) + jobs_by_input: dict[str, RegisteredFunctionJobDB] = { + _ensure_str(row.inputs): RegisteredFunctionJobDB.model_validate(row) + for row in results + } - return None + # Return results in the same order as inputs, with None for missing jobs + return [jobs_by_input.get(json_input, None) for json_input in json_inputs] async def get_function_job_status( diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py index b7749f9e8e1d..bf2da4d381a7 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py @@ -9,8 +9,8 @@ FunctionDB, FunctionGroupAccessRights, FunctionID, - FunctionInputs, FunctionInputSchema, + FunctionInputsList, FunctionJob, FunctionJobClassSpecificData, FunctionJobCollection, @@ -423,62 +423,58 @@ async def find_cached_function_jobs( user_id: UserID, product_name: ProductName, function_id: FunctionID, - inputs: FunctionInputs, -) -> list[RegisteredFunctionJob] | None: + inputs: FunctionInputsList, + status_filter: list[FunctionJobStatus] | None = None, +) -> list[RegisteredFunctionJob | None]: returned_function_jobs = await _function_jobs_repository.find_cached_function_jobs( app=app, user_id=user_id, product_name=product_name, function_id=function_id, inputs=inputs, - ) - if returned_function_jobs is None or len(returned_function_jobs) == 0: - return None - - to_return_function_jobs: list[RegisteredFunctionJob] = [] - for returned_function_job in returned_function_jobs: - if returned_function_job.function_class == FunctionClass.PROJECT: - to_return_function_jobs.append( - RegisteredProjectFunctionJob( - uid=returned_function_job.uuid, - title=returned_function_job.title, - description=returned_function_job.description, - function_uid=returned_function_job.function_uuid, - inputs=returned_function_job.inputs, - outputs=None, - project_job_id=returned_function_job.class_specific_data[ - "project_job_id" - ], - job_creation_task_id=returned_function_job.class_specific_data.get( - "job_creation_task_id" - ), - created_at=returned_function_job.created, - ) + status_filter=status_filter, + ) + assert len(returned_function_jobs) == len(inputs) # nosec + + def _map_db_model_to_domain_model( + job: RegisteredFunctionJobDB | None, + ) -> RegisteredFunctionJob | None: + if job is None: + return None + if job.function_class == FunctionClass.PROJECT: + return RegisteredProjectFunctionJob( + uid=job.uuid, + title=job.title, + description=job.description, + function_uid=job.function_uuid, + inputs=job.inputs, + outputs=None, + project_job_id=job.class_specific_data["project_job_id"], + job_creation_task_id=job.class_specific_data.get( + "job_creation_task_id" + ), + created_at=job.created, ) - elif returned_function_job.function_class == FunctionClass.SOLVER: - to_return_function_jobs.append( - RegisteredSolverFunctionJob( - uid=returned_function_job.uuid, - title=returned_function_job.title, - description=returned_function_job.description, - function_uid=returned_function_job.function_uuid, - inputs=returned_function_job.inputs, - outputs=None, - solver_job_id=returned_function_job.class_specific_data.get( - "solver_job_id" - ), - job_creation_task_id=returned_function_job.class_specific_data.get( - "job_creation_task_id" - ), - created_at=returned_function_job.created, - ) + elif job.function_class == FunctionClass.SOLVER: + return RegisteredSolverFunctionJob( + uid=job.uuid, + title=job.title, + description=job.description, + function_uid=job.function_uuid, + inputs=job.inputs, + outputs=None, + solver_job_id=job.class_specific_data.get("solver_job_id"), + job_creation_task_id=job.class_specific_data.get( + "job_creation_task_id" + ), + created_at=job.created, ) else: raise UnsupportedFunctionJobClassError( - function_job_class=returned_function_job.function_class + function_job_class=job.function_class ) - return to_return_function_jobs + return [_map_db_model_to_domain_model(job) for job in returned_function_jobs] async def get_function_input_schema( diff --git a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py index 0a22743775b5..851e4a6ea414 100644 --- a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py @@ -443,7 +443,7 @@ async def test_find_cached_function_jobs( # Find cached function jobs cached_jobs = await webserver_rpc_client.functions.find_cached_function_jobs( function_id=registered_function.uid, - inputs={"input1": 1}, + inputs=[{"input1": 1}, {"input1": 10}], user_id=logged_user["id"], product_name=osparc_product_name, ) @@ -451,20 +451,91 @@ async def test_find_cached_function_jobs( # Assert the cached jobs contain the registered job assert cached_jobs is not None assert len(cached_jobs) == 2 - assert {job.uid for job in cached_jobs} == { - registered_function_jobs[1].uid, - registered_function_jobs[4].uid, - } + job0 = cached_jobs[0] + assert job0 is not None + assert job0.inputs == {"input1": 1} + assert cached_jobs[1] is None cached_jobs = await webserver_rpc_client.functions.find_cached_function_jobs( function_id=registered_function.uid, - inputs={"input1": 1}, + inputs=[{"input1": 1}, {"input1": 10}], user_id=other_logged_user["id"], product_name=osparc_product_name, ) # Assert the cached jobs does not contain the registered job for the other user - assert cached_jobs is None + assert len(cached_jobs) == 2 + assert all(elm is None for elm in cached_jobs) + + +@pytest.mark.parametrize( + "user_role", + [UserRole.USER], +) +async def test_find_cached_function_jobs_with_status( + client: TestClient, + webserver_rpc_client: WebServerRpcClient, + add_user_function_api_access_rights: None, + logged_user: UserInfoDict, + other_logged_user: UserInfoDict, + osparc_product_name: ProductName, + create_fake_function_obj: Callable[[FunctionClass], Function], + clean_functions: None, +): + # Register the function first + job_statuses = [ + FunctionJobStatus(status="RUNNING"), + FunctionJobStatus(status="FAILED"), + ] + registered_function = await webserver_rpc_client.functions.register_function( + function=create_fake_function_obj(FunctionClass.PROJECT), + user_id=logged_user["id"], + product_name=osparc_product_name, + ) + input = {"input1": 1.0} + + for status in job_statuses: + function_job = ProjectFunctionJob( + function_uid=registered_function.uid, + title="Test Function Job", + description="A test function job", + project_job_id=uuid4(), + inputs=input, + outputs={"output1": "result1"}, + job_creation_task_id=None, + ) + + # Register the function job + registered_job = await webserver_rpc_client.functions.register_function_job( + function_job=function_job, + user_id=logged_user["id"], + product_name=osparc_product_name, + ) + await webserver_rpc_client.functions.update_function_job_status( + user_id=logged_user["id"], + product_name=osparc_product_name, + function_job_id=registered_job.uid, + job_status=status, + ) + + status = job_statuses[0] + cached_jobs = await webserver_rpc_client.functions.find_cached_function_jobs( + function_id=registered_function.uid, + product_name=osparc_product_name, + user_id=logged_user["id"], + inputs=[input], + status_filter=[status], + ) + assert len(cached_jobs) == 1 + cached_job = cached_jobs[0] + assert cached_job is not None + assert cached_job.inputs == input + cached_job_status = await webserver_rpc_client.functions.get_function_job_status( + product_name=osparc_product_name, + function_job_id=cached_job.uid, + user_id=logged_user["id"], + ) + assert status == cached_job_status @pytest.mark.parametrize( From 9924f3db909e654e84b4a34fd55d4d5f2f980d5b Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Mon, 13 Oct 2025 11:48:02 +0200 Subject: [PATCH 02/63] improve sql query --- .../functions/_function_jobs_repository.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py index 4664a0ee348e..343aff0057ce 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py @@ -326,9 +326,15 @@ async def find_cached_function_jobs( ), ) - # Single query to find all jobs matching any of the inputs with access check + # Use DISTINCT ON to get only one job per input (the most recent one) results = await conn.execute( - function_jobs_table.select().where(filter_conditions) + function_jobs_table.select() + .distinct(cast(function_jobs_table.c.inputs, Text)) + .where(filter_conditions) + .order_by( + cast(function_jobs_table.c.inputs, Text), + function_jobs_table.c.created.desc(), + ) ) # Create a mapping from JSON inputs to jobs From db49588d2eebfe59809169bb66a76a11a9e46a01 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Mon, 13 Oct 2025 12:41:32 +0200 Subject: [PATCH 03/63] start using cache --- .../_service_function_jobs_task_client.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py index 80a4af8763ac..fd5a5e16c547 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py @@ -1,5 +1,4 @@ # pylint: disable=too-many-instance-attributes -import contextlib import logging from dataclasses import dataclass @@ -10,6 +9,7 @@ FunctionClass, FunctionID, FunctionInputs, + FunctionInputsList, FunctionJobCollectionID, FunctionJobID, FunctionJobStatus, @@ -329,7 +329,7 @@ async def create_function_job_creation_task( self, *, function: RegisteredFunction, - function_inputs: FunctionInputs, + function_inputs: FunctionInputsList, user_identity: Identity, pricing_spec: JobPricingSpecification | None, job_links: JobLinks, @@ -340,12 +340,13 @@ async def create_function_job_creation_task( function=function, function_inputs=function_inputs ) - # check if results are cached - with contextlib.suppress(FunctionJobCacheNotFoundError): - return await self.get_cached_function_job( - function=function, - job_inputs=job_inputs, - ) + cached_jobs = await self._web_rpc_client.find_cached_function_jobs( + user_id=user_identity.user_id, + product_name=user_identity.product_name, + function_id=function.uid, + inputs=function_inputs, + status_filter=[FunctionJobStatus(status=RunningState.SUCCESS)], + ) pre_registered_function_job_data = ( await self._function_job_service.pre_register_function_job( From 4a954f651cd317ca5bd6fe9acde8c9adabc8799d Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Mon, 13 Oct 2025 13:52:26 +0200 Subject: [PATCH 04/63] implement cache in api-server --- .../_service_function_jobs.py | 2 +- .../_service_function_jobs_task_client.py | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py index e71ef46de027..1644a3bb7b9e 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py @@ -119,7 +119,7 @@ async def validate_function_inputs( f"Unsupported function schema class {function.input_schema.schema_class}", ) - async def create_function_job_inputs( # pylint: disable=no-self-use + def create_function_job_inputs( # pylint: disable=no-self-use self, *, function: RegisteredFunction, diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py index fd5a5e16c547..083ec67d9b3b 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py @@ -30,6 +30,7 @@ from models_library.rest_pagination import PageMetaInfoLimitOffset, PageOffsetInt from models_library.rpc_pagination import PageLimitInt from models_library.users import UserID +from pydantic import TypeAdapter from servicelib.celery.models import ExecutionMetadata, TasksQueue, TaskUUID from servicelib.celery.task_manager import TaskManager from simcore_service_api_server.models.schemas.functions import ( @@ -336,18 +337,27 @@ async def create_function_job_creation_task( parent_project_uuid: ProjectID | None = None, parent_node_id: NodeID | None = None, ) -> RegisteredFunctionJob: - job_inputs = await self._function_job_service.create_function_job_inputs( - function=function, function_inputs=function_inputs - ) + inputs = [ + self._function_job_service.create_function_job_inputs( + function=function, function_inputs=input_ + ) + for input_ in function_inputs + ] cached_jobs = await self._web_rpc_client.find_cached_function_jobs( user_id=user_identity.user_id, product_name=user_identity.product_name, function_id=function.uid, - inputs=function_inputs, + inputs=TypeAdapter(FunctionInputsList).validate_python(inputs), status_filter=[FunctionJobStatus(status=RunningState.SUCCESS)], ) + assert len(cached_jobs) == len(inputs) # nosec + + yet_to_run_inputs = [ + input_ for input_, job in zip(inputs, cached_jobs) if job is None + ] + pre_registered_function_job_data = ( await self._function_job_service.pre_register_function_job( function=function, From d3b935399d23f44ad6a5d8980bd4c6e0a5e82c24 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Mon, 13 Oct 2025 14:13:21 +0200 Subject: [PATCH 05/63] first attempt at batching function job creation --- .../src/models_library/functions.py | 8 +++ .../_service_function_jobs_task_client.py | 2 +- .../functions/_controller/_functions_rpc.py | 4 +- .../functions/_function_jobs_repository.py | 55 ++++++++++--------- .../functions/_functions_service.py | 24 ++++---- 5 files changed, 51 insertions(+), 42 deletions(-) diff --git a/packages/models-library/src/models_library/functions.py b/packages/models-library/src/models_library/functions.py index cb300abb117a..b0fbdd29e51d 100644 --- a/packages/models-library/src/models_library/functions.py +++ b/packages/models-library/src/models_library/functions.py @@ -83,6 +83,7 @@ class FunctionClass(str, Enum): Field(max_length=50, min_length=1), ] + FunctionOutputs: TypeAlias = dict[str, Any] | None FunctionOutputsLogfile: TypeAlias = Any @@ -238,6 +239,9 @@ class RegisteredPythonCodeFunctionJobPatch(BaseModel): ProjectFunctionJob | PythonCodeFunctionJob | SolverFunctionJob, Field(discriminator="function_class"), ] +FunctionJobList: TypeAlias = Annotated[ + list[FunctionJob], Field(max_length=50, min_length=1) +] class RegisteredFunctionJobBase(FunctionJobBase): @@ -263,6 +267,10 @@ class RegisteredPythonCodeFunctionJob(PythonCodeFunctionJob, RegisteredFunctionJ | RegisteredSolverFunctionJob, Field(discriminator="function_class"), ] +RegisteredFunctionJobList: TypeAlias = Annotated[ + list[RegisteredFunctionJob], Field(max_length=50, min_length=1) +] + RegisteredFunctionJobPatch = Annotated[ RegisteredProjectFunctionJobPatch diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py index 083ec67d9b3b..e93f94598cd6 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py @@ -354,7 +354,7 @@ async def create_function_job_creation_task( assert len(cached_jobs) == len(inputs) # nosec - yet_to_run_inputs = [ + uncached_inputs = [ input_ for input_, job in zip(inputs, cached_jobs) if job is None ] diff --git a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py index 8300ec55dc48..be57625a92ec 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py @@ -9,11 +9,11 @@ FunctionID, FunctionInputSchema, FunctionInputsList, - FunctionJob, FunctionJobCollection, FunctionJobCollectionID, FunctionJobCollectionsListFilters, FunctionJobID, + FunctionJobList, FunctionJobStatus, FunctionOutputs, FunctionOutputSchema, @@ -93,7 +93,7 @@ async def register_function_job( *, user_id: UserID, product_name: ProductName, - function_job: FunctionJob, + function_job: FunctionJobList, ) -> RegisteredFunctionJob: return await _functions_service.register_function_job( app=app, user_id=user_id, product_name=product_name, function_job=function_job diff --git a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py index 343aff0057ce..8acc61f5aeb4 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py @@ -5,12 +5,10 @@ import sqlalchemy from aiohttp import web from models_library.functions import ( - FunctionClass, FunctionID, - FunctionInputs, FunctionInputsList, - FunctionJobClassSpecificData, FunctionJobCollectionID, + FunctionJobDB, FunctionJobID, FunctionJobStatus, FunctionOutputs, @@ -55,20 +53,14 @@ ) -async def create_function_job( # noqa: PLR0913 +async def create_function_jobs( # noqa: PLR0913 app: web.Application, connection: AsyncConnection | None = None, *, user_id: UserID, product_name: ProductName, - function_class: FunctionClass, - function_uid: FunctionID, - title: str, - description: str, - inputs: FunctionInputs, - outputs: FunctionOutputs, - class_specific_data: FunctionJobClassSpecificData, -) -> RegisteredFunctionJobDB: + function_jobs: list[FunctionJobDB], +) -> list[RegisteredFunctionJobDB]: async with transaction_context(get_asyncpg_engine(app), connection) as transaction: await check_user_api_access_rights( app, @@ -79,40 +71,51 @@ async def create_function_job( # noqa: PLR0913 FunctionsApiAccessRights.WRITE_FUNCTION_JOBS, ], ) + + # Prepare values for batch insert + values_to_insert = [ + { + "function_uuid": job.function_uuid, + "inputs": job.inputs, + "outputs": job.outputs, + "function_class": job.function_class, + "class_specific_data": job.class_specific_data, + "title": job.title, + "description": job.description, + "status": "created", + } + for job in function_jobs + ] + + # Batch insert all function jobs in a single query result = await transaction.execute( function_jobs_table.insert() - .values( - function_uuid=function_uid, - inputs=inputs, - outputs=outputs, - function_class=function_class, - class_specific_data=class_specific_data, - title=title, - description=description, - status="created", - ) + .values(values_to_insert) .returning(*_FUNCTION_JOBS_TABLE_COLS) ) - row = result.one() - registered_function_job = RegisteredFunctionJobDB.model_validate(row) + # Get all created jobs + created_jobs = [RegisteredFunctionJobDB.model_validate(row) for row in result] + # Get user primary group and set permissions for all jobs user_primary_group_id = await users_service.get_user_primary_group_id( app, user_id=user_id ) + job_uuids = [job.uuid for job in created_jobs] + await _internal_set_group_permissions( app, connection=transaction, permission_group_id=user_primary_group_id, product_name=product_name, object_type="function_job", - object_ids=[registered_function_job.uuid], + object_ids=job_uuids, read=True, write=True, execute=True, ) - return registered_function_job + return created_jobs async def patch_function_job( diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py index bf2da4d381a7..cb427f3ecc61 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py @@ -18,6 +18,7 @@ FunctionJobCollectionsListFilters, FunctionJobDB, FunctionJobID, + FunctionJobList, FunctionJobStatus, FunctionOutputs, FunctionOutputSchema, @@ -29,6 +30,7 @@ RegisteredFunctionJob, RegisteredFunctionJobCollection, RegisteredFunctionJobDB, + RegisteredFunctionJobList, RegisteredFunctionJobPatch, RegisteredFunctionJobWithStatus, RegisteredFunctionJobWithStatusDB, @@ -49,6 +51,7 @@ from models_library.rest_ordering import OrderBy from models_library.rest_pagination import PageMetaInfoLimitOffset from models_library.users import UserID +from pydantic import TypeAdapter from servicelib.rabbitmq import RPCRouter from . import ( @@ -90,22 +93,17 @@ async def register_function_job( *, user_id: UserID, product_name: ProductName, - function_job: FunctionJob, -) -> RegisteredFunctionJob: - encoded_function_job = _encode_functionjob(function_job) - created_function_job_db = await _function_jobs_repository.create_function_job( + function_jobs: FunctionJobList, +) -> RegisteredFunctionJobList: + TypeAdapter(FunctionJobList).validate_python(function_jobs) + encoded_function_jobs = [_encode_functionjob(job) for job in function_jobs] + created_function_jobs_db = await _function_jobs_repository.create_function_jobs( app=app, user_id=user_id, product_name=product_name, - function_class=encoded_function_job.function_class, - title=encoded_function_job.title, - description=encoded_function_job.description, - function_uid=encoded_function_job.function_uuid, - inputs=encoded_function_job.inputs, - outputs=encoded_function_job.outputs, - class_specific_data=encoded_function_job.class_specific_data, - ) - return _decode_functionjob(created_function_job_db) + function_jobs=encoded_function_jobs, + ) + return [_decode_functionjob(job) for job in created_function_jobs_db] async def patch_registered_function_job( From f5bec461fd493ab441817f670ce485a4cec224ca Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Mon, 13 Oct 2025 15:10:43 +0200 Subject: [PATCH 06/63] fix tests using register rpc endpoint --- .../rpc_interfaces/webserver/v1/functions.py | 11 +-- .../functions/_controller/_functions_rpc.py | 7 +- .../functions/_function_jobs_repository.py | 4 +- ...function_job_collections_controller_rpc.py | 84 +++++++++---------- .../test_function_jobs_controller_rpc.py | 62 +++++++++----- 5 files changed, 92 insertions(+), 76 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py index ca426f8cb758..249f96397d38 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py @@ -7,7 +7,6 @@ FunctionID, FunctionInputs, FunctionInputSchema, - FunctionJob, FunctionJobCollection, FunctionJobCollectionID, FunctionJobCollectionsListFilters, @@ -21,10 +20,12 @@ FunctionClass, FunctionGroupAccessRights, FunctionInputsList, + FunctionJobList, FunctionJobStatus, FunctionOutputs, FunctionUserAccessRights, FunctionUserApiAccessRights, + RegisteredFunctionJobList, RegisteredFunctionJobPatch, RegisteredFunctionJobWithStatus, ) @@ -318,15 +319,15 @@ async def register_function_job( *, product_name: ProductName, user_id: UserID, - function_job: FunctionJob, - ) -> RegisteredFunctionJob: + function_jobs: FunctionJobList, + ) -> RegisteredFunctionJobList: """Register a function job.""" - return TypeAdapter(RegisteredFunctionJob).validate_python( + return TypeAdapter(RegisteredFunctionJobList).validate_python( await self._request( "register_function_job", product_name=product_name, user_id=user_id, - function_job=function_job, + function_jobs=function_jobs, ), ) diff --git a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py index be57625a92ec..36b389a6e2e1 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py @@ -22,6 +22,7 @@ RegisteredFunction, RegisteredFunctionJob, RegisteredFunctionJobCollection, + RegisteredFunctionJobList, RegisteredFunctionJobPatch, RegisteredFunctionJobWithStatus, ) @@ -93,10 +94,10 @@ async def register_function_job( *, user_id: UserID, product_name: ProductName, - function_job: FunctionJobList, -) -> RegisteredFunctionJob: + function_jobs: FunctionJobList, +) -> RegisteredFunctionJobList: return await _functions_service.register_function_job( - app=app, user_id=user_id, product_name=product_name, function_job=function_job + app=app, user_id=user_id, product_name=product_name, function_jobs=function_jobs ) diff --git a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py index 8acc61f5aeb4..514148a2aa8b 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py @@ -95,7 +95,9 @@ async def create_function_jobs( # noqa: PLR0913 ) # Get all created jobs - created_jobs = [RegisteredFunctionJobDB.model_validate(row) for row in result] + created_jobs = TypeAdapter(list[RegisteredFunctionJobDB]).validate_python( + list(result) + ) # Get user primary group and set permissions for all jobs user_primary_group_id = await users_service.get_user_primary_group_id( diff --git a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_job_collections_controller_rpc.py b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_job_collections_controller_rpc.py index 32e04bc8936d..4686711dcc8b 100644 --- a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_job_collections_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_job_collections_controller_rpc.py @@ -17,6 +17,7 @@ Function, FunctionClass, FunctionJobCollectionsListFilters, + FunctionJobList, ) from models_library.functions_errors import ( FunctionJobCollectionReadAccessDeniedError, @@ -25,6 +26,7 @@ FunctionJobIDNotFoundError, ) from models_library.products import ProductName +from pydantic import TypeAdapter from pytest_simcore.helpers.webserver_users import UserInfoDict from servicelib.rabbitmq.rpc_interfaces.webserver.v1 import WebServerRpcClient @@ -53,19 +55,9 @@ async def test_function_job_collection( ) assert registered_function.uid is not None - registered_function_job = ProjectFunctionJob( - function_uid=registered_function.uid, - title="Test Function Job", - description="A test function job", - project_job_id=uuid4(), - inputs={"input1": "value1"}, - outputs={"output1": "result1"}, - job_creation_task_id=None, - ) - # Register the function job - function_job_ids = [] - for _ in range(3): - registered_function_job = ProjectFunctionJob( + # Register the function jobs + function_jobs = [ + ProjectFunctionJob( function_uid=registered_function.uid, title="Test Function Job", description="A test function job", @@ -74,14 +66,17 @@ async def test_function_job_collection( outputs={"output1": "result1"}, job_creation_task_id=None, ) - # Register the function job - registered_job = await webserver_rpc_client.functions.register_function_job( - function_job=registered_function_job, - user_id=logged_user["id"], - product_name=osparc_product_name, - ) - assert registered_job.uid is not None - function_job_ids.append(registered_job.uid) + for _ in range(3) + ] + # Register the function jobs + registered_jobs = await webserver_rpc_client.functions.register_function_job( + function_jobs=TypeAdapter(FunctionJobList).validate_python(function_jobs), + user_id=logged_user["id"], + product_name=osparc_product_name, + ) + assert len(registered_jobs) == 3 + assert all(job.uid is not None for job in registered_jobs) + function_job_ids = [job.uid for job in registered_jobs] function_job_collection = FunctionJobCollection( title="Test Function Job Collection", @@ -261,9 +256,8 @@ async def test_list_function_job_collections( assert registered_function.uid is not None # Create a function job collection - function_job_ids = [] - for _ in range(3): - registered_function_job = ProjectFunctionJob( + function_jobs = [ + ProjectFunctionJob( function_uid=registered_function.uid, title="Test Function Job", description="A test function job", @@ -272,19 +266,20 @@ async def test_list_function_job_collections( outputs={"output1": "result1"}, job_creation_task_id=None, ) - # Register the function job - registered_job = await webserver_rpc_client.functions.register_function_job( - function_job=registered_function_job, - user_id=logged_user["id"], - product_name=osparc_product_name, - ) - assert registered_job.uid is not None - function_job_ids.append(registered_job.uid) + for _ in range(3) + ] + # Register the function jobs + registered_jobs = await webserver_rpc_client.functions.register_function_job( + function_jobs=TypeAdapter(FunctionJobList).validate_python(function_jobs), + user_id=logged_user["id"], + product_name=osparc_product_name, + ) + assert all(job.uid is not None for job in registered_jobs) function_job_collection = FunctionJobCollection( title="Test Function Job Collection", description="A test function job collection", - job_ids=function_job_ids, + job_ids=[job.uid for job in registered_jobs], ) # Register the function job collection @@ -357,9 +352,8 @@ async def test_list_function_job_collections_filtered_function_id( else: function_id = other_registered_function.uid # Create a function job collection - function_job_ids = [] - for _ in range(3): - registered_function_job = ProjectFunctionJob( + function_jobs = [ + ProjectFunctionJob( function_uid=function_id, title="Test Function Job", description="A test function job", @@ -368,14 +362,16 @@ async def test_list_function_job_collections_filtered_function_id( outputs={"output1": "result1"}, job_creation_task_id=None, ) - # Register the function job - registered_job = await webserver_rpc_client.functions.register_function_job( - function_job=registered_function_job, - user_id=logged_user["id"], - product_name=osparc_product_name, - ) - assert registered_job.uid is not None - function_job_ids.append(registered_job.uid) + for _ in range(3) + ] + # Register the function job + registered_jobs = await webserver_rpc_client.functions.register_function_job( + function_jobs=TypeAdapter(FunctionJobList).validate_python(function_jobs), + user_id=logged_user["id"], + product_name=osparc_product_name, + ) + assert all(job.uid for job in registered_jobs) + function_job_ids = [job.uid for job in registered_jobs] function_job_collection = FunctionJobCollection( title="Test Function Job Collection", diff --git a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py index 851e4a6ea414..656a177b18f6 100644 --- a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py @@ -73,11 +73,13 @@ async def test_register_get_delete_function_job( ) # Register the function job - registered_job = await webserver_rpc_client.functions.register_function_job( - function_job=function_job, + registered_jobs = await webserver_rpc_client.functions.register_function_job( + function_jobs=[function_job], user_id=logged_user["id"], product_name=osparc_product_name, ) + assert len(registered_jobs) == 1 + registered_job = registered_jobs[0] # Assert the registered job matches the input job assert registered_job.function_uid == function_job.function_uid @@ -191,11 +193,13 @@ async def test_list_function_jobs( ) # Register the function job - registered_job = await webserver_rpc_client.functions.register_function_job( - function_job=function_job, + registered_jobs = await webserver_rpc_client.functions.register_function_job( + function_jobs=[function_job], user_id=logged_user["id"], product_name=osparc_product_name, ) + assert len(registered_jobs) == 1 + registered_job = registered_jobs[0] # List function jobs jobs, _ = await webserver_rpc_client.functions.list_function_jobs( @@ -241,11 +245,13 @@ async def test_list_function_jobs_with_status( ) # Register the function job - registered_job = await webserver_rpc_client.functions.register_function_job( - function_job=function_job, + registered_jobs = await webserver_rpc_client.functions.register_function_job( + function_jobs=[function_job], user_id=logged_user["id"], product_name=osparc_product_name, ) + assert len(registered_jobs) == 1 + registered_job = registered_jobs[0] # List function jobs jobs, _ = await webserver_rpc_client.functions.list_function_jobs_with_status( @@ -299,9 +305,9 @@ async def test_list_function_jobs_filtering( job_creation_task_id=None, ) # Register the function job - first_registered_function_jobs.append( + first_registered_function_jobs += ( await webserver_rpc_client.functions.register_function_job( - function_job=function_job, + function_jobs=[function_job], user_id=logged_user["id"], product_name=osparc_product_name, ) @@ -317,9 +323,9 @@ async def test_list_function_jobs_filtering( job_creation_task_id=None, ) # Register the function job - second_registered_function_jobs.append( + second_registered_function_jobs += ( await webserver_rpc_client.functions.register_function_job( - function_job=function_job, + function_jobs=[function_job], user_id=logged_user["id"], product_name=osparc_product_name, ) @@ -433,12 +439,12 @@ async def test_find_cached_function_jobs( ) # Register the function job - registered_job = await webserver_rpc_client.functions.register_function_job( - function_job=function_job, + registered_jobs = await webserver_rpc_client.functions.register_function_job( + function_jobs=[function_job], user_id=logged_user["id"], product_name=osparc_product_name, ) - registered_function_jobs.append(registered_job) + registered_function_jobs += registered_jobs # Find cached function jobs cached_jobs = await webserver_rpc_client.functions.find_cached_function_jobs( @@ -506,11 +512,13 @@ async def test_find_cached_function_jobs_with_status( ) # Register the function job - registered_job = await webserver_rpc_client.functions.register_function_job( - function_job=function_job, + registered_jobs = await webserver_rpc_client.functions.register_function_job( + function_jobs=[function_job], user_id=logged_user["id"], product_name=osparc_product_name, ) + assert len(registered_jobs) == 1 + registered_job = registered_jobs[0] await webserver_rpc_client.functions.update_function_job_status( user_id=logged_user["id"], product_name=osparc_product_name, @@ -607,11 +615,13 @@ async def test_patch_registered_function_jobs( # Register the function job function_job.function_uid = registered_function.uid - registered_job = await webserver_rpc_client.functions.register_function_job( - function_job=function_job, + registered_jobs = await webserver_rpc_client.functions.register_function_job( + function_jobs=[function_job], user_id=logged_user["id"], product_name=osparc_product_name, ) + assert len(registered_jobs) == 1 + registered_job = registered_jobs[0] registered_job = await webserver_rpc_client.functions.patch_registered_function_job( user_id=logged_user["id"], @@ -681,11 +691,13 @@ async def test_incompatible_patch_model_error( product_name=osparc_product_name, ) function_job.function_uid = registered_function.uid - registered_job = await webserver_rpc_client.functions.register_function_job( - function_job=function_job, + registered_jobs = await webserver_rpc_client.functions.register_function_job( + function_jobs=[function_job], user_id=logged_user["id"], product_name=osparc_product_name, ) + assert len(registered_jobs) == 1 + registered_job = registered_jobs[0] with pytest.raises(FunctionJobPatchModelIncompatibleError): registered_job = ( await webserver_rpc_client.functions.patch_registered_function_job( @@ -740,11 +752,13 @@ async def test_update_function_job_status_output( ) # Register the function job - registered_job = await webserver_rpc_client.functions.register_function_job( - function_job=function_job, + registered_jobs = await webserver_rpc_client.functions.register_function_job( + function_jobs=[function_job], user_id=logged_user["id"], product_name=osparc_product_name, ) + assert len(registered_jobs) == 1 + registered_job = registered_jobs[0] old_job_status = await webserver_rpc_client.functions.get_function_job_status( function_job_id=registered_job.uid, @@ -830,11 +844,13 @@ async def test_update_function_job_outputs( ) # Register the function job - registered_job = await webserver_rpc_client.functions.register_function_job( - function_job=function_job, + registered_jobs = await webserver_rpc_client.functions.register_function_job( + function_jobs=[function_job], user_id=logged_user["id"], product_name=osparc_product_name, ) + assert len(registered_jobs) == 1 + registered_job = registered_jobs[0] received_outputs = await webserver_rpc_client.functions.get_function_job_outputs( function_job_id=registered_job.uid, From 366c4b634b8e9fc9a3cf8e7e66f166a11846fd21 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Mon, 13 Oct 2025 15:41:39 +0200 Subject: [PATCH 07/63] propagate usage of preregistration function to api-server --- .../_service_function_jobs.py | 61 ++++++++++++------- .../_service_function_jobs_task_client.py | 40 ++++++------ .../api/routes/functions_routes.py | 3 +- .../services_rpc/wb_api_server.py | 13 ++-- 4 files changed, 70 insertions(+), 47 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py index 1644a3bb7b9e..6728b8bc44a5 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py @@ -9,6 +9,7 @@ FunctionInputs, FunctionJobCollectionID, FunctionJobID, + FunctionJobList, FunctionSchemaClass, ProjectFunctionJob, RegisteredFunction, @@ -30,7 +31,7 @@ from models_library.rest_pagination import PageMetaInfoLimitOffset, PageOffsetInt from models_library.rpc_pagination import PageLimitInt from models_library.users import UserID -from pydantic import ValidationError +from pydantic import TypeAdapter, ValidationError from simcore_service_api_server._service_functions import FunctionService from simcore_service_api_server.services_rpc.storage import StorageService @@ -91,13 +92,8 @@ async def list_function_jobs( ) async def validate_function_inputs( - self, *, function_id: FunctionID, inputs: FunctionInputs + self, *, function: RegisteredFunction, job_inputs: list[JobInputs] ) -> tuple[bool, str]: - function = await self._web_rpc_client.get_function( - function_id=function_id, - user_id=self.user_id, - product_name=self.product_name, - ) if ( function.input_schema is None @@ -107,8 +103,12 @@ async def validate_function_inputs( if function.input_schema.schema_class == FunctionSchemaClass.json_schema: try: - jsonschema.validate( - instance=inputs, schema=function.input_schema.schema_content + all( + jsonschema.validate( + instance=input.values, + schema=function.input_schema.schema_content, + ) + for input in job_inputs ) except ValidationError as err: return False, str(err) @@ -137,42 +137,54 @@ async def pre_register_function_job( self, *, function: RegisteredFunction, - job_inputs: JobInputs, - ) -> PreRegisteredFunctionJobData: + job_inputs: list[JobInputs], + ) -> list[PreRegisteredFunctionJobData]: if function.input_schema is not None: is_valid, validation_str = await self.validate_function_inputs( - function_id=function.uid, - inputs=job_inputs.values, + function=function, + job_inputs=job_inputs, ) if not is_valid: raise FunctionInputsValidationError(error=validation_str) if function.function_class == FunctionClass.PROJECT: - job = await self._web_rpc_client.register_function_job( - function_job=ProjectFunctionJob( + function_jobs = [ + ProjectFunctionJob( function_uid=function.uid, title=f"Function job of function {function.uid}", description=function.description, - inputs=job_inputs.values, + inputs=input_.values, outputs=None, project_job_id=None, job_creation_task_id=None, + ) + for input_ in job_inputs + ] + jobs = await self._web_rpc_client.register_function_job( + function_jobs=TypeAdapter(FunctionJobList).validate_python( + function_jobs ), user_id=self.user_id, product_name=self.product_name, ) elif function.function_class == FunctionClass.SOLVER: - job = await self._web_rpc_client.register_function_job( - function_job=SolverFunctionJob( + function_jobs = [ + SolverFunctionJob( function_uid=function.uid, title=f"Function job of function {function.uid}", description=function.description, - inputs=job_inputs.values, + inputs=input_.values, outputs=None, solver_job_id=None, job_creation_task_id=None, + ) + for input_ in job_inputs + ] + jobs = await self._web_rpc_client.register_function_job( + function_jobs=TypeAdapter(FunctionJobList).validate_python( + function_jobs ), user_id=self.user_id, product_name=self.product_name, @@ -182,10 +194,13 @@ async def pre_register_function_job( function_class=function.function_class, ) - return PreRegisteredFunctionJobData( - function_job_id=job.uid, - job_inputs=job_inputs, - ) + return [ + PreRegisteredFunctionJobData( + function_job_id=job.uid, + job_inputs=input_, + ) + for job, input_ in zip(jobs, job_inputs) + ] @overload async def patch_registered_function_job( diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py index e93f94598cd6..075c20a5544f 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py @@ -358,10 +358,10 @@ async def create_function_job_creation_task( input_ for input_, job in zip(inputs, cached_jobs) if job is None ] - pre_registered_function_job_data = ( + pre_registered_function_job_data_list = ( await self._function_job_service.pre_register_function_job( function=function, - job_inputs=job_inputs, + job_inputs=uncached_inputs, ) ) @@ -370,27 +370,29 @@ async def create_function_job_creation_task( owner_metadata = ApiServerOwnerMetadata( user_id=user_identity.user_id, product_name=user_identity.product_name ) - - task_uuid = await self._celery_task_manager.submit_task( - ExecutionMetadata( - name="run_function", - ephemeral=False, - queue=TasksQueue.API_WORKER_QUEUE, - ), - owner_metadata=owner_metadata, - user_identity=user_identity, - function=function, - pre_registered_function_job_data=pre_registered_function_job_data, - pricing_spec=pricing_spec, - job_links=job_links, - x_simcore_parent_project_uuid=parent_project_uuid, - x_simcore_parent_node_id=parent_node_id, - ) + task_uuids = [ + await self._celery_task_manager.submit_task( + ExecutionMetadata( + name="run_function", + ephemeral=False, + queue=TasksQueue.API_WORKER_QUEUE, + ), + owner_metadata=owner_metadata, + user_identity=user_identity, + function=function, + pre_registered_function_job_data=pre_registered_function_job_data, + pricing_spec=pricing_spec, + job_links=job_links, + x_simcore_parent_project_uuid=parent_project_uuid, + x_simcore_parent_node_id=parent_node_id, + ) + for pre_registered_function_job_data in pre_registered_function_job_data_list + ] return await self._function_job_service.patch_registered_function_job( user_id=user_identity.user_id, product_name=user_identity.product_name, - function_job_id=pre_registered_function_job_data.function_job_id, + function_job_id=pre_registered_function_job_data_list.function_job_id, function_class=function.function_class, job_creation_task_id=TaskID(task_uuid), ) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index f009a0079f57..38935d3b08e0 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -299,12 +299,13 @@ async def get_function_outputschema( async def validate_function_inputs( function_id: FunctionID, inputs: FunctionInputs, + function: Annotated[RegisteredFunction, Depends(get_function)], function_job_service: Annotated[ FunctionJobService, Depends(get_function_job_service) ], ) -> tuple[bool, str]: return await function_job_service.validate_function_inputs( - function_id=function_id, + function=function, inputs=inputs, ) diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index 345e9c529723..7c7af65f826b 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -12,7 +12,6 @@ FunctionID, FunctionInputs, FunctionInputSchema, - FunctionJob, FunctionJobCollection, FunctionJobCollectionID, FunctionJobCollectionsListFilters, @@ -25,10 +24,12 @@ from models_library.api_schemas_webserver.licensed_items import LicensedItemRpcGetPage from models_library.functions import ( FunctionInputsList, + FunctionJobList, FunctionJobStatus, FunctionOutputs, FunctionUserAccessRights, FunctionUserApiAccessRights, + RegisteredFunctionJobList, RegisteredFunctionJobPatch, RegisteredFunctionJobWithStatus, ) @@ -482,12 +483,16 @@ async def delete_function_job( ) async def register_function_job( - self, *, user_id: UserID, function_job: FunctionJob, product_name: ProductName - ) -> RegisteredFunctionJob: + self, + *, + user_id: UserID, + function_jobs: FunctionJobList, + product_name: ProductName, + ) -> RegisteredFunctionJobList: return await self._rpc_client.functions.register_function_job( user_id=user_id, product_name=product_name, - function_job=function_job, + function_jobs=function_jobs, ) async def patch_registered_function_job( From 2c8f63ba07ca0a0ba2e172114c3df913a2416d2d Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 14 Oct 2025 11:32:13 +0200 Subject: [PATCH 08/63] first attempt at implementing batch patch function job method --- .../functions/_function_jobs_repository.py | 98 ++++++++++++++++--- 1 file changed, 85 insertions(+), 13 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py index 514148a2aa8b..83dfaf4aa7ae 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py @@ -5,6 +5,7 @@ import sqlalchemy from aiohttp import web from models_library.functions import ( + FunctionClass, FunctionID, FunctionInputsList, FunctionJobCollectionID, @@ -14,10 +15,12 @@ FunctionOutputs, FunctionsApiAccessRights, RegisteredFunctionJobDB, + RegisteredFunctionJobPatchInput, RegisteredFunctionJobWithStatusDB, ) from models_library.functions_errors import ( FunctionJobIDNotFoundError, + UnsupportedFunctionJobClassError, ) from models_library.products import ProductName from models_library.rest_pagination import PageMetaInfoLimitOffset @@ -36,7 +39,7 @@ pass_or_acquire_connection, transaction_context, ) -from sqlalchemy import Text, cast +from sqlalchemy import Text, case, cast, func from sqlalchemy.ext.asyncio import AsyncConnection from sqlalchemy.sql import func @@ -126,8 +129,8 @@ async def patch_function_job( *, user_id: UserID, product_name: ProductName, - registered_function_job_db: RegisteredFunctionJobDB, -) -> RegisteredFunctionJobDB: + registered_function_job_patch_inputs: list[RegisteredFunctionJobPatchInput], +) -> list[RegisteredFunctionJobDB]: async with transaction_context(get_asyncpg_engine(app), connection) as transaction: await check_user_api_access_rights( @@ -139,23 +142,92 @@ async def patch_function_job( FunctionsApiAccessRights.WRITE_FUNCTION_JOBS, ], ) + + case_inputs = case(value=function_jobs_table.c.uuid) + case_outputs = case(value=function_jobs_table.c.uuid) + case_class_specific_data = case(value=function_jobs_table.c.uuid) + case_title = case(value=function_jobs_table.c.uuid) + case_description = case(value=function_jobs_table.c.uuid) + + function_job_uids = [] + for patch in registered_function_job_patch_inputs: + function_job_uids.append(patch.uid) + if patch.patch.inputs is not None: + case_inputs = case_inputs.when(patch.uid, patch.patch.inputs) + if patch.patch.outputs is not None: + case_outputs = case_outputs.when(patch.uid, patch.patch.outputs) + if patch.patch.title is not None: + case_title = case_title.when(patch.uid, patch.patch.title) + if patch.patch.description is not None: + case_description = case_description.when( + patch.uid, patch.patch.description + ) + + # patch class specific data + if patch.patch.function_class == FunctionClass.PROJECT: + if patch.patch.project_job_id is not None: + case_class_specific_data = case_class_specific_data.when( + patch.uid, + func.jsonb_set( + function_jobs_table.c.class_specific_data, + "{project_job_id}", + f'"{patch.patch.project_job_id}"', + ), + ) + if patch.patch.job_creation_task_id is not None: + case_class_specific_data = case_class_specific_data.when( + patch.uid, + func.jsonb_set( + function_jobs_table.c.class_specific_data, + "{job_creation_task_id}", + f'"{patch.patch.job_creation_task_id}"', + ), + ) + elif patch.patch.function_class == FunctionClass.SOLVER: + if patch.patch.solver_job_id is not None: + case_class_specific_data = case_class_specific_data.when( + patch.uid, + func.jsonb_set( + function_jobs_table.c.class_specific_data, + "{solver_job_id}", + f'"{patch.patch.solver_job_id}"', + ), + ) + if patch.patch.job_creation_task_id is not None: + case_class_specific_data = case_class_specific_data.when( + patch.uid, + func.jsonb_set( + function_jobs_table.c.class_specific_data, + "{job_creation_task_id}", + f'"{patch.patch.job_creation_task_id}"', + ), + ) + else: + raise UnsupportedFunctionJobClassError( + function_job_class=patch.patch.function_class + ) + result = await transaction.execute( function_jobs_table.update() - .where(function_jobs_table.c.uuid == f"{registered_function_job_db.uuid}") .values( - inputs=registered_function_job_db.inputs, - outputs=registered_function_job_db.outputs, - function_class=registered_function_job_db.function_class, - class_specific_data=registered_function_job_db.class_specific_data, - title=registered_function_job_db.title, - description=registered_function_job_db.description, - status="created", + inputs=case_inputs, + outputs=case_outputs, + title=case_title, + description=case_description, + class_specific_data=case_class_specific_data, ) + .where(function_jobs_table.c.uuid.in_(function_job_uids)) .returning(*_FUNCTION_JOBS_TABLE_COLS) ) - row = result.one() + jobs = { + row.uuid: RegisteredFunctionJobDB.model_validate(row) + for row in result.fetchall() + } + assert {patch.uid for patch in registered_function_job_patch_inputs} == set( + jobs.keys() + ) - return RegisteredFunctionJobDB.model_validate(row) + return [jobs[patch.uid] for patch in registered_function_job_patch_inputs] async def list_function_jobs_with_status( From 308f8538e7c940130a8378024ceb03664a171e26 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 14 Oct 2025 11:57:58 +0200 Subject: [PATCH 09/63] update models --- .../src/models_library/functions.py | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/packages/models-library/src/models_library/functions.py b/packages/models-library/src/models_library/functions.py index b0fbdd29e51d..6d38eba121e5 100644 --- a/packages/models-library/src/models_library/functions.py +++ b/packages/models-library/src/models_library/functions.py @@ -1,7 +1,7 @@ import datetime from collections.abc import Mapping from enum import Enum -from typing import Annotated, Any, Literal, TypeAlias +from typing import Annotated, Any, Final, Literal, TypeAlias from uuid import UUID from models_library import projects @@ -23,6 +23,8 @@ FileID: TypeAlias = UUID InputTypes: TypeAlias = FileID | float | int | bool | str | list +_MIN_LIST_LENGTH: Final[int] = 1 +_MAX_LIST_LENGTH: Final[int] = 50 class FunctionSchemaClass(str, Enum): @@ -80,7 +82,7 @@ class FunctionClass(str, Enum): FunctionInputsList: TypeAlias = Annotated[ list[FunctionInputs], - Field(max_length=50, min_length=1), + Field(max_length=_MAX_LIST_LENGTH, min_length=_MIN_LIST_LENGTH), ] @@ -240,7 +242,7 @@ class RegisteredPythonCodeFunctionJobPatch(BaseModel): Field(discriminator="function_class"), ] FunctionJobList: TypeAlias = Annotated[ - list[FunctionJob], Field(max_length=50, min_length=1) + list[FunctionJob], Field(max_length=_MAX_LIST_LENGTH, min_length=_MIN_LIST_LENGTH) ] @@ -268,7 +270,8 @@ class RegisteredPythonCodeFunctionJob(PythonCodeFunctionJob, RegisteredFunctionJ Field(discriminator="function_class"), ] RegisteredFunctionJobList: TypeAlias = Annotated[ - list[RegisteredFunctionJob], Field(max_length=50, min_length=1) + list[RegisteredFunctionJob], + Field(max_length=_MAX_LIST_LENGTH, min_length=_MIN_LIST_LENGTH), ] @@ -280,6 +283,28 @@ class RegisteredPythonCodeFunctionJob(PythonCodeFunctionJob, RegisteredFunctionJ ] +class RegisteredProjectFunctionJobPatchInput(BaseModel): + uid: FunctionJobID + patch: RegisteredProjectFunctionJobPatch + + +RegisteredProjectFunctionJobPatchInputList: TypeAlias = Annotated[ + list[RegisteredProjectFunctionJobPatchInput], + Field(max_length=_MAX_LIST_LENGTH, min_length=_MIN_LIST_LENGTH), +] + + +class RegisteredSolverFunctionJobPatchInput(BaseModel): + uid: FunctionJobID + patch: RegisteredSolverFunctionJobPatch + + +RegisteredSolverFunctionJobPatchInputList: TypeAlias = Annotated[ + list[RegisteredSolverFunctionJobPatchInput], + Field(max_length=_MAX_LIST_LENGTH, min_length=_MIN_LIST_LENGTH), +] + + class FunctionJobStatus(BaseModel): status: str From cb9d1389ef3bd8ed81845e69284d75584c1e4b4e Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 14 Oct 2025 12:28:53 +0200 Subject: [PATCH 10/63] ensure meaningful error message is raised --- .../functions/_function_jobs_repository.py | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py index 83dfaf4aa7ae..23bed141b497 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py @@ -15,11 +15,13 @@ FunctionOutputs, FunctionsApiAccessRights, RegisteredFunctionJobDB, - RegisteredFunctionJobPatchInput, RegisteredFunctionJobWithStatusDB, + RegisteredProjectFunctionJobPatchInputList, + RegisteredSolverFunctionJobPatchInputList, ) from models_library.functions_errors import ( FunctionJobIDNotFoundError, + FunctionJobPatchModelIncompatibleError, UnsupportedFunctionJobClassError, ) from models_library.products import ProductName @@ -129,9 +131,19 @@ async def patch_function_job( *, user_id: UserID, product_name: ProductName, - registered_function_job_patch_inputs: list[RegisteredFunctionJobPatchInput], + registered_function_job_patch_inputs: ( + RegisteredProjectFunctionJobPatchInputList + | RegisteredSolverFunctionJobPatchInputList + ), ) -> list[RegisteredFunctionJobDB]: + # check only a single function class is used + TypeAdapter( + RegisteredProjectFunctionJobPatchInputList + | RegisteredSolverFunctionJobPatchInputList + ).validate_python(registered_function_job_patch_inputs) + used_function_class = registered_function_job_patch_inputs[0].patch.function_class + async with transaction_context(get_asyncpg_engine(app), connection) as transaction: await check_user_api_access_rights( app, @@ -217,15 +229,32 @@ async def patch_function_job( class_specific_data=case_class_specific_data, ) .where(function_jobs_table.c.uuid.in_(function_job_uids)) + .where(function_jobs_table.c.function_class == used_function_class) .returning(*_FUNCTION_JOBS_TABLE_COLS) ) jobs = { row.uuid: RegisteredFunctionJobDB.model_validate(row) for row in result.fetchall() } - assert {patch.uid for patch in registered_function_job_patch_inputs} == set( - jobs.keys() - ) + if set(function_job_uids) != set(jobs.keys()): + # ensure meaningful error message is raised + missing_uids = { + patch.uid for patch in registered_function_job_patch_inputs + } - set(jobs.keys()) + missing_uid = missing_uids.pop() + function_job = await get_function_job( + app, + user_id=user_id, + product_name=product_name, + function_job_id=missing_uid, + ) + if function_job.function_class != used_function_class: + raise FunctionJobPatchModelIncompatibleError( + function_id=missing_uid, + product_name=product_name, + ) + # reaching this far means the uid is missing for other reasons + raise FunctionJobIDNotFoundError(function_job_id=missing_uid) return [jobs[patch.uid] for patch in registered_function_job_patch_inputs] From 91eda8d93823503126fd7f2776219f7a416d95a6 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 14 Oct 2025 12:29:26 +0200 Subject: [PATCH 11/63] small fix --- .../functions/_function_jobs_repository.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py index 23bed141b497..cff4b94ccf09 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py @@ -238,9 +238,7 @@ async def patch_function_job( } if set(function_job_uids) != set(jobs.keys()): # ensure meaningful error message is raised - missing_uids = { - patch.uid for patch in registered_function_job_patch_inputs - } - set(jobs.keys()) + missing_uids = set(function_job_uids) - set(jobs.keys()) missing_uid = missing_uids.pop() function_job = await get_function_job( app, From d11dd51401b20b195746e02a4d8bfaee10442f52 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 14 Oct 2025 12:42:01 +0200 Subject: [PATCH 12/63] add exception in case of unrecoverable error --- .../src/models_library/functions_errors.py | 5 +++++ .../functions/_function_jobs_repository.py | 20 +++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/packages/models-library/src/models_library/functions_errors.py b/packages/models-library/src/models_library/functions_errors.py index 629c3dc1c7e9..1f2762f0d4c2 100644 --- a/packages/models-library/src/models_library/functions_errors.py +++ b/packages/models-library/src/models_library/functions_errors.py @@ -170,3 +170,8 @@ class FunctionJobCollectionsExecuteApiAccessDeniedError(FunctionBaseError): class FunctionJobPatchModelIncompatibleError(FunctionBaseError): msg_template = "Incompatible patch model for Function '{function_id}' in product '{product_name}'." status_code: int = 422 + + +class FunctionUnrecoverableError(FunctionBaseError): + msg_template = "Unrecoverable error." + status_code: int = 500 diff --git a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py index cff4b94ccf09..560ee8b54cd1 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py @@ -1,9 +1,11 @@ # pylint: disable=too-many-arguments import json +import logging import sqlalchemy from aiohttp import web +from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from models_library.functions import ( FunctionClass, FunctionID, @@ -22,6 +24,7 @@ from models_library.functions_errors import ( FunctionJobIDNotFoundError, FunctionJobPatchModelIncompatibleError, + FunctionUnrecoverableError, UnsupportedFunctionJobClassError, ) from models_library.products import ProductName @@ -57,6 +60,8 @@ _FUNCTION_JOBS_TABLE_COLS, ) +_logger = logging.getLogger(__name__) + async def create_function_jobs( # noqa: PLR0913 app: web.Application, @@ -251,8 +256,19 @@ async def patch_function_job( function_id=missing_uid, product_name=product_name, ) - # reaching this far means the uid is missing for other reasons - raise FunctionJobIDNotFoundError(function_job_id=missing_uid) + # Should not reach this far + exception = FunctionUnrecoverableError() + _logger.exception( + **create_troubleshooting_log_kwargs( + user_error_msg="Inconsistent state detected, please contact support.", + error=exception, + error_context={ + "function_job_uid": missing_uid, + "used_function_class": f"{used_function_class}", + }, + ) + ) + raise exception return [jobs[patch.uid] for patch in registered_function_job_patch_inputs] From e959e4dea4222e4eaf6a2348b22a80d4f4bcdd51 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 14 Oct 2025 12:43:30 +0200 Subject: [PATCH 13/63] improve comments --- .../functions/_function_jobs_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py index 560ee8b54cd1..091850a17196 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py @@ -242,7 +242,7 @@ async def patch_function_job( for row in result.fetchall() } if set(function_job_uids) != set(jobs.keys()): - # ensure meaningful error message is raised + # ensure meaningful error is raised missing_uids = set(function_job_uids) - set(jobs.keys()) missing_uid = missing_uids.pop() function_job = await get_function_job( From 4f1b9b41488cde88d2b1761c50ac62d9907b52e2 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 11:11:56 +0200 Subject: [PATCH 14/63] improve patch in repository --- .../functions/_function_jobs_repository.py | 168 +++++++----------- 1 file changed, 61 insertions(+), 107 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py index 091850a17196..5c6ebeaa7851 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py @@ -5,9 +5,9 @@ import sqlalchemy from aiohttp import web -from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from models_library.functions import ( FunctionClass, + FunctionClassSpecificData, FunctionID, FunctionInputsList, FunctionJobCollectionID, @@ -17,14 +17,13 @@ FunctionOutputs, FunctionsApiAccessRights, RegisteredFunctionJobDB, + RegisteredFunctionJobPatch, RegisteredFunctionJobWithStatusDB, RegisteredProjectFunctionJobPatchInputList, RegisteredSolverFunctionJobPatchInputList, ) from models_library.functions_errors import ( FunctionJobIDNotFoundError, - FunctionJobPatchModelIncompatibleError, - FunctionUnrecoverableError, UnsupportedFunctionJobClassError, ) from models_library.products import ProductName @@ -44,7 +43,7 @@ pass_or_acquire_connection, transaction_context, ) -from sqlalchemy import Text, case, cast, func +from sqlalchemy import Text, cast, func from sqlalchemy.ext.asyncio import AsyncConnection from sqlalchemy.sql import func @@ -159,118 +158,40 @@ async def patch_function_job( FunctionsApiAccessRights.WRITE_FUNCTION_JOBS, ], ) + updated_jobs = [] + for patch_input in registered_function_job_patch_inputs: - case_inputs = case(value=function_jobs_table.c.uuid) - case_outputs = case(value=function_jobs_table.c.uuid) - case_class_specific_data = case(value=function_jobs_table.c.uuid) - case_title = case(value=function_jobs_table.c.uuid) - case_description = case(value=function_jobs_table.c.uuid) - - function_job_uids = [] - for patch in registered_function_job_patch_inputs: - function_job_uids.append(patch.uid) - if patch.patch.inputs is not None: - case_inputs = case_inputs.when(patch.uid, patch.patch.inputs) - if patch.patch.outputs is not None: - case_outputs = case_outputs.when(patch.uid, patch.patch.outputs) - if patch.patch.title is not None: - case_title = case_title.when(patch.uid, patch.patch.title) - if patch.patch.description is not None: - case_description = case_description.when( - patch.uid, patch.patch.description - ) - - # patch class specific data - if patch.patch.function_class == FunctionClass.PROJECT: - if patch.patch.project_job_id is not None: - case_class_specific_data = case_class_specific_data.when( - patch.uid, - func.jsonb_set( - function_jobs_table.c.class_specific_data, - "{project_job_id}", - f'"{patch.patch.project_job_id}"', - ), - ) - if patch.patch.job_creation_task_id is not None: - case_class_specific_data = case_class_specific_data.when( - patch.uid, - func.jsonb_set( - function_jobs_table.c.class_specific_data, - "{job_creation_task_id}", - f'"{patch.patch.job_creation_task_id}"', - ), - ) - elif patch.patch.function_class == FunctionClass.SOLVER: - if patch.patch.solver_job_id is not None: - case_class_specific_data = case_class_specific_data.when( - patch.uid, - func.jsonb_set( - function_jobs_table.c.class_specific_data, - "{solver_job_id}", - f'"{patch.patch.solver_job_id}"', - ), - ) - if patch.patch.job_creation_task_id is not None: - case_class_specific_data = case_class_specific_data.when( - patch.uid, - func.jsonb_set( - function_jobs_table.c.class_specific_data, - "{job_creation_task_id}", - f'"{patch.patch.job_creation_task_id}"', - ), - ) - else: - raise UnsupportedFunctionJobClassError( - function_job_class=patch.patch.function_class - ) - - result = await transaction.execute( - function_jobs_table.update() - .values( - inputs=case_inputs, - outputs=case_outputs, - title=case_title, - description=case_description, - class_specific_data=case_class_specific_data, - ) - .where(function_jobs_table.c.uuid.in_(function_job_uids)) - .where(function_jobs_table.c.function_class == used_function_class) - .returning(*_FUNCTION_JOBS_TABLE_COLS) - ) - jobs = { - row.uuid: RegisteredFunctionJobDB.model_validate(row) - for row in result.fetchall() - } - if set(function_job_uids) != set(jobs.keys()): - # ensure meaningful error is raised - missing_uids = set(function_job_uids) - set(jobs.keys()) - missing_uid = missing_uids.pop() - function_job = await get_function_job( + job = await get_function_job( app, + connection=transaction, user_id=user_id, product_name=product_name, - function_job_id=missing_uid, + function_job_id=patch_input.uid, + ) + class_specific_data = _update_class_specific_data( + class_specific_data=job.class_specific_data, patch=patch_input.patch ) - if function_job.function_class != used_function_class: - raise FunctionJobPatchModelIncompatibleError( - function_id=missing_uid, - product_name=product_name, + + result = await transaction.execute( + function_jobs_table.update() + .where(function_jobs_table.c.uuid == f"{patch_input.uid}") + .where( + function_jobs_table.c.function_class == used_function_class.value ) - # Should not reach this far - exception = FunctionUnrecoverableError() - _logger.exception( - **create_troubleshooting_log_kwargs( - user_error_msg="Inconsistent state detected, please contact support.", - error=exception, - error_context={ - "function_job_uid": missing_uid, - "used_function_class": f"{used_function_class}", - }, + .values( + inputs=patch_input.patch.inputs, + outputs=patch_input.patch.outputs, + class_specific_data=class_specific_data, + title=patch_input.patch.title, + description=patch_input.patch.description, + status="created", ) + .returning(*_FUNCTION_JOBS_TABLE_COLS) ) - raise exception + row = result.one() + updated_jobs.append(RegisteredFunctionJobDB.model_validate(row)) - return [jobs[patch.uid] for patch in registered_function_job_patch_inputs] + return updated_jobs async def list_function_jobs_with_status( @@ -606,3 +527,36 @@ async def update_function_job_outputs( raise FunctionJobIDNotFoundError(function_job_id=function_job_id) return TypeAdapter(FunctionOutputs).validate_python(row.outputs) + + +def _update_class_specific_data( + class_specific_data: dict, + patch: RegisteredFunctionJobPatch, +) -> FunctionClassSpecificData: + if patch.function_class == FunctionClass.PROJECT: + return FunctionClassSpecificData( + project_job_id=( + f"{patch.project_job_id}" + if patch.project_job_id + else class_specific_data.get("project_job_id") + ), + job_creation_task_id=( + f"{patch.job_creation_task_id}" + if patch.job_creation_task_id + else class_specific_data.get("job_creation_task_id") + ), + ) + if patch.function_class == FunctionClass.SOLVER: + return FunctionClassSpecificData( + solver_job_id=( + f"{patch.solver_job_id}" + if patch.solver_job_id + else class_specific_data.get("solver_job_id") + ), + job_creation_task_id=( + f"{patch.job_creation_task_id}" + if patch.job_creation_task_id + else class_specific_data.get("job_creation_task_id") + ), + ) + raise UnsupportedFunctionJobClassError(function_job_class=patch.function_class) From 7f11bab58c947447c64d8ea7c1b4168033cecd99 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 11:27:42 +0200 Subject: [PATCH 15/63] implement patch function job endpoint --- .../functions/_controller/_functions_rpc.py | 14 +-- .../functions/_function_jobs_repository.py | 7 +- .../functions/_functions_service.py | 99 ++----------------- .../test_function_jobs_controller_rpc.py | 34 ++++++- 4 files changed, 52 insertions(+), 102 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py index 36b389a6e2e1..c3756717b404 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py @@ -23,8 +23,9 @@ RegisteredFunctionJob, RegisteredFunctionJobCollection, RegisteredFunctionJobList, - RegisteredFunctionJobPatch, RegisteredFunctionJobWithStatus, + RegisteredProjectFunctionJobPatchInputList, + RegisteredSolverFunctionJobPatchInputList, ) from models_library.functions_errors import ( FunctionIDNotFoundError, @@ -113,16 +114,17 @@ async def patch_registered_function_job( *, user_id: UserID, product_name: ProductName, - function_job_uuid: FunctionJobID, - registered_function_job_patch: RegisteredFunctionJobPatch, -) -> RegisteredFunctionJob: + registered_function_job_patch_inputs: ( + RegisteredProjectFunctionJobPatchInputList + | RegisteredSolverFunctionJobPatchInputList + ), +) -> list[RegisteredFunctionJob]: return await _functions_service.patch_registered_function_job( app=app, user_id=user_id, product_name=product_name, - function_job_uuid=function_job_uuid, - registered_function_job_patch=registered_function_job_patch, + registered_function_job_patch_inputs=registered_function_job_patch_inputs, ) diff --git a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py index 5c6ebeaa7851..112c91eee500 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py @@ -24,6 +24,7 @@ ) from models_library.functions_errors import ( FunctionJobIDNotFoundError, + FunctionJobPatchModelIncompatibleError, UnsupportedFunctionJobClassError, ) from models_library.products import ProductName @@ -188,7 +189,11 @@ async def patch_function_job( ) .returning(*_FUNCTION_JOBS_TABLE_COLS) ) - row = result.one() + row = result.one_or_none() + if row is None: + raise FunctionJobPatchModelIncompatibleError( + function_id=job.function_uuid, product_name=product_name + ) updated_jobs.append(RegisteredFunctionJobDB.model_validate(row)) return updated_jobs diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py index cb427f3ecc61..92b02cdcd4cf 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py @@ -31,18 +31,18 @@ RegisteredFunctionJobCollection, RegisteredFunctionJobDB, RegisteredFunctionJobList, - RegisteredFunctionJobPatch, RegisteredFunctionJobWithStatus, RegisteredFunctionJobWithStatusDB, RegisteredProjectFunction, RegisteredProjectFunctionJob, + RegisteredProjectFunctionJobPatchInputList, RegisteredProjectFunctionJobWithStatus, RegisteredSolverFunction, RegisteredSolverFunctionJob, + RegisteredSolverFunctionJobPatchInputList, RegisteredSolverFunctionJobWithStatus, ) from models_library.functions_errors import ( - FunctionJobPatchModelIncompatibleError, UnsupportedFunctionClassError, UnsupportedFunctionJobClassError, ) @@ -111,30 +111,19 @@ async def patch_registered_function_job( *, user_id: UserID, product_name: ProductName, - function_job_uuid: FunctionJobID, - registered_function_job_patch: RegisteredFunctionJobPatch, -) -> RegisteredFunctionJob: - job = await _function_jobs_repository.get_function_job( - app=app, - user_id=user_id, - product_name=product_name, - function_job_id=function_job_uuid, - ) - if job.function_class != registered_function_job_patch.function_class: - raise FunctionJobPatchModelIncompatibleError( - function_id=job.function_uuid, - product_name=product_name, - ) - - patched_job = _patch_functionjob(job, registered_function_job_patch) + registered_function_job_patch_inputs: ( + RegisteredProjectFunctionJobPatchInputList + | RegisteredSolverFunctionJobPatchInputList + ), +) -> list[RegisteredFunctionJob]: result = await _function_jobs_repository.patch_function_job( app=app, user_id=user_id, product_name=product_name, - registered_function_job_db=patched_job, + registered_function_job_patch_inputs=registered_function_job_patch_inputs, ) - return _decode_functionjob(result) + return [_decode_functionjob(job) for job in result] async def register_function_job_collection( @@ -924,73 +913,3 @@ def _decode_functionjob_wso( raise UnsupportedFunctionJobClassError( function_job_class=functionjob_db.function_class ) - - -def _patch_functionjob( - function_job_db: RegisteredFunctionJobDB, - patch: RegisteredFunctionJobPatch, -) -> RegisteredFunctionJobDB: - if function_job_db.function_class == FunctionClass.PROJECT: - assert patch.function_class == FunctionClass.PROJECT # nosec - return RegisteredFunctionJobDB( - function_class=FunctionClass.PROJECT, - function_uuid=function_job_db.function_uuid, - title=patch.title or function_job_db.title, - uuid=function_job_db.uuid, - description=patch.description or function_job_db.description, - inputs=patch.inputs or function_job_db.inputs, - outputs=patch.outputs or function_job_db.outputs, - created=function_job_db.created, - class_specific_data=FunctionClassSpecificData( - project_job_id=( - f"{patch.project_job_id}" - if patch.project_job_id - else function_job_db.class_specific_data.get("project_job_id") - ), - job_creation_task_id=( - f"{patch.job_creation_task_id}" - if patch.job_creation_task_id - else function_job_db.class_specific_data.get("job_creation_task_id") - ), - ), - ) - if function_job_db.function_class == FunctionClass.SOLVER: - assert patch.function_class == FunctionClass.SOLVER # nosec - return RegisteredFunctionJobDB( - function_class=FunctionClass.SOLVER, - function_uuid=function_job_db.function_uuid, - title=patch.title or function_job_db.title, - uuid=function_job_db.uuid, - description=patch.description or function_job_db.description, - inputs=patch.inputs or function_job_db.inputs, - outputs=patch.outputs or function_job_db.outputs, - created=function_job_db.created, - class_specific_data=FunctionClassSpecificData( - solver_job_id=( - f"{patch.solver_job_id}" - if patch.solver_job_id - else function_job_db.class_specific_data.get("solver_job_id") - ), - job_creation_task_id=( - f"{patch.job_creation_task_id}" - if patch.job_creation_task_id - else function_job_db.class_specific_data.get("job_creation_task_id") - ), - ), - ) - if function_job_db.function_class == FunctionClass.PYTHON_CODE: - assert patch.function_class == FunctionClass.PYTHON_CODE # nosec - return RegisteredFunctionJobDB( - function_class=FunctionClass.PYTHON_CODE, - function_uuid=function_job_db.function_uuid, - title=patch.title or function_job_db.title, - uuid=function_job_db.uuid, - description=patch.description or function_job_db.description, - inputs=patch.inputs or function_job_db.inputs, - outputs=patch.outputs or function_job_db.outputs, - created=function_job_db.created, - class_specific_data=function_job_db.class_specific_data, - ) - raise UnsupportedFunctionJobClassError( - function_job_class=function_job_db.function_class - ) diff --git a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py index 656a177b18f6..a5633c72b255 100644 --- a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py @@ -19,7 +19,11 @@ RegisteredFunctionJob, RegisteredFunctionJobPatch, RegisteredProjectFunctionJobPatch, + RegisteredProjectFunctionJobPatchInput, + RegisteredProjectFunctionJobPatchInputList, RegisteredSolverFunctionJobPatch, + RegisteredSolverFunctionJobPatchInput, + RegisteredSolverFunctionJobPatchInputList, SolverFunctionJob, ) from models_library.functions_errors import ( @@ -622,13 +626,33 @@ async def test_patch_registered_function_jobs( ) assert len(registered_jobs) == 1 registered_job = registered_jobs[0] + patch_inputs: ( + RegisteredProjectFunctionJobPatchInputList + | RegisteredSolverFunctionJobPatchInputList + ) + if function.function_class == FunctionClass.PROJECT: + assert isinstance(patch, RegisteredProjectFunctionJobPatch) + patch_inputs = [ + RegisteredProjectFunctionJobPatchInput(uid=registered_job.uid, patch=patch) + ] + elif function.function_class == FunctionClass.SOLVER: + assert isinstance(patch, RegisteredSolverFunctionJobPatch) + patch_inputs = [ + RegisteredSolverFunctionJobPatchInput(uid=registered_job.uid, patch=patch) + ] + else: + pytest.fail("Unsupported function class") - registered_job = await webserver_rpc_client.functions.patch_registered_function_job( - user_id=logged_user["id"], - function_job_uuid=registered_job.uid, - product_name=osparc_product_name, - registered_function_job_patch=patch, + registered_jobs = ( + await webserver_rpc_client.functions.patch_registered_function_job( + user_id=logged_user["id"], + function_job_uuid=registered_job.uid, + product_name=osparc_product_name, + registered_function_job_patch_inputs=patch_inputs, + ) ) + assert len(registered_jobs) == 1 + registered_job = registered_jobs[0] assert registered_job.title == patch.title assert registered_job.description == patch.description assert registered_job.inputs == patch.inputs From f2c3fb6ca7006a798ff91e82e0ead460cf9a7619 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 11:28:16 +0200 Subject: [PATCH 16/63] fixes --- .../functions/functions_rpc_interface.py | 16 +++++++++------- .../rpc_interfaces/webserver/v1/functions.py | 15 +++++++++------ 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py index c43610bddb83..43ab02fa657f 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py @@ -26,8 +26,9 @@ FunctionOutputs, FunctionUserAccessRights, FunctionUserApiAccessRights, - RegisteredFunctionJobPatch, RegisteredFunctionJobWithStatus, + RegisteredProjectFunctionJobPatchInputList, + RegisteredSolverFunctionJobPatchInputList, ) from models_library.products import ProductName from models_library.rabbitmq_basic_types import RPCMethodName @@ -366,19 +367,20 @@ async def patch_registered_function_job( *, user_id: UserID, product_name: ProductName, - function_job_uuid: FunctionJobID, - registered_function_job_patch: RegisteredFunctionJobPatch, -) -> RegisteredFunctionJob: + registered_function_job_patch_inputs: ( + RegisteredProjectFunctionJobPatchInputList + | RegisteredSolverFunctionJobPatchInputList + ), +) -> list[RegisteredFunctionJob]: result = await rabbitmq_rpc_client.request( DEFAULT_WEBSERVER_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("patch_registered_function_job"), user_id=user_id, product_name=product_name, - function_job_uuid=function_job_uuid, - registered_function_job_patch=registered_function_job_patch, + registered_function_job_patch_inputs=registered_function_job_patch_inputs, timeout_s=_FUNCTION_RPC_TIMEOUT_SEC, ) - return TypeAdapter(RegisteredFunctionJob).validate_python( + return TypeAdapter(list[RegisteredFunctionJob]).validate_python( result ) # Validates the result as a RegisteredFunctionJob diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py index 249f96397d38..80d0e2ca6360 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py @@ -26,8 +26,9 @@ FunctionUserAccessRights, FunctionUserApiAccessRights, RegisteredFunctionJobList, - RegisteredFunctionJobPatch, RegisteredFunctionJobWithStatus, + RegisteredProjectFunctionJobPatchInputList, + RegisteredSolverFunctionJobPatchInputList, ) from models_library.products import ProductName from models_library.rabbitmq_basic_types import RPCNamespace @@ -337,16 +338,18 @@ async def patch_registered_function_job( product_name: ProductName, user_id: UserID, function_job_uuid: FunctionJobID, - registered_function_job_patch: RegisteredFunctionJobPatch, - ) -> RegisteredFunctionJob: + registered_function_job_patch_inputs: ( + RegisteredProjectFunctionJobPatchInputList + | RegisteredSolverFunctionJobPatchInputList + ), + ) -> list[RegisteredFunctionJob]: """Patch a registered function job.""" - return TypeAdapter(RegisteredFunctionJob).validate_python( + return TypeAdapter(list[RegisteredFunctionJob]).validate_python( await self._request( "patch_registered_function_job", product_name=product_name, user_id=user_id, - function_job_uuid=function_job_uuid, - registered_function_job_patch=registered_function_job_patch, + registered_function_job_patch_inputs=registered_function_job_patch_inputs, ), ) From 4a3a3bbf7faec2870c9d56feed383c7d27da247f Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 11:30:42 +0200 Subject: [PATCH 17/63] minor cleanup --- .../functions/_function_jobs_repository.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py index 112c91eee500..b93299e2222c 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py @@ -169,6 +169,11 @@ async def patch_function_job( product_name=product_name, function_job_id=patch_input.uid, ) + if job.function_class != used_function_class: + raise FunctionJobPatchModelIncompatibleError( + function_id=job.function_uuid, product_name=product_name + ) + class_specific_data = _update_class_specific_data( class_specific_data=job.class_specific_data, patch=patch_input.patch ) @@ -176,9 +181,6 @@ async def patch_function_job( result = await transaction.execute( function_jobs_table.update() .where(function_jobs_table.c.uuid == f"{patch_input.uid}") - .where( - function_jobs_table.c.function_class == used_function_class.value - ) .values( inputs=patch_input.patch.inputs, outputs=patch_input.patch.outputs, @@ -189,12 +191,7 @@ async def patch_function_job( ) .returning(*_FUNCTION_JOBS_TABLE_COLS) ) - row = result.one_or_none() - if row is None: - raise FunctionJobPatchModelIncompatibleError( - function_id=job.function_uuid, product_name=product_name - ) - updated_jobs.append(RegisteredFunctionJobDB.model_validate(row)) + updated_jobs.append(RegisteredFunctionJobDB.model_validate(result.one())) return updated_jobs From 4f2918d0649ea883068a47a72b3aaf14b401c632 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 12:31:58 +0200 Subject: [PATCH 18/63] use new patch method in run_function --- .../rpc_interfaces/webserver/v1/functions.py | 1 - .../_service_function_jobs.py | 148 ++++++++---------- .../models/domain/functions.py | 20 +++ .../services_rpc/wb_api_server.py | 14 +- 4 files changed, 92 insertions(+), 91 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py index 80d0e2ca6360..afc60a234948 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py @@ -337,7 +337,6 @@ async def patch_registered_function_job( *, product_name: ProductName, user_id: UserID, - function_job_uuid: FunctionJobID, registered_function_job_patch_inputs: ( RegisteredProjectFunctionJobPatchInputList | RegisteredSolverFunctionJobPatchInputList diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py index 6728b8bc44a5..3e9d2f4859b5 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import overload +from typing import Annotated import jsonschema from common_library.exclude import as_dict_exclude_none @@ -14,30 +14,30 @@ ProjectFunctionJob, RegisteredFunction, RegisteredFunctionJob, - RegisteredFunctionJobPatch, RegisteredProjectFunctionJobPatch, RegisteredSolverFunctionJobPatch, SolverFunctionJob, - SolverJobID, - TaskID, ) from models_library.functions_errors import ( FunctionInputsValidationError, UnsupportedFunctionClassError, ) from models_library.products import ProductName -from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.rest_pagination import PageMetaInfoLimitOffset, PageOffsetInt from models_library.rpc_pagination import PageLimitInt from models_library.users import UserID -from pydantic import TypeAdapter, ValidationError +from pydantic import Field, TypeAdapter, ValidationError, validate_call from simcore_service_api_server._service_functions import FunctionService from simcore_service_api_server.services_rpc.storage import StorageService from ._service_jobs import JobService from .models.api_resources import JobLinks -from .models.domain.functions import PreRegisteredFunctionJobData +from .models.domain.functions import ( + PreRegisteredFunctionJobData, + ProjectFunctionJobPatch, + SolverFunctionJobPatch, +) from .models.schemas.jobs import JobInputs, JobPricingSpecification from .services_http.webserver import AuthSession from .services_rpc.wb_api_server import WbApiRpcClient @@ -202,81 +202,51 @@ async def pre_register_function_job( for job, input_ in zip(jobs, job_inputs) ] - @overload - async def patch_registered_function_job( - self, - *, - user_id: UserID, - product_name: ProductName, - function_job_id: FunctionJobID, - function_class: FunctionClass, - job_creation_task_id: TaskID | None, - ) -> RegisteredFunctionJob: ... - - @overload - async def patch_registered_function_job( - self, - *, - user_id: UserID, - product_name: ProductName, - function_job_id: FunctionJobID, - function_class: FunctionClass, - job_creation_task_id: TaskID | None, - project_job_id: ProjectID | None, - ) -> RegisteredFunctionJob: ... - - @overload - async def patch_registered_function_job( - self, - *, - user_id: UserID, - product_name: ProductName, - function_job_id: FunctionJobID, - function_class: FunctionClass, - job_creation_task_id: TaskID | None, - solver_job_id: SolverJobID | None, - ) -> RegisteredFunctionJob: ... - + @validate_call async def patch_registered_function_job( self, *, user_id: UserID, product_name: ProductName, - function_job_id: FunctionJobID, - function_class: FunctionClass, - job_creation_task_id: TaskID | None, - project_job_id: ProjectID | None = None, - solver_job_id: SolverJobID | None = None, - ) -> RegisteredFunctionJob: - # Only allow one of project_job_id or solver_job_id depending on function_class - patch: RegisteredFunctionJobPatch - if function_class == FunctionClass.PROJECT: - patch = RegisteredProjectFunctionJobPatch( - title=None, - description=None, - inputs=None, - outputs=None, - job_creation_task_id=job_creation_task_id, - project_job_id=project_job_id, - ) - elif function_class == FunctionClass.SOLVER: - patch = RegisteredSolverFunctionJobPatch( - title=None, - description=None, - inputs=None, - outputs=None, - job_creation_task_id=job_creation_task_id, - solver_job_id=solver_job_id, - ) - else: - raise UnsupportedFunctionClassError( - function_class=function_class, - ) + patches: Annotated[ + list[ProjectFunctionJobPatch] | list[SolverFunctionJobPatch], + Field(max_length=50, min_length=1), + ], + ) -> list[RegisteredFunctionJob]: + patch_inputs = [] + for patch in patches: + if patch.function_class == FunctionClass.PROJECT: + assert isinstance(patch, ProjectFunctionJobPatch) # nosec + patch_inputs.append( + RegisteredProjectFunctionJobPatch( + title=None, + description=None, + inputs=None, + outputs=None, + job_creation_task_id=patch.job_creation_task_id, + project_job_id=patch.project_job_id, + ) + ) + elif patch.function_class == FunctionClass.SOLVER: + assert isinstance(patch, SolverFunctionJobPatch) # nosec + patch_inputs.append( + RegisteredSolverFunctionJobPatch( + title=None, + description=None, + inputs=None, + outputs=None, + job_creation_task_id=patch.job_creation_task_id, + solver_job_id=patch.solver_job_id, + ) + ) + else: + raise UnsupportedFunctionClassError( + function_class=patch.function_class, + ) return await self._web_rpc_client.patch_registered_function_job( user_id=user_id, product_name=product_name, - function_job_id=function_job_id, - registered_function_job_patch=patch, + registered_function_job_patch_inputs=patch_inputs, ) async def run_function( @@ -305,14 +275,19 @@ async def run_function( job_id=study_job.id, pricing_spec=pricing_spec, ) - return await self.patch_registered_function_job( + registered_jobs = await self.patch_registered_function_job( user_id=self.user_id, product_name=self.product_name, - function_job_id=pre_registered_function_job_data.function_job_id, - function_class=FunctionClass.PROJECT, - job_creation_task_id=None, - project_job_id=study_job.id, + patches=[ + ProjectFunctionJobPatch( + function_job_id=pre_registered_function_job_data.function_job_id, + job_creation_task_id=None, + project_job_id=study_job.id, + ) + ], ) + assert len(registered_jobs) == 1 + return registered_jobs[0] if function.function_class == FunctionClass.SOLVER: solver_job = await self._job_service.create_solver_job( @@ -330,14 +305,19 @@ async def run_function( job_id=solver_job.id, pricing_spec=pricing_spec, ) - return await self.patch_registered_function_job( + registered_jobs = await self.patch_registered_function_job( user_id=self.user_id, product_name=self.product_name, - function_job_id=pre_registered_function_job_data.function_job_id, - function_class=FunctionClass.SOLVER, - job_creation_task_id=None, - solver_job_id=solver_job.id, + patches=[ + SolverFunctionJobPatch( + function_job_id=pre_registered_function_job_data.function_job_id, + job_creation_task_id=None, + solver_job_id=solver_job.id, + ) + ], ) + assert len(registered_jobs) == 1 + return registered_jobs[0] raise UnsupportedFunctionClassError( function_class=function.function_class, diff --git a/services/api-server/src/simcore_service_api_server/models/domain/functions.py b/services/api-server/src/simcore_service_api_server/models/domain/functions.py index 1f75f7441f29..7397ffa4b910 100644 --- a/services/api-server/src/simcore_service_api_server/models/domain/functions.py +++ b/services/api-server/src/simcore_service_api_server/models/domain/functions.py @@ -1,8 +1,14 @@ +from typing import NamedTuple + from models_library.functions import ( + FunctionClass, FunctionJobID, RegisteredFunctionJob, RegisteredFunctionJobWithStatus, + SolverJobID, + TaskID, ) +from models_library.projects import ProjectID from pydantic import BaseModel from ...models.pagination import Page @@ -20,3 +26,17 @@ class PageRegisteredFunctionJobWithorWithoutStatus( # This class is created specifically to provide a name for this in openapi.json. # When using an alias the python-client generates too long file name pass + + +class ProjectFunctionJobPatch(NamedTuple): + function_class = FunctionClass.PROJECT + function_job_id: FunctionJobID + job_creation_task_id: TaskID | None + project_job_id: ProjectID | None + + +class SolverFunctionJobPatch(NamedTuple): + function_class = FunctionClass.SOLVER + function_job_id: FunctionJobID + job_creation_task_id: TaskID | None + solver_job_id: SolverJobID | None diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index 7c7af65f826b..a4a38cbfd405 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -30,8 +30,9 @@ FunctionUserAccessRights, FunctionUserApiAccessRights, RegisteredFunctionJobList, - RegisteredFunctionJobPatch, RegisteredFunctionJobWithStatus, + RegisteredProjectFunctionJobPatchInputList, + RegisteredSolverFunctionJobPatchInputList, ) from models_library.licenses import LicensedItemID from models_library.products import ProductName @@ -500,14 +501,15 @@ async def patch_registered_function_job( *, user_id: UserID, product_name: ProductName, - function_job_id: FunctionJobID, - registered_function_job_patch: RegisteredFunctionJobPatch, - ) -> RegisteredFunctionJob: + registered_function_job_patch_inputs: ( + RegisteredProjectFunctionJobPatchInputList + | RegisteredSolverFunctionJobPatchInputList + ), + ) -> list[RegisteredFunctionJob]: return await self._rpc_client.functions.patch_registered_function_job( user_id=user_id, product_name=product_name, - function_job_uuid=function_job_id, - registered_function_job_patch=registered_function_job_patch, + registered_function_job_patch_inputs=registered_function_job_patch_inputs, ) async def get_function_input_schema( From fa34d1c52b5327210e00dcfd5b9c2c246e5b5bb9 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 13:46:57 +0200 Subject: [PATCH 19/63] propagate task creation all the way out --- .../_service_function_jobs.py | 13 ++-- .../_service_function_jobs_task_client.py | 60 +++++++++---------- .../api/routes/functions_routes.py | 46 +++++--------- .../models/domain/functions.py | 39 +++++++----- 4 files changed, 73 insertions(+), 85 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py index 3e9d2f4859b5..64e19a59d35a 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py @@ -34,9 +34,8 @@ from ._service_jobs import JobService from .models.api_resources import JobLinks from .models.domain.functions import ( + FunctionJobPatch, PreRegisteredFunctionJobData, - ProjectFunctionJobPatch, - SolverFunctionJobPatch, ) from .models.schemas.jobs import JobInputs, JobPricingSpecification from .services_http.webserver import AuthSession @@ -209,14 +208,13 @@ async def patch_registered_function_job( user_id: UserID, product_name: ProductName, patches: Annotated[ - list[ProjectFunctionJobPatch] | list[SolverFunctionJobPatch], + list[FunctionJobPatch], Field(max_length=50, min_length=1), ], ) -> list[RegisteredFunctionJob]: patch_inputs = [] for patch in patches: if patch.function_class == FunctionClass.PROJECT: - assert isinstance(patch, ProjectFunctionJobPatch) # nosec patch_inputs.append( RegisteredProjectFunctionJobPatch( title=None, @@ -228,7 +226,6 @@ async def patch_registered_function_job( ) ) elif patch.function_class == FunctionClass.SOLVER: - assert isinstance(patch, SolverFunctionJobPatch) # nosec patch_inputs.append( RegisteredSolverFunctionJobPatch( title=None, @@ -279,7 +276,8 @@ async def run_function( user_id=self.user_id, product_name=self.product_name, patches=[ - ProjectFunctionJobPatch( + FunctionJobPatch( + function_class=FunctionClass.PROJECT, function_job_id=pre_registered_function_job_data.function_job_id, job_creation_task_id=None, project_job_id=study_job.id, @@ -309,7 +307,8 @@ async def run_function( user_id=self.user_id, product_name=self.product_name, patches=[ - SolverFunctionJobPatch( + FunctionJobPatch( + function_class=FunctionClass.SOLVER, function_job_id=pre_registered_function_job_data.function_job_id, job_creation_task_id=None, solver_job_id=solver_job.id, diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py index 075c20a5544f..d86b0620f952 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py @@ -38,6 +38,7 @@ ) from sqlalchemy.ext.asyncio import AsyncEngine +from ._meta import APP_NAME from ._service_function_jobs import FunctionJobService from ._service_functions import FunctionService from ._service_jobs import JobService @@ -50,6 +51,11 @@ from .models.api_resources import JobLinks from .models.domain.celery_models import ApiServerOwnerMetadata from .models.schemas.jobs import JobInputs, JobPricingSpecification +from .models.api_resources import JobLinks +from .models.domain.celery_models import ApiServerOwnerMetadata +from .models.domain.functions import FunctionJobPatch +from .models.schemas.functions import FunctionJobCreationTaskStatus +from .models.schemas.jobs import JobPricingSpecification from .services_http.webserver import AuthSession from .services_rpc.storage import StorageService from .services_rpc.wb_api_server import WbApiRpcClient @@ -239,30 +245,7 @@ async def inspect_function_job( check_write_permissions=False, ) - async def get_cached_function_job( - self, - *, - function: RegisteredFunction, - job_inputs: JobInputs, - ) -> RegisteredFunctionJob: - """Raises FunctionJobCacheNotFoundError if no cached job is found""" - if cached_function_jobs := await self._web_rpc_client.find_cached_function_jobs( - function_id=function.uid, - inputs=job_inputs.values, - user_id=self.user_id, - product_name=self.product_name, - ): - for cached_function_job in cached_function_jobs: - job_status = await self.inspect_function_job( - function=function, - function_job=cached_function_job, - ) - if job_status.status == RunningState.SUCCESS: - return cached_function_job - - raise FunctionJobCacheNotFoundError - - async def function_job_outputs( # noqa: PLR0911 # pylint: disable=too-many-return-statements + async def function_job_outputs( self, *, function: RegisteredFunction, @@ -326,7 +309,7 @@ async def function_job_outputs( # noqa: PLR0911 # pylint: disable=too-many-retu check_write_permissions=False, ) - async def create_function_job_creation_task( + async def create_function_job_creation_tasks( self, *, function: RegisteredFunction, @@ -336,7 +319,7 @@ async def create_function_job_creation_task( job_links: JobLinks, parent_project_uuid: ProjectID | None = None, parent_node_id: NodeID | None = None, - ) -> RegisteredFunctionJob: + ) -> list[RegisteredFunctionJob]: inputs = [ self._function_job_service.create_function_job_inputs( function=function, function_inputs=input_ @@ -366,9 +349,10 @@ async def create_function_job_creation_task( ) # run function in celery task - owner_metadata = ApiServerOwnerMetadata( - user_id=user_identity.user_id, product_name=user_identity.product_name + user_id=user_identity.user_id, + product_name=user_identity.product_name, + owner=APP_NAME, ) task_uuids = [ await self._celery_task_manager.submit_task( @@ -389,10 +373,22 @@ async def create_function_job_creation_task( for pre_registered_function_job_data in pre_registered_function_job_data_list ] - return await self._function_job_service.patch_registered_function_job( + patched_jobs = await self._function_job_service.patch_registered_function_job( user_id=user_identity.user_id, product_name=user_identity.product_name, - function_job_id=pre_registered_function_job_data_list.function_job_id, - function_class=function.function_class, - job_creation_task_id=TaskID(task_uuid), + patches=[ + FunctionJobPatch( + function_class=function.function_class, + function_job_id=pre_registered_function_job_data.function_job_id, + job_creation_task_id=TaskID(task_uuid), + project_job_id=None, + solver_job_id=None, + ) + for task_uuid, pre_registered_function_job_data in zip( + task_uuids, pre_registered_function_job_data_list + ) + ], ) + patched_jobs_iter = iter(patched_jobs) + _ = lambda job: job if job is not None else next(patched_jobs_iter) + return [_(job) for job in cached_jobs] diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index 38935d3b08e0..f36f0f5485a9 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -16,13 +16,12 @@ RegisteredFunctionJob, RegisteredFunctionJobCollection, ) -from models_library.functions import FunctionJobCollection, FunctionJobID +from models_library.functions import FunctionJobCollection from models_library.products import ProductName from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.users import UserID from servicelib.fastapi.dependencies import get_reverse_url_mapper -from servicelib.utils import limited_gather from ..._service_function_jobs import FunctionJobService from ..._service_function_jobs_task_client import FunctionJobTaskClientService @@ -350,15 +349,17 @@ async def run_function( ) job_links = await function_service.get_function_job_links(to_run_function, url_for) - return await function_job_task_client_service.create_function_job_creation_task( + jobs = await function_job_task_client_service.create_function_job_creation_tasks( function=to_run_function, - function_inputs=function_inputs, + function_inputs=[function_inputs], user_identity=user_identity, pricing_spec=pricing_spec, job_links=job_links, parent_project_uuid=parent_project_uuid, parent_node_id=parent_node_id, ) + assert len(jobs) == 1 # nosec + return jobs[0] @function_router.delete( @@ -429,42 +430,23 @@ async def map_function( ) job_links = await function_service.get_function_job_links(to_run_function, url_for) - async def _run_single_function(function_inputs: FunctionInputs) -> FunctionJobID: - result = ( - await function_job_task_client_service.create_function_job_creation_task( - function=to_run_function, - function_inputs=function_inputs, - user_identity=user_identity, - pricing_spec=pricing_spec, - job_links=job_links, - parent_project_uuid=parent_project_uuid, - parent_node_id=parent_node_id, - ) - ) - return result.uid - - # Run all tasks concurrently, allowing them to complete even if some fail - results = await limited_gather( - *[ - _run_single_function(function_inputs) - for function_inputs in function_inputs_list - ], - reraise=False, - limit=1, + jobs = await function_job_task_client_service.create_function_job_creation_tasks( + function=to_run_function, + function_inputs=function_inputs_list, + user_identity=user_identity, + pricing_spec=pricing_spec, + job_links=job_links, + parent_project_uuid=parent_project_uuid, + parent_node_id=parent_node_id, ) - # Check if any tasks raised exceptions and raise the first one found - for result in results: - if isinstance(result, BaseException): - raise result - # At this point, all results are FunctionJobID since we've checked for exceptions function_job_collection_description = f"Function job collection of map of function {to_run_function.uid} with {len(function_inputs_list)} inputs" return await web_api_rpc_client.register_function_job_collection( function_job_collection=FunctionJobCollection( title="Function job collection of function map", description=function_job_collection_description, - job_ids=results, # type: ignore + job_ids=jobs, # type: ignore ), user_id=user_identity.user_id, product_name=user_identity.product_name, diff --git a/services/api-server/src/simcore_service_api_server/models/domain/functions.py b/services/api-server/src/simcore_service_api_server/models/domain/functions.py index 7397ffa4b910..1fcb734cd6ae 100644 --- a/services/api-server/src/simcore_service_api_server/models/domain/functions.py +++ b/services/api-server/src/simcore_service_api_server/models/domain/functions.py @@ -1,5 +1,3 @@ -from typing import NamedTuple - from models_library.functions import ( FunctionClass, FunctionJobID, @@ -9,7 +7,7 @@ TaskID, ) from models_library.projects import ProjectID -from pydantic import BaseModel +from pydantic import BaseModel, model_validator from ...models.pagination import Page from ...models.schemas.jobs import JobInputs @@ -28,15 +26,28 @@ class PageRegisteredFunctionJobWithorWithoutStatus( pass -class ProjectFunctionJobPatch(NamedTuple): - function_class = FunctionClass.PROJECT - function_job_id: FunctionJobID - job_creation_task_id: TaskID | None - project_job_id: ProjectID | None - - -class SolverFunctionJobPatch(NamedTuple): - function_class = FunctionClass.SOLVER +class FunctionJobPatch(BaseModel): + function_class: FunctionClass function_job_id: FunctionJobID - job_creation_task_id: TaskID | None - solver_job_id: SolverJobID | None + job_creation_task_id: TaskID | None = None + project_job_id: ProjectID | None = None + solver_job_id: SolverJobID | None = None + + @model_validator(mode="after") + def validate_function_class_consistency(self) -> "FunctionJobPatch": + """Validate consistency between function_class and job IDs.""" + if ( + self.solver_job_id is not None + and self.function_class != FunctionClass.SOLVER + ): + msg = f"solver_job_id must be None when function_class is {self.function_class}, expected {FunctionClass.SOLVER}" + raise ValueError(msg) + + if ( + self.project_job_id is not None + and self.function_class != FunctionClass.PROJECT + ): + msg = f"project_job_id must be None when function_class is {self.function_class}, expected {FunctionClass.PROJECT}" + raise ValueError(msg) + + return self From b0cc3fae91fe9835337e53f5e81f5bfc08720328 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 13:49:26 +0200 Subject: [PATCH 20/63] update openapi specs --- services/api-server/openapi.json | 1 + 1 file changed, 1 insertion(+) diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index c6aa0416b5b1..c755705a1a74 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -8240,6 +8240,7 @@ } ] }, + "minItems": 1, "maxItems": 50, "title": "Function Inputs List" } From 661bd3acfdbc1a2299a5ca0b5c40ed5273033d02 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 13:58:55 +0200 Subject: [PATCH 21/63] small change --- .../src/simcore_service_api_server/_service_function_jobs.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py index 64e19a59d35a..25b74669a7d9 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py @@ -102,13 +102,11 @@ async def validate_function_inputs( if function.input_schema.schema_class == FunctionSchemaClass.json_schema: try: - all( + for input in job_inputs: jsonschema.validate( instance=input.values, schema=function.input_schema.schema_content, ) - for input in job_inputs - ) except ValidationError as err: return False, str(err) return True, "Inputs are valid" @@ -147,6 +145,7 @@ async def pre_register_function_job( if not is_valid: raise FunctionInputsValidationError(error=validation_str) + function_jobs: list[ProjectFunctionJob | SolverFunctionJob] if function.function_class == FunctionClass.PROJECT: function_jobs = [ ProjectFunctionJob( From 7804a048623df083a3f1371dcd11cb5c1e99a66b Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 14:47:47 +0200 Subject: [PATCH 22/63] start fixing tests --- .../_service_function_jobs.py | 36 +++++++---- .../_service_function_jobs_task_client.py | 4 +- .../celery/test_functions_celery.py | 64 +++++++++++++++---- 3 files changed, 78 insertions(+), 26 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py index 25b74669a7d9..c7e0c0d642eb 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py @@ -15,7 +15,9 @@ RegisteredFunction, RegisteredFunctionJob, RegisteredProjectFunctionJobPatch, + RegisteredProjectFunctionJobPatchInput, RegisteredSolverFunctionJobPatch, + RegisteredSolverFunctionJobPatchInput, SolverFunctionJob, ) from models_library.functions_errors import ( @@ -215,24 +217,30 @@ async def patch_registered_function_job( for patch in patches: if patch.function_class == FunctionClass.PROJECT: patch_inputs.append( - RegisteredProjectFunctionJobPatch( - title=None, - description=None, - inputs=None, - outputs=None, - job_creation_task_id=patch.job_creation_task_id, - project_job_id=patch.project_job_id, + RegisteredProjectFunctionJobPatchInput( + uid=patch.function_job_id, + patch=RegisteredProjectFunctionJobPatch( + title=None, + description=None, + inputs=None, + outputs=None, + job_creation_task_id=patch.job_creation_task_id, + project_job_id=patch.project_job_id, + ), ) ) elif patch.function_class == FunctionClass.SOLVER: patch_inputs.append( - RegisteredSolverFunctionJobPatch( - title=None, - description=None, - inputs=None, - outputs=None, - job_creation_task_id=patch.job_creation_task_id, - solver_job_id=patch.solver_job_id, + RegisteredSolverFunctionJobPatchInput( + uid=patch.function_job_id, + patch=RegisteredSolverFunctionJobPatch( + title=None, + description=None, + inputs=None, + outputs=None, + job_creation_task_id=patch.job_creation_task_id, + solver_job_id=patch.solver_job_id, + ), ) ) else: diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py index d86b0620f952..6c9fdb74f547 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py @@ -331,7 +331,9 @@ async def create_function_job_creation_tasks( user_id=user_identity.user_id, product_name=user_identity.product_name, function_id=function.uid, - inputs=TypeAdapter(FunctionInputsList).validate_python(inputs), + inputs=TypeAdapter(FunctionInputsList).validate_python( + [input_.values for input_ in inputs] + ), status_filter=[FunctionJobStatus(status=RunningState.SUCCESS)], ) diff --git a/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py b/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py index 75088e96df30..faf93fae7cd2 100644 --- a/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py +++ b/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py @@ -26,8 +26,11 @@ from models_library.functions import ( FunctionClass, FunctionID, + FunctionInputsList, FunctionJobCollection, FunctionJobID, + FunctionJobList, + FunctionJobStatus, FunctionUserAccessRights, FunctionUserApiAccessRights, RegisteredFunction, @@ -35,8 +38,10 @@ RegisteredFunctionJobCollection, RegisteredProjectFunction, RegisteredProjectFunctionJob, - RegisteredProjectFunctionJobPatch, + RegisteredProjectFunctionJobPatchInputList, + RegisteredSolverFunctionJobPatchInputList, ) +from models_library.products import ProductName from models_library.projects import ProjectID from models_library.users import UserID from pytest_mock import MockType @@ -151,15 +156,48 @@ def _(celery_app: Celery) -> None: async def _patch_registered_function_job_side_effect( - mock_registered_project_function_job: RegisteredFunctionJob, *args, **kwargs + mock_registered_project_function_job: RegisteredFunctionJob, + product_name: ProductName, + user_id: UserID, + registered_function_job_patch_inputs: ( + RegisteredProjectFunctionJobPatchInputList + | RegisteredSolverFunctionJobPatchInputList + ), ): - registered_function_job_patch = kwargs["registered_function_job_patch"] - assert isinstance(registered_function_job_patch, RegisteredProjectFunctionJobPatch) - job_creation_task_id = registered_function_job_patch.job_creation_task_id - uid = kwargs["function_job_uuid"] - return mock_registered_project_function_job.model_copy( - update={"job_creation_task_id": job_creation_task_id, "uid": uid} - ) + return [ + mock_registered_project_function_job.model_copy( + update={ + "job_creation_task_id": patch.patch.job_creation_task_id, + "uid": patch.uid, + } + ) + for patch in registered_function_job_patch_inputs + ] + + +async def _find_cached_function_jobs_side_effect( + *, + user_id: UserID, + product_name: ProductName, + function_id: FunctionID, + inputs: FunctionInputsList, + status_filter: list[FunctionJobStatus] | None, +): + return [None] * len(inputs) + + +async def _register_function_job_side_effect( + registered_function_job: RegisteredFunctionJob, + user_id: UserID, + function_jobs: FunctionJobList, + product_name: ProductName, +): + return [ + registered_function_job.model_copy( + update={"uid": FunctionJobID(_faker.uuid4())} + ) + for _ in function_jobs + ] @pytest.mark.parametrize("register_celery_tasks", [_register_fake_run_function_task()]) @@ -216,10 +254,14 @@ async def test_with_fake_run_function( "get_function", return_value=fake_registered_project_function ) mock_handler_in_functions_rpc_interface( - "find_cached_function_jobs", return_value=[] + "find_cached_function_jobs", side_effect=_find_cached_function_jobs_side_effect ) mock_handler_in_functions_rpc_interface( - "register_function_job", return_value=fake_registered_project_function_job + "register_function_job", + side_effect=partial( + _register_function_job_side_effect, + fake_registered_project_function_job, + ), ) mock_handler_in_functions_rpc_interface( From f4bae45d8e0b3aebf4f7c7368dd5229e0aa987ae Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 15:13:40 +0200 Subject: [PATCH 23/63] fix more tests --- .../api/routes/functions_routes.py | 2 +- .../celery/test_functions_celery.py | 23 +++++++++++++------ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index f36f0f5485a9..2ef0df2cd75e 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -446,7 +446,7 @@ async def map_function( function_job_collection=FunctionJobCollection( title="Function job collection of function map", description=function_job_collection_description, - job_ids=jobs, # type: ignore + job_ids=[job.uid for job in jobs], # type: ignore ), user_id=user_identity.user_id, product_name=user_identity.product_name, diff --git a/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py b/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py index faf93fae7cd2..8421dc30937e 100644 --- a/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py +++ b/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py @@ -419,10 +419,14 @@ def _default_side_effect( "get_function", return_value=fake_registered_project_function ) mock_handler_in_functions_rpc_interface( - "find_cached_function_jobs", return_value=[] + "find_cached_function_jobs", side_effect=_find_cached_function_jobs_side_effect ) mock_handler_in_functions_rpc_interface( - "register_function_job", return_value=fake_registered_project_function_job + "register_function_job", + side_effect=partial( + _register_function_job_side_effect, + fake_registered_project_function_job, + ), ) mock_handler_in_functions_rpc_interface( "get_functions_user_api_access_rights", @@ -539,10 +543,14 @@ def _default_side_effect( "get_function", return_value=fake_registered_project_function ) mock_handler_in_functions_rpc_interface( - "find_cached_function_jobs", return_value=[] + "find_cached_function_jobs", side_effect=_find_cached_function_jobs_side_effect ) mock_handler_in_functions_rpc_interface( - "register_function_job", return_value=fake_registered_project_function_job + "register_function_job", + side_effect=partial( + _register_function_job_side_effect, + fake_registered_project_function_job, + ), ) mock_handler_in_functions_rpc_interface( "get_functions_user_api_access_rights", @@ -593,9 +601,10 @@ def _default_side_effect( if expected_status_code == status.HTTP_200_OK: FunctionJobCollection.model_validate(response.json()) - task_id = patch_mock.call_args.kwargs[ - "registered_function_job_patch" - ].job_creation_task_id + task_id = patch_mock.call_args.kwargs["registered_function_job_patch_inputs"][ + 0 + ].patch.job_creation_task_id + assert task_id is not None await _wait_for_task_result(client, auth, f"{task_id}") assert side_effect_checks["headers_checked"] is True From 20088b092d512671bb0554a60acccc442b0ef052 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 15:36:09 +0200 Subject: [PATCH 24/63] fix yet another test --- .../celery/test_functions_celery.py | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py b/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py index 8421dc30937e..4c4951b73538 100644 --- a/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py +++ b/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py @@ -655,7 +655,7 @@ def _default_side_effect( "get_function", return_value=fake_registered_project_function ) mock_handler_in_functions_rpc_interface( - "find_cached_function_jobs", return_value=[] + "find_cached_function_jobs", side_effect=_find_cached_function_jobs_side_effect ) mock_handler_in_projects_rpc_interface("mark_project_as_job", return_value=None) @@ -663,11 +663,19 @@ def _default_side_effect( _generated_function_job_ids: list[FunctionJobID] = [] async def _register_function_job_side_effect( - generated_function_job_ids: list[FunctionJobID], *args, **kwargs + generated_function_job_ids: list, + user_id: UserID, + function_jobs: FunctionJobList, + product_name: ProductName, ): - uid = FunctionJobID(_faker.uuid4()) - generated_function_job_ids.append(uid) - return fake_registered_project_function_job.model_copy(update={"uid": uid}) + registered_jobs = [] + for _ in function_jobs: + uid = FunctionJobID(_faker.uuid4()) + generated_function_job_ids.append(uid) + registered_jobs.append( + fake_registered_project_function_job.model_copy(update={"uid": uid}) + ) + return registered_jobs mock_handler_in_functions_rpc_interface( "register_function_job", @@ -726,10 +734,17 @@ async def _register_function_job_collection_side_effect(*args, **kwargs): assert ( job_collection.job_ids == _generated_function_job_ids ), "Job ID did not preserve order or were incorrectly propagated" - celery_task_ids = { - elm.kwargs["registered_function_job_patch"].job_creation_task_id - for elm in patch_mock.call_args_list - } + + celery_task_ids = set() + for args in patch_mock.call_args_list: + inputs = args.kwargs["registered_function_job_patch_inputs"] + celery_task_ids = celery_task_ids.union( + { + input_.patch.job_creation_task_id + for input_ in inputs + if input_.patch.job_creation_task_id + } + ) assert len(celery_task_ids) == len(_inputs) for task_id in celery_task_ids: await _wait_for_task_result(client, auth, f"{task_id}") From 76a6f6eeddf240d88f2f5bc0f9914f6f88d5e1be Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 16:14:56 +0200 Subject: [PATCH 25/63] fix tests --- .../api/routes/function_jobs_routes.py | 6 ++-- .../api/routes/functions_routes.py | 4 +-- .../test_api_routers_function_jobs.py | 13 +++++-- .../test_api_routers_functions.py | 36 +++++++++++++++++-- 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/function_jobs_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/function_jobs_routes.py index 8bedd5efb1e9..44b909c9d025 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/function_jobs_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/function_jobs_routes.py @@ -172,9 +172,11 @@ async def register_function_job( user_id: Annotated[UserID, Depends(get_current_user_id)], product_name: Annotated[ProductName, Depends(get_product_name)], ) -> RegisteredFunctionJob: - return await wb_api_rpc.register_function_job( - function_job=function_job, user_id=user_id, product_name=product_name + registered_jobs = await wb_api_rpc.register_function_job( + function_jobs=[function_job], user_id=user_id, product_name=product_name ) + assert len(registered_jobs) == 1 # nosec + return registered_jobs[0] @function_job_router.get( diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index 2ef0df2cd75e..f96475d12876 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -28,7 +28,7 @@ from ..._service_functions import FunctionService from ...models.pagination import Page, PaginationParams from ...models.schemas.errors import ErrorGet -from ...models.schemas.jobs import JobPricingSpecification +from ...models.schemas.jobs import JobInputs, JobPricingSpecification from ...services_rpc.wb_api_server import WbApiRpcClient from ..dependencies.authentication import ( Identity, @@ -305,7 +305,7 @@ async def validate_function_inputs( ) -> tuple[bool, str]: return await function_job_service.validate_function_inputs( function=function, - inputs=inputs, + job_inputs=[JobInputs(values=inputs or {})], ) diff --git a/services/api-server/tests/unit/api_functions/test_api_routers_function_jobs.py b/services/api-server/tests/unit/api_functions/test_api_routers_function_jobs.py index e3dda6a18ef8..99fea7684af1 100644 --- a/services/api-server/tests/unit/api_functions/test_api_routers_function_jobs.py +++ b/services/api-server/tests/unit/api_functions/test_api_routers_function_jobs.py @@ -20,6 +20,7 @@ RegisteredProjectFunctionJob, ) from models_library.functions import ( + FunctionJobList, FunctionJobStatus, RegisteredProjectFunction, RegisteredProjectFunctionJobWithStatus, @@ -65,15 +66,23 @@ async def test_delete_function_job( async def test_register_function_job( client: AsyncClient, - mock_handler_in_functions_rpc_interface: Callable[[str, Any], None], + mock_handler_in_functions_rpc_interface: Callable, fake_project_function_job: ProjectFunctionJob, fake_registered_project_function_job: RegisteredProjectFunctionJob, auth: httpx.BasicAuth, ) -> None: """Test the register_function_job endpoint.""" + async def _register_function_job_side_effect( + user_id: UserID, + function_jobs: FunctionJobList, + product_name: ProductName, + ): + assert len(function_jobs) == 1 + return [fake_registered_project_function_job] + mock_handler_in_functions_rpc_interface( - "register_function_job", fake_registered_project_function_job + "register_function_job", side_effect=_register_function_job_side_effect ) response = await client.post( diff --git a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py index fcc8b8e4fafc..130b3d36145c 100644 --- a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py +++ b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py @@ -19,6 +19,7 @@ from httpx import AsyncClient from models_library.api_schemas_long_running_tasks.tasks import TaskGet from models_library.functions import ( + FunctionJobList, FunctionUserAccessRights, FunctionUserApiAccessRights, ProjectFunction, @@ -26,11 +27,14 @@ RegisteredFunctionJob, RegisteredProjectFunction, RegisteredProjectFunctionJob, + RegisteredProjectFunctionJobPatchInputList, + RegisteredSolverFunctionJobPatchInputList, ) from models_library.functions_errors import ( FunctionIDNotFoundError, FunctionReadAccessDeniedError, ) +from models_library.products import ProductName from models_library.rest_pagination import PageMetaInfoLimitOffset from models_library.users import UserID from pydantic import EmailStr @@ -471,8 +475,16 @@ def _default_side_effect( "get_function", fake_registered_project_function ) mock_handler_in_functions_rpc_interface("find_cached_function_jobs", []) + + async def _register_function_job_side_effect( + user_id: UserID, + function_jobs: FunctionJobList, + product_name: ProductName, + ): + return [fake_registered_project_function_job] * len(function_jobs) + mock_handler_in_functions_rpc_interface( - "register_function_job", fake_registered_project_function_job + "register_function_job", side_effect=_register_function_job_side_effect ) mock_handler_in_functions_rpc_interface( "get_functions_user_api_access_rights", @@ -483,8 +495,28 @@ def _default_side_effect( read_functions=True, ), ) + + async def _patch_registered_function_job_side_effect( + product_name: ProductName, + user_id: UserID, + registered_function_job_patch_inputs: ( + RegisteredProjectFunctionJobPatchInputList + | RegisteredSolverFunctionJobPatchInputList + ), + ): + return [ + fake_registered_project_function_job.model_copy( + update={ + "job_creation_task_id": patch.patch.job_creation_task_id, + "uid": patch.uid, + } + ) + for patch in registered_function_job_patch_inputs + ] + mock_handler_in_functions_rpc_interface( - "patch_registered_function_job", fake_registered_project_function_job + "patch_registered_function_job", + side_effect=_patch_registered_function_job_side_effect, ) pre_registered_function_job_data = PreRegisteredFunctionJobData( From 8f5945c0c805173e7d5d787247888fd294ac6a00 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 16:15:12 +0200 Subject: [PATCH 26/63] cleanup --- .../tests/unit/api_functions/test_api_routers_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py index 130b3d36145c..3f72eb174fe6 100644 --- a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py +++ b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py @@ -411,7 +411,7 @@ async def test_run_project_function( mocked_webserver_rpc_api: dict[str, MockType], app: FastAPI, client: AsyncClient, - mock_handler_in_functions_rpc_interface: Callable[[str, Any], None], + mock_handler_in_functions_rpc_interface: Callable, fake_registered_project_function: RegisteredProjectFunction, fake_registered_project_function_job: RegisteredFunctionJob, auth: httpx.BasicAuth, From faefeed3010a5033c9d2afb26581274c1910785a Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 15 Oct 2025 17:50:59 +0200 Subject: [PATCH 27/63] =?UTF-8?q?=20=20=F0=9F=8E=A8=20Check=20study=20and?= =?UTF-8?q?=20solver=20job=20status=20before=20returning=20output=20(#8511?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../_service_function_jobs_task_client.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py index 6c9fdb74f547..ffe88ef3c945 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py @@ -56,6 +56,14 @@ from .models.domain.functions import FunctionJobPatch from .models.schemas.functions import FunctionJobCreationTaskStatus from .models.schemas.jobs import JobPricingSpecification +from .exceptions.backend_errors import ( + SolverJobOutputRequestButNotSucceededError, + StudyJobOutputRequestButNotSucceededError, +) +from .exceptions.function_errors import FunctionJobCacheNotFoundError +from .models.api_resources import JobLinks +from .models.domain.celery_models import ApiServerOwnerMetadata +from .models.schemas.jobs import JobInputs, JobPricingSpecification from .services_http.webserver import AuthSession from .services_rpc.storage import StorageService from .services_rpc.wb_api_server import WbApiRpcClient From afef3084304643f3dc1506f0feab0bda54764f05 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Wed, 15 Oct 2025 17:50:59 +0200 Subject: [PATCH 28/63] =?UTF-8?q?=20=20=F0=9F=8E=A8=20Check=20study=20and?= =?UTF-8?q?=20solver=20job=20status=20before=20returning=20output=20(#8511?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../_service_function_jobs_task_client.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py index ffe88ef3c945..a7f5a641fda9 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py @@ -63,6 +63,13 @@ from .exceptions.function_errors import FunctionJobCacheNotFoundError from .models.api_resources import JobLinks from .models.domain.celery_models import ApiServerOwnerMetadata +from .exceptions.backend_errors import ( + SolverJobOutputRequestButNotSucceededError, + StudyJobOutputRequestButNotSucceededError, +) +from .exceptions.function_errors import FunctionJobCacheNotFoundError +from .models.api_resources import JobLinks +from .models.domain.celery_models import ApiServerOwnerMetadata from .models.schemas.jobs import JobInputs, JobPricingSpecification from .services_http.webserver import AuthSession from .services_rpc.storage import StorageService From 2c489ea453a2f00429e26ee95210f599471bfe83 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 22:15:17 +0200 Subject: [PATCH 29/63] clean up imports --- .../_service_function_jobs_task_client.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py index a7f5a641fda9..8a93712f505e 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py @@ -47,30 +47,11 @@ SolverJobOutputRequestButNotSucceededError, StudyJobOutputRequestButNotSucceededError, ) -from .exceptions.function_errors import FunctionJobCacheNotFoundError -from .models.api_resources import JobLinks -from .models.domain.celery_models import ApiServerOwnerMetadata -from .models.schemas.jobs import JobInputs, JobPricingSpecification from .models.api_resources import JobLinks from .models.domain.celery_models import ApiServerOwnerMetadata from .models.domain.functions import FunctionJobPatch from .models.schemas.functions import FunctionJobCreationTaskStatus from .models.schemas.jobs import JobPricingSpecification -from .exceptions.backend_errors import ( - SolverJobOutputRequestButNotSucceededError, - StudyJobOutputRequestButNotSucceededError, -) -from .exceptions.function_errors import FunctionJobCacheNotFoundError -from .models.api_resources import JobLinks -from .models.domain.celery_models import ApiServerOwnerMetadata -from .exceptions.backend_errors import ( - SolverJobOutputRequestButNotSucceededError, - StudyJobOutputRequestButNotSucceededError, -) -from .exceptions.function_errors import FunctionJobCacheNotFoundError -from .models.api_resources import JobLinks -from .models.domain.celery_models import ApiServerOwnerMetadata -from .models.schemas.jobs import JobInputs, JobPricingSpecification from .services_http.webserver import AuthSession from .services_rpc.storage import StorageService from .services_rpc.wb_api_server import WbApiRpcClient From 2154ff43c36479b5363d484a4e7357c9f6e0c923 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 23:14:10 +0200 Subject: [PATCH 30/63] several fixes --- .../src/simcore_service_api_server/main.py | 3 +-- .../simcore_service_dynamic_scheduler/main.py | 3 +-- .../functions/_function_jobs_repository.py | 17 +++++++++-------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/main.py b/services/api-server/src/simcore_service_api_server/main.py index ace337b472e1..0dfca9daa02a 100644 --- a/services/api-server/src/simcore_service_api_server/main.py +++ b/services/api-server/src/simcore_service_api_server/main.py @@ -7,11 +7,10 @@ from fastapi import FastAPI from servicelib.fastapi.logging_lifespan import create_logging_shutdown_event from servicelib.tracing import TracingConfig +from simcore_service_api_server._meta import APP_NAME from simcore_service_api_server.core.application import create_app from simcore_service_api_server.core.settings import ApplicationSettings -from ._meta import APP_NAME - _logger = logging.getLogger(__name__) _NOISY_LOGGERS: Final[tuple[str, ...]] = ( diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py index a08411cc2c5c..786de55895e0 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/main.py @@ -9,11 +9,10 @@ create_logging_lifespan, ) from servicelib.tracing import TracingConfig +from simcore_service_dynamic_scheduler._meta import APP_NAME from simcore_service_dynamic_scheduler.core.application import create_app from simcore_service_dynamic_scheduler.core.settings import ApplicationSettings -from ._meta import APP_NAME - _logger = logging.getLogger(__name__) _NOISY_LOGGERS: Final[tuple[str, ...]] = ( diff --git a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py index b93299e2222c..8a23f31a6748 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py @@ -177,18 +177,19 @@ async def patch_function_job( class_specific_data = _update_class_specific_data( class_specific_data=job.class_specific_data, patch=patch_input.patch ) + update_values = { + "inputs": patch_input.patch.inputs, + "outputs": patch_input.patch.outputs, + "class_specific_data": class_specific_data, + "title": patch_input.patch.title, + "description": patch_input.patch.description, + "status": "created", + } result = await transaction.execute( function_jobs_table.update() .where(function_jobs_table.c.uuid == f"{patch_input.uid}") - .values( - inputs=patch_input.patch.inputs, - outputs=patch_input.patch.outputs, - class_specific_data=class_specific_data, - title=patch_input.patch.title, - description=patch_input.patch.description, - status="created", - ) + .values(**{k: v for k, v in update_values.items() if v is not None}) .returning(*_FUNCTION_JOBS_TABLE_COLS) ) updated_jobs.append(RegisteredFunctionJobDB.model_validate(result.one())) From a5ec531c797a54bd68199c6b950ff2cba6951b06 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 23:18:54 +0200 Subject: [PATCH 31/63] try to make pylint happy --- .../src/simcore_service_api_server/_service_function_jobs.py | 4 ++-- .../simcore_service_api_server/api/routes/functions_routes.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py index c7e0c0d642eb..e0bbe8995d8f 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py @@ -216,7 +216,7 @@ async def patch_registered_function_job( patch_inputs = [] for patch in patches: if patch.function_class == FunctionClass.PROJECT: - patch_inputs.append( + patch_inputs.append( # type: ignore RegisteredProjectFunctionJobPatchInput( uid=patch.function_job_id, patch=RegisteredProjectFunctionJobPatch( @@ -230,7 +230,7 @@ async def patch_registered_function_job( ) ) elif patch.function_class == FunctionClass.SOLVER: - patch_inputs.append( + patch_inputs.append( # type: ignore RegisteredSolverFunctionJobPatchInput( uid=patch.function_job_id, patch=RegisteredSolverFunctionJobPatch( diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index f96475d12876..15a9c31ab113 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -446,7 +446,7 @@ async def map_function( function_job_collection=FunctionJobCollection( title="Function job collection of function map", description=function_job_collection_description, - job_ids=[job.uid for job in jobs], # type: ignore + job_ids=[job.uid for job in jobs], ), user_id=user_identity.user_id, product_name=user_identity.product_name, From 023b5b1556ddf1c12764a53988d0d5eea39b5260 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 23:27:37 +0200 Subject: [PATCH 32/63] yet another attempt --- .../_service_function_jobs.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py index e0bbe8995d8f..4152107d0591 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py @@ -16,8 +16,10 @@ RegisteredFunctionJob, RegisteredProjectFunctionJobPatch, RegisteredProjectFunctionJobPatchInput, + RegisteredProjectFunctionJobPatchInputList, RegisteredSolverFunctionJobPatch, RegisteredSolverFunctionJobPatchInput, + RegisteredSolverFunctionJobPatchInputList, SolverFunctionJob, ) from models_library.functions_errors import ( @@ -216,8 +218,8 @@ async def patch_registered_function_job( patch_inputs = [] for patch in patches: if patch.function_class == FunctionClass.PROJECT: - patch_inputs.append( # type: ignore - RegisteredProjectFunctionJobPatchInput( + patch_inputs.append( + RegisteredProjectFunctionJobPatchInput( # type: ignore uid=patch.function_job_id, patch=RegisteredProjectFunctionJobPatch( title=None, @@ -230,8 +232,8 @@ async def patch_registered_function_job( ) ) elif patch.function_class == FunctionClass.SOLVER: - patch_inputs.append( # type: ignore - RegisteredSolverFunctionJobPatchInput( + patch_inputs.append( + RegisteredSolverFunctionJobPatchInput( # type: ignore uid=patch.function_job_id, patch=RegisteredSolverFunctionJobPatch( title=None, @@ -247,10 +249,14 @@ async def patch_registered_function_job( raise UnsupportedFunctionClassError( function_class=patch.function_class, ) + return await self._web_rpc_client.patch_registered_function_job( user_id=user_id, product_name=product_name, - registered_function_job_patch_inputs=patch_inputs, + registered_function_job_patch_inputs=TypeAdapter( + RegisteredProjectFunctionJobPatchInputList + | RegisteredSolverFunctionJobPatchInputList + ).validate_python(patch_inputs), ) async def run_function( From 12e1ab10f299827b25251360e47a921e867fde74 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 23:29:42 +0200 Subject: [PATCH 33/63] yet another attempt --- .../simcore_service_api_server/_service_function_jobs.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py index 4152107d0591..242528d002cd 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py @@ -215,11 +215,14 @@ async def patch_registered_function_job( Field(max_length=50, min_length=1), ], ) -> list[RegisteredFunctionJob]: - patch_inputs = [] + patch_inputs: list[ + RegisteredProjectFunctionJobPatchInput + | RegisteredSolverFunctionJobPatchInput + ] = [] for patch in patches: if patch.function_class == FunctionClass.PROJECT: patch_inputs.append( - RegisteredProjectFunctionJobPatchInput( # type: ignore + RegisteredProjectFunctionJobPatchInput( uid=patch.function_job_id, patch=RegisteredProjectFunctionJobPatch( title=None, @@ -233,7 +236,7 @@ async def patch_registered_function_job( ) elif patch.function_class == FunctionClass.SOLVER: patch_inputs.append( - RegisteredSolverFunctionJobPatchInput( # type: ignore + RegisteredSolverFunctionJobPatchInput( uid=patch.function_job_id, patch=RegisteredSolverFunctionJobPatch( title=None, From ce080bd5a94c099985b99e6230d54ccf24462ca5 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 23:50:29 +0200 Subject: [PATCH 34/63] pylint --- .../simcore_service_api_server/_service_function_jobs.py | 6 +++--- .../_service_function_jobs_task_client.py | 2 +- .../functions/_function_jobs_repository.py | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py index 242528d002cd..2088e45e55c0 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py @@ -94,7 +94,7 @@ async def list_function_jobs( **pagination_kwargs, ) - async def validate_function_inputs( + async def validate_function_inputs( # pylint: disable=no-self-use self, *, function: RegisteredFunction, job_inputs: list[JobInputs] ) -> tuple[bool, str]: @@ -106,9 +106,9 @@ async def validate_function_inputs( if function.input_schema.schema_class == FunctionSchemaClass.json_schema: try: - for input in job_inputs: + for input_ in job_inputs: jsonschema.validate( - instance=input.values, + instance=input_.values, schema=function.input_schema.schema_content, ) except ValidationError as err: diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py index 8a93712f505e..2684c273686f 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py @@ -241,7 +241,7 @@ async def inspect_function_job( check_write_permissions=False, ) - async def function_job_outputs( + async def function_job_outputs( # pylint: disable=too-many-return-statements self, *, function: RegisteredFunction, diff --git a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py index 8a23f31a6748..c23fcfda3b81 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py @@ -46,7 +46,6 @@ ) from sqlalchemy import Text, cast, func from sqlalchemy.ext.asyncio import AsyncConnection -from sqlalchemy.sql import func from ..db.plugin import get_asyncpg_engine from ..groups.api import list_all_user_groups_ids From 54a004a487172749559c8a6dee6aed61bdca75e0 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 23:50:49 +0200 Subject: [PATCH 35/63] pylint --- .../functions/_functions_service.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py index 92b02cdcd4cf..639144702a40 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py @@ -442,7 +442,7 @@ def _map_db_model_to_domain_model( ), created_at=job.created, ) - elif job.function_class == FunctionClass.SOLVER: + if job.function_class == FunctionClass.SOLVER: return RegisteredSolverFunctionJob( uid=job.uuid, title=job.title, @@ -456,10 +456,7 @@ def _map_db_model_to_domain_model( ), created_at=job.created, ) - else: - raise UnsupportedFunctionJobClassError( - function_job_class=job.function_class - ) + raise UnsupportedFunctionJobClassError(function_job_class=job.function_class) return [_map_db_model_to_domain_model(job) for job in returned_function_jobs] From 5869282bc98d99d9e02d4c6850ed6d7923b79818 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 15 Oct 2025 23:59:14 +0200 Subject: [PATCH 36/63] pylint --- .../wb-api-server/test_function_jobs_controller_rpc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py index a5633c72b255..713e5c93f1bb 100644 --- a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py @@ -502,7 +502,7 @@ async def test_find_cached_function_jobs_with_status( user_id=logged_user["id"], product_name=osparc_product_name, ) - input = {"input1": 1.0} + input_ = {"input1": 1.0} for status in job_statuses: function_job = ProjectFunctionJob( @@ -510,7 +510,7 @@ async def test_find_cached_function_jobs_with_status( title="Test Function Job", description="A test function job", project_job_id=uuid4(), - inputs=input, + inputs=input_, outputs={"output1": "result1"}, job_creation_task_id=None, ) @@ -535,13 +535,13 @@ async def test_find_cached_function_jobs_with_status( function_id=registered_function.uid, product_name=osparc_product_name, user_id=logged_user["id"], - inputs=[input], + inputs=[input_], status_filter=[status], ) assert len(cached_jobs) == 1 cached_job = cached_jobs[0] assert cached_job is not None - assert cached_job.inputs == input + assert cached_job.inputs == input_ cached_job_status = await webserver_rpc_client.functions.get_function_job_status( product_name=osparc_product_name, function_job_id=cached_job.uid, From ee0000d00aa644f8bbccb77cebc29d6b599272c9 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 16 Oct 2025 00:05:52 +0200 Subject: [PATCH 37/63] pylint --- .../simcore_service_api_server/api/routes/functions_routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py index 15a9c31ab113..878ceef4fe20 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/functions_routes.py @@ -296,7 +296,7 @@ async def get_function_outputschema( ), ) async def validate_function_inputs( - function_id: FunctionID, + function_id: FunctionID, # pylint: disable=unused-argument inputs: FunctionInputs, function: Annotated[RegisteredFunction, Depends(get_function)], function_job_service: Annotated[ From 472147e9b48a4d99be2925093694009f5307db9d Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 16 Oct 2025 07:49:39 +0200 Subject: [PATCH 38/63] test fixes --- ...function_job_collections_controller_rpc.py | 6 ++++-- .../test_function_jobs_controller_rpc.py | 20 +++++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_job_collections_controller_rpc.py b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_job_collections_controller_rpc.py index 4686711dcc8b..2442dab85fbc 100644 --- a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_job_collections_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_job_collections_controller_rpc.py @@ -197,11 +197,13 @@ async def test_create_function_job_collection_same_function_job_uuid( job_creation_task_id=None, ) # Register the function job - registered_job = await webserver_rpc_client.functions.register_function_job( - function_job=registered_function_job, + registered_jobs = await webserver_rpc_client.functions.register_function_job( + function_jobs=[registered_function_job], user_id=logged_user["id"], product_name=osparc_product_name, ) + assert len(registered_jobs) == 1 + registered_job = registered_jobs[0] assert registered_job.uid is not None function_job_ids = [registered_job.uid] * 3 diff --git a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py index 713e5c93f1bb..2c7ec9b9c1a0 100644 --- a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py @@ -35,6 +35,7 @@ ) from models_library.products import ProductName from models_library.projects import ProjectID +from pydantic import TypeAdapter from pytest_simcore.helpers.webserver_users import UserInfoDict from servicelib.celery.models import TaskKey from servicelib.rabbitmq.rpc_interfaces.webserver.v1 import WebServerRpcClient @@ -646,7 +647,6 @@ async def test_patch_registered_function_jobs( registered_jobs = ( await webserver_rpc_client.functions.patch_registered_function_job( user_id=logged_user["id"], - function_job_uuid=registered_job.uid, product_name=osparc_product_name, registered_function_job_patch_inputs=patch_inputs, ) @@ -705,7 +705,7 @@ async def test_incompatible_patch_model_error( create_fake_function_obj: Callable[[FunctionClass], Function], clean_functions: None, function_job: RegisteredFunctionJob, - patch: RegisteredFunctionJobPatch, + patch: RegisteredProjectFunctionJobPatch | RegisteredSolverFunctionJobPatch, ): function = create_fake_function_obj(function_job.function_class) @@ -722,13 +722,25 @@ async def test_incompatible_patch_model_error( ) assert len(registered_jobs) == 1 registered_job = registered_jobs[0] + if function.function_class == FunctionClass.PROJECT: + assert isinstance(patch, RegisteredSolverFunctionJobPatch) + patch_input = RegisteredSolverFunctionJobPatchInput( + uid=registered_job.uid, patch=patch + ) + elif function.function_class == FunctionClass.SOLVER: + assert isinstance(patch, RegisteredProjectFunctionJobPatch) + patch_input = RegisteredProjectFunctionJobPatchInput( + uid=registered_job.uid, patch=patch + ) with pytest.raises(FunctionJobPatchModelIncompatibleError): registered_job = ( await webserver_rpc_client.functions.patch_registered_function_job( user_id=logged_user["id"], - function_job_uuid=registered_job.uid, product_name=osparc_product_name, - registered_function_job_patch=patch, + registered_function_job_patch_inputs=TypeAdapter( + RegisteredSolverFunctionJobPatchInputList + | RegisteredSolverFunctionJobPatchInputList + ).validate_python([patch_input]), ) ) From 786928f9eb8c0ee7dbb5f8893573f0378db24624 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 16 Oct 2025 07:58:39 +0200 Subject: [PATCH 39/63] pylint --- .../wb-api-server/test_function_jobs_controller_rpc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py index 2c7ec9b9c1a0..fdb727db4fa8 100644 --- a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py @@ -732,6 +732,8 @@ async def test_incompatible_patch_model_error( patch_input = RegisteredProjectFunctionJobPatchInput( uid=registered_job.uid, patch=patch ) + else: + pytest.fail("Unsupported function class") with pytest.raises(FunctionJobPatchModelIncompatibleError): registered_job = ( await webserver_rpc_client.functions.patch_registered_function_job( From d4385e5db28af2059d25e0fec7107a6984b72140 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 16 Oct 2025 08:08:16 +0200 Subject: [PATCH 40/63] pylint fix --- .../wb-api-server/test_function_jobs_controller_rpc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py index fdb727db4fa8..7a606c74cc56 100644 --- a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py @@ -722,18 +722,18 @@ async def test_incompatible_patch_model_error( ) assert len(registered_jobs) == 1 registered_job = registered_jobs[0] + patch_input = None if function.function_class == FunctionClass.PROJECT: assert isinstance(patch, RegisteredSolverFunctionJobPatch) patch_input = RegisteredSolverFunctionJobPatchInput( uid=registered_job.uid, patch=patch ) - elif function.function_class == FunctionClass.SOLVER: + if function.function_class == FunctionClass.SOLVER: assert isinstance(patch, RegisteredProjectFunctionJobPatch) patch_input = RegisteredProjectFunctionJobPatchInput( uid=registered_job.uid, patch=patch ) - else: - pytest.fail("Unsupported function class") + assert patch_input is not None with pytest.raises(FunctionJobPatchModelIncompatibleError): registered_job = ( await webserver_rpc_client.functions.patch_registered_function_job( From e1d43b688f3fa661f74f57613482c7f0f00fe907 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Fri, 17 Oct 2025 11:43:14 +0200 Subject: [PATCH 41/63] add register endpoint both for batch operation and for a single input --- .../rpc_interfaces/webserver/v1/functions.py | 20 +++++++++++++++++- .../functions/_controller/_functions_rpc.py | 21 ++++++++++++++++++- .../functions/_functions_service.py | 18 ++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py index afc60a234948..1a9512068358 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py @@ -20,6 +20,7 @@ FunctionClass, FunctionGroupAccessRights, FunctionInputsList, + FunctionJob, FunctionJobList, FunctionJobStatus, FunctionOutputs, @@ -316,6 +317,23 @@ async def run_function( ) async def register_function_job( + self, + *, + product_name: ProductName, + user_id: UserID, + function_job: FunctionJob, + ) -> RegisteredFunctionJob: + """Register a function job.""" + return TypeAdapter(RegisteredFunctionJob).validate_python( + await self._request( + "register_function_job", + product_name=product_name, + user_id=user_id, + function_job=function_job, + ), + ) + + async def register_function_job_batch( self, *, product_name: ProductName, @@ -325,7 +343,7 @@ async def register_function_job( """Register a function job.""" return TypeAdapter(RegisteredFunctionJobList).validate_python( await self._request( - "register_function_job", + "register_function_job_map", product_name=product_name, user_id=user_id, function_jobs=function_jobs, diff --git a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py index c3756717b404..eeb8b0bc7078 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py @@ -9,6 +9,7 @@ FunctionID, FunctionInputSchema, FunctionInputsList, + FunctionJob, FunctionJobCollection, FunctionJobCollectionID, FunctionJobCollectionsListFilters, @@ -91,13 +92,31 @@ async def register_function( ) ) async def register_function_job( + app: web.Application, + *, + user_id: UserID, + product_name: ProductName, + function_job: FunctionJob, +) -> RegisteredFunctionJob: + return await _functions_service.register_function_job( + app=app, user_id=user_id, product_name=product_name, function_job=function_job + ) + + +@router.expose( + reraise_if_error_type=( + UnsupportedFunctionJobClassError, + FunctionJobsWriteApiAccessDeniedError, + ) +) +async def register_function_job_batch( app: web.Application, *, user_id: UserID, product_name: ProductName, function_jobs: FunctionJobList, ) -> RegisteredFunctionJobList: - return await _functions_service.register_function_job( + return await _functions_service.register_function_job_batch( app=app, user_id=user_id, product_name=product_name, function_jobs=function_jobs ) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py index 639144702a40..38f10ab0a7ed 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py @@ -89,6 +89,24 @@ async def register_function( async def register_function_job( + app: web.Application, + *, + user_id: UserID, + product_name: ProductName, + function_job: FunctionJob, +) -> RegisteredFunctionJob: + encoded_function_jobs = _encode_functionjob(function_job) + created_function_jobs_db = await _function_jobs_repository.create_function_jobs( + app=app, + user_id=user_id, + product_name=product_name, + function_jobs=[encoded_function_jobs], + ) + assert len(created_function_jobs_db) == 1 # nosec + return _decode_functionjob(created_function_jobs_db[0]) + + +async def register_function_job_batch( app: web.Application, *, user_id: UserID, From 91fccb87780f08f416987986000aee4415acad87 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Fri, 17 Oct 2025 12:00:45 +0200 Subject: [PATCH 42/63] ensure webserver unit tests pass with new batch operation --- .../rpc_interfaces/webserver/v1/functions.py | 2 +- ...function_job_collections_controller_rpc.py | 18 +++-- .../test_function_jobs_controller_rpc.py | 71 ++++++++----------- 3 files changed, 41 insertions(+), 50 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py index 1a9512068358..e49eecc41647 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py @@ -343,7 +343,7 @@ async def register_function_job_batch( """Register a function job.""" return TypeAdapter(RegisteredFunctionJobList).validate_python( await self._request( - "register_function_job_map", + "register_function_job_batch", product_name=product_name, user_id=user_id, function_jobs=function_jobs, diff --git a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_job_collections_controller_rpc.py b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_job_collections_controller_rpc.py index 2442dab85fbc..f6986e01945a 100644 --- a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_job_collections_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_job_collections_controller_rpc.py @@ -69,7 +69,7 @@ async def test_function_job_collection( for _ in range(3) ] # Register the function jobs - registered_jobs = await webserver_rpc_client.functions.register_function_job( + registered_jobs = await webserver_rpc_client.functions.register_function_job_batch( function_jobs=TypeAdapter(FunctionJobList).validate_python(function_jobs), user_id=logged_user["id"], product_name=osparc_product_name, @@ -197,7 +197,7 @@ async def test_create_function_job_collection_same_function_job_uuid( job_creation_task_id=None, ) # Register the function job - registered_jobs = await webserver_rpc_client.functions.register_function_job( + registered_jobs = await webserver_rpc_client.functions.register_function_job_batch( function_jobs=[registered_function_job], user_id=logged_user["id"], product_name=osparc_product_name, @@ -271,7 +271,7 @@ async def test_list_function_job_collections( for _ in range(3) ] # Register the function jobs - registered_jobs = await webserver_rpc_client.functions.register_function_job( + registered_jobs = await webserver_rpc_client.functions.register_function_job_batch( function_jobs=TypeAdapter(FunctionJobList).validate_python(function_jobs), user_id=logged_user["id"], product_name=osparc_product_name, @@ -367,10 +367,14 @@ async def test_list_function_job_collections_filtered_function_id( for _ in range(3) ] # Register the function job - registered_jobs = await webserver_rpc_client.functions.register_function_job( - function_jobs=TypeAdapter(FunctionJobList).validate_python(function_jobs), - user_id=logged_user["id"], - product_name=osparc_product_name, + registered_jobs = ( + await webserver_rpc_client.functions.register_function_job_batch( + function_jobs=TypeAdapter(FunctionJobList).validate_python( + function_jobs + ), + user_id=logged_user["id"], + product_name=osparc_product_name, + ) ) assert all(job.uid for job in registered_jobs) function_job_ids = [job.uid for job in registered_jobs] diff --git a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py index 7a606c74cc56..7ecac56347e2 100644 --- a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py @@ -15,6 +15,7 @@ Function, FunctionClass, FunctionJobCollection, + FunctionJobList, FunctionJobStatus, RegisteredFunctionJob, RegisteredFunctionJobPatch, @@ -78,7 +79,7 @@ async def test_register_get_delete_function_job( ) # Register the function job - registered_jobs = await webserver_rpc_client.functions.register_function_job( + registered_jobs = await webserver_rpc_client.functions.register_function_job_batch( function_jobs=[function_job], user_id=logged_user["id"], product_name=osparc_product_name, @@ -198,7 +199,7 @@ async def test_list_function_jobs( ) # Register the function job - registered_jobs = await webserver_rpc_client.functions.register_function_job( + registered_jobs = await webserver_rpc_client.functions.register_function_job_batch( function_jobs=[function_job], user_id=logged_user["id"], product_name=osparc_product_name, @@ -250,13 +251,11 @@ async def test_list_function_jobs_with_status( ) # Register the function job - registered_jobs = await webserver_rpc_client.functions.register_function_job( - function_jobs=[function_job], + registered_job = await webserver_rpc_client.functions.register_function_job( + function_job=function_job, user_id=logged_user["id"], product_name=osparc_product_name, ) - assert len(registered_jobs) == 1 - registered_job = registered_jobs[0] # List function jobs jobs, _ = await webserver_rpc_client.functions.list_function_jobs_with_status( @@ -310,9 +309,9 @@ async def test_list_function_jobs_filtering( job_creation_task_id=None, ) # Register the function job - first_registered_function_jobs += ( + first_registered_function_jobs.append( await webserver_rpc_client.functions.register_function_job( - function_jobs=[function_job], + function_job=function_job, user_id=logged_user["id"], product_name=osparc_product_name, ) @@ -328,9 +327,9 @@ async def test_list_function_jobs_filtering( job_creation_task_id=None, ) # Register the function job - second_registered_function_jobs += ( + second_registered_function_jobs.append( await webserver_rpc_client.functions.register_function_job( - function_jobs=[function_job], + function_job=function_job, user_id=logged_user["id"], product_name=osparc_product_name, ) @@ -431,9 +430,8 @@ async def test_find_cached_function_jobs( product_name=osparc_product_name, ) - registered_function_jobs = [] - for value in range(5): - function_job = ProjectFunctionJob( + function_jobs = [ + ProjectFunctionJob( function_uid=registered_function.uid, title="Test Function Job", description="A test function job", @@ -442,14 +440,15 @@ async def test_find_cached_function_jobs( outputs={"output1": "result1"}, job_creation_task_id=None, ) + for value in range(5) + ] - # Register the function job - registered_jobs = await webserver_rpc_client.functions.register_function_job( - function_jobs=[function_job], - user_id=logged_user["id"], - product_name=osparc_product_name, - ) - registered_function_jobs += registered_jobs + # Register the function job + await webserver_rpc_client.functions.register_function_job_batch( + function_jobs=TypeAdapter(FunctionJobList).validate_python(function_jobs), + user_id=logged_user["id"], + product_name=osparc_product_name, + ) # Find cached function jobs cached_jobs = await webserver_rpc_client.functions.find_cached_function_jobs( @@ -517,13 +516,11 @@ async def test_find_cached_function_jobs_with_status( ) # Register the function job - registered_jobs = await webserver_rpc_client.functions.register_function_job( - function_jobs=[function_job], + registered_job = await webserver_rpc_client.functions.register_function_job( + function_job=function_job, user_id=logged_user["id"], product_name=osparc_product_name, ) - assert len(registered_jobs) == 1 - registered_job = registered_jobs[0] await webserver_rpc_client.functions.update_function_job_status( user_id=logged_user["id"], product_name=osparc_product_name, @@ -620,13 +617,11 @@ async def test_patch_registered_function_jobs( # Register the function job function_job.function_uid = registered_function.uid - registered_jobs = await webserver_rpc_client.functions.register_function_job( - function_jobs=[function_job], + registered_job = await webserver_rpc_client.functions.register_function_job( + function_job=function_job, user_id=logged_user["id"], product_name=osparc_product_name, ) - assert len(registered_jobs) == 1 - registered_job = registered_jobs[0] patch_inputs: ( RegisteredProjectFunctionJobPatchInputList | RegisteredSolverFunctionJobPatchInputList @@ -715,13 +710,11 @@ async def test_incompatible_patch_model_error( product_name=osparc_product_name, ) function_job.function_uid = registered_function.uid - registered_jobs = await webserver_rpc_client.functions.register_function_job( - function_jobs=[function_job], + registered_job = await webserver_rpc_client.functions.register_function_job( + function_job=function_job, user_id=logged_user["id"], product_name=osparc_product_name, ) - assert len(registered_jobs) == 1 - registered_job = registered_jobs[0] patch_input = None if function.function_class == FunctionClass.PROJECT: assert isinstance(patch, RegisteredSolverFunctionJobPatch) @@ -790,14 +783,11 @@ async def test_update_function_job_status_output( ) # Register the function job - registered_jobs = await webserver_rpc_client.functions.register_function_job( - function_jobs=[function_job], + registered_job = await webserver_rpc_client.functions.register_function_job( + function_job=function_job, user_id=logged_user["id"], product_name=osparc_product_name, ) - assert len(registered_jobs) == 1 - registered_job = registered_jobs[0] - old_job_status = await webserver_rpc_client.functions.get_function_job_status( function_job_id=registered_job.uid, user_id=logged_user["id"], @@ -882,14 +872,11 @@ async def test_update_function_job_outputs( ) # Register the function job - registered_jobs = await webserver_rpc_client.functions.register_function_job( - function_jobs=[function_job], + registered_job = await webserver_rpc_client.functions.register_function_job( + function_job=function_job, user_id=logged_user["id"], product_name=osparc_product_name, ) - assert len(registered_jobs) == 1 - registered_job = registered_jobs[0] - received_outputs = await webserver_rpc_client.functions.get_function_job_outputs( function_job_id=registered_job.uid, user_id=logged_user["id"], From 38026e394beb3b3727635baf1df26ea89d7e430e Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Mon, 20 Oct 2025 11:07:31 +0200 Subject: [PATCH 43/63] use batch models/schemas in register function job rpc endpoints --- .../src/models_library/batch_operations.py | 17 +++++++++++++++++ .../src/models_library/functions.py | 15 +++++++++++---- .../functions/_function_jobs_repository.py | 5 +++-- .../functions/_functions_service.py | 15 ++++++++++----- 4 files changed, 41 insertions(+), 11 deletions(-) diff --git a/packages/models-library/src/models_library/batch_operations.py b/packages/models-library/src/models_library/batch_operations.py index a757ea45a447..b78c32a492f6 100644 --- a/packages/models-library/src/models_library/batch_operations.py +++ b/packages/models-library/src/models_library/batch_operations.py @@ -85,3 +85,20 @@ class BatchGetEnvelope(BaseModel, Generic[ResourceT, IdentifierT]): description="List of identifiers for items that were not found", ), ] = DEFAULT_FACTORY + + +class BatchCreateEnvelope(BaseModel, Generic[SchemaT]): + """Generic envelope model for batch-create operations. + + This model represents the result of a strict batch create operation, + containing the list of created items. The operation is expected to be "strict" + in the sense that it either creates all requested items or fails entirely. + """ + + created_items: Annotated[ + list[SchemaT], + Field( + min_length=1, + description="List of successfully created items", + ), + ] diff --git a/packages/models-library/src/models_library/functions.py b/packages/models-library/src/models_library/functions.py index 6d38eba121e5..bdd24a6d4d38 100644 --- a/packages/models-library/src/models_library/functions.py +++ b/packages/models-library/src/models_library/functions.py @@ -7,6 +7,7 @@ from models_library import projects from models_library.basic_regex import UUID_RE_BASE from models_library.basic_types import ConstrainedStr +from models_library.batch_operations import BatchCreateEnvelope from models_library.groups import GroupID from models_library.products import ProductName from models_library.services_types import ServiceKey, ServiceVersion @@ -269,10 +270,12 @@ class RegisteredPythonCodeFunctionJob(PythonCodeFunctionJob, RegisteredFunctionJ | RegisteredSolverFunctionJob, Field(discriminator="function_class"), ] -RegisteredFunctionJobList: TypeAlias = Annotated[ - list[RegisteredFunctionJob], - Field(max_length=_MAX_LIST_LENGTH, min_length=_MIN_LIST_LENGTH), -] + + +class BatchCreateRegisteredFunctionJobs(BatchCreateEnvelope[RegisteredFunctionJob]): + """Envelope model for batch registering function jobs""" + + pass RegisteredFunctionJobPatch = Annotated[ @@ -373,6 +376,10 @@ class RegisteredFunctionJobDB(FunctionJobDB): created: datetime.datetime +class BatchCreateRegisteredFunctionJobsDB(BatchCreateEnvelope[RegisteredFunctionJobDB]): + pass + + class RegisteredFunctionJobWithStatusDB(FunctionJobDB): uuid: FunctionJobID created: datetime.datetime diff --git a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py index c23fcfda3b81..6f6fa7b87e43 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py @@ -6,6 +6,7 @@ import sqlalchemy from aiohttp import web from models_library.functions import ( + BatchCreateRegisteredFunctionJobsDB, FunctionClass, FunctionClassSpecificData, FunctionID, @@ -69,7 +70,7 @@ async def create_function_jobs( # noqa: PLR0913 user_id: UserID, product_name: ProductName, function_jobs: list[FunctionJobDB], -) -> list[RegisteredFunctionJobDB]: +) -> BatchCreateRegisteredFunctionJobsDB: async with transaction_context(get_asyncpg_engine(app), connection) as transaction: await check_user_api_access_rights( app, @@ -126,7 +127,7 @@ async def create_function_jobs( # noqa: PLR0913 execute=True, ) - return created_jobs + return BatchCreateRegisteredFunctionJobsDB(created_items=created_jobs) async def patch_function_job( diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py index 38f10ab0a7ed..dee7a2fb3ed6 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py @@ -3,6 +3,7 @@ from aiohttp import web from models_library.basic_types import IDStr from models_library.functions import ( + BatchCreateRegisteredFunctionJobs, Function, FunctionClass, FunctionClassSpecificData, @@ -30,7 +31,6 @@ RegisteredFunctionJob, RegisteredFunctionJobCollection, RegisteredFunctionJobDB, - RegisteredFunctionJobList, RegisteredFunctionJobWithStatus, RegisteredFunctionJobWithStatusDB, RegisteredProjectFunction, @@ -102,8 +102,9 @@ async def register_function_job( product_name=product_name, function_jobs=[encoded_function_jobs], ) - assert len(created_function_jobs_db) == 1 # nosec - return _decode_functionjob(created_function_jobs_db[0]) + created_items = created_function_jobs_db.created_items + assert len(created_items) == 1 # nosec + return _decode_functionjob(created_items[0]) async def register_function_job_batch( @@ -112,7 +113,7 @@ async def register_function_job_batch( user_id: UserID, product_name: ProductName, function_jobs: FunctionJobList, -) -> RegisteredFunctionJobList: +) -> BatchCreateRegisteredFunctionJobs: TypeAdapter(FunctionJobList).validate_python(function_jobs) encoded_function_jobs = [_encode_functionjob(job) for job in function_jobs] created_function_jobs_db = await _function_jobs_repository.create_function_jobs( @@ -121,7 +122,11 @@ async def register_function_job_batch( product_name=product_name, function_jobs=encoded_function_jobs, ) - return [_decode_functionjob(job) for job in created_function_jobs_db] + return BatchCreateRegisteredFunctionJobs( + created_items=[ + _decode_functionjob(job) for job in created_function_jobs_db.created_items + ] + ) async def patch_registered_function_job( From 7b38b4d6ee81a788195f8ca51f69ba753d3dd6df Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Mon, 20 Oct 2025 11:25:23 +0200 Subject: [PATCH 44/63] fix tests --- .../rpc_interfaces/webserver/v1/functions.py | 6 +-- .../functions/_controller/_functions_rpc.py | 4 +- ...function_job_collections_controller_rpc.py | 38 ++++++++++++------- .../test_function_jobs_controller_rpc.py | 32 ++++++++++------ 4 files changed, 50 insertions(+), 30 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py index e49eecc41647..4c62defa40b3 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py @@ -17,6 +17,7 @@ RegisteredFunctionJobCollection, ) from models_library.functions import ( + BatchCreateRegisteredFunctionJobs, FunctionClass, FunctionGroupAccessRights, FunctionInputsList, @@ -26,7 +27,6 @@ FunctionOutputs, FunctionUserAccessRights, FunctionUserApiAccessRights, - RegisteredFunctionJobList, RegisteredFunctionJobWithStatus, RegisteredProjectFunctionJobPatchInputList, RegisteredSolverFunctionJobPatchInputList, @@ -339,9 +339,9 @@ async def register_function_job_batch( product_name: ProductName, user_id: UserID, function_jobs: FunctionJobList, - ) -> RegisteredFunctionJobList: + ) -> BatchCreateRegisteredFunctionJobs: """Register a function job.""" - return TypeAdapter(RegisteredFunctionJobList).validate_python( + return TypeAdapter(BatchCreateRegisteredFunctionJobs).validate_python( await self._request( "register_function_job_batch", product_name=product_name, diff --git a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py index eeb8b0bc7078..eb5c880f6c06 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py @@ -2,6 +2,7 @@ from aiohttp import web from models_library.functions import ( + BatchCreateRegisteredFunctionJobs, Function, FunctionAccessRights, FunctionClass, @@ -23,7 +24,6 @@ RegisteredFunction, RegisteredFunctionJob, RegisteredFunctionJobCollection, - RegisteredFunctionJobList, RegisteredFunctionJobWithStatus, RegisteredProjectFunctionJobPatchInputList, RegisteredSolverFunctionJobPatchInputList, @@ -115,7 +115,7 @@ async def register_function_job_batch( user_id: UserID, product_name: ProductName, function_jobs: FunctionJobList, -) -> RegisteredFunctionJobList: +) -> BatchCreateRegisteredFunctionJobs: return await _functions_service.register_function_job_batch( app=app, user_id=user_id, product_name=product_name, function_jobs=function_jobs ) diff --git a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_job_collections_controller_rpc.py b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_job_collections_controller_rpc.py index f6986e01945a..58f7f4a3e22f 100644 --- a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_job_collections_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_job_collections_controller_rpc.py @@ -69,11 +69,14 @@ async def test_function_job_collection( for _ in range(3) ] # Register the function jobs - registered_jobs = await webserver_rpc_client.functions.register_function_job_batch( - function_jobs=TypeAdapter(FunctionJobList).validate_python(function_jobs), - user_id=logged_user["id"], - product_name=osparc_product_name, + registered_jobs_batch_create = ( + await webserver_rpc_client.functions.register_function_job_batch( + function_jobs=TypeAdapter(FunctionJobList).validate_python(function_jobs), + user_id=logged_user["id"], + product_name=osparc_product_name, + ) ) + registered_jobs = registered_jobs_batch_create.created_items assert len(registered_jobs) == 3 assert all(job.uid is not None for job in registered_jobs) function_job_ids = [job.uid for job in registered_jobs] @@ -197,11 +200,14 @@ async def test_create_function_job_collection_same_function_job_uuid( job_creation_task_id=None, ) # Register the function job - registered_jobs = await webserver_rpc_client.functions.register_function_job_batch( - function_jobs=[registered_function_job], - user_id=logged_user["id"], - product_name=osparc_product_name, + registered_jobs_batch_create = ( + await webserver_rpc_client.functions.register_function_job_batch( + function_jobs=[registered_function_job], + user_id=logged_user["id"], + product_name=osparc_product_name, + ) ) + registered_jobs = registered_jobs_batch_create.created_items assert len(registered_jobs) == 1 registered_job = registered_jobs[0] assert registered_job.uid is not None @@ -271,11 +277,15 @@ async def test_list_function_job_collections( for _ in range(3) ] # Register the function jobs - registered_jobs = await webserver_rpc_client.functions.register_function_job_batch( - function_jobs=TypeAdapter(FunctionJobList).validate_python(function_jobs), - user_id=logged_user["id"], - product_name=osparc_product_name, + registered_jobs_batch_create = ( + await webserver_rpc_client.functions.register_function_job_batch( + function_jobs=TypeAdapter(FunctionJobList).validate_python(function_jobs), + user_id=logged_user["id"], + product_name=osparc_product_name, + ) ) + registered_jobs = registered_jobs_batch_create.created_items + assert len(registered_jobs) == 3 assert all(job.uid is not None for job in registered_jobs) function_job_collection = FunctionJobCollection( @@ -367,7 +377,7 @@ async def test_list_function_job_collections_filtered_function_id( for _ in range(3) ] # Register the function job - registered_jobs = ( + registered_jobs_batch_create = ( await webserver_rpc_client.functions.register_function_job_batch( function_jobs=TypeAdapter(FunctionJobList).validate_python( function_jobs @@ -376,6 +386,8 @@ async def test_list_function_job_collections_filtered_function_id( product_name=osparc_product_name, ) ) + registered_jobs = registered_jobs_batch_create.created_items + assert len(registered_jobs) == 3 assert all(job.uid for job in registered_jobs) function_job_ids = [job.uid for job in registered_jobs] diff --git a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py index 7ecac56347e2..7a73599278a6 100644 --- a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py @@ -79,11 +79,14 @@ async def test_register_get_delete_function_job( ) # Register the function job - registered_jobs = await webserver_rpc_client.functions.register_function_job_batch( - function_jobs=[function_job], - user_id=logged_user["id"], - product_name=osparc_product_name, + registered_jobs_batch_create = ( + await webserver_rpc_client.functions.register_function_job_batch( + function_jobs=[function_job], + user_id=logged_user["id"], + product_name=osparc_product_name, + ) ) + registered_jobs = registered_jobs_batch_create.created_items assert len(registered_jobs) == 1 registered_job = registered_jobs[0] @@ -199,11 +202,14 @@ async def test_list_function_jobs( ) # Register the function job - registered_jobs = await webserver_rpc_client.functions.register_function_job_batch( - function_jobs=[function_job], - user_id=logged_user["id"], - product_name=osparc_product_name, + registered_jobs_batch_create = ( + await webserver_rpc_client.functions.register_function_job_batch( + function_jobs=[function_job], + user_id=logged_user["id"], + product_name=osparc_product_name, + ) ) + registered_jobs = registered_jobs_batch_create.created_items assert len(registered_jobs) == 1 registered_job = registered_jobs[0] @@ -444,10 +450,12 @@ async def test_find_cached_function_jobs( ] # Register the function job - await webserver_rpc_client.functions.register_function_job_batch( - function_jobs=TypeAdapter(FunctionJobList).validate_python(function_jobs), - user_id=logged_user["id"], - product_name=osparc_product_name, + registered_jobs_batch_create = ( + await webserver_rpc_client.functions.register_function_job_batch( + function_jobs=TypeAdapter(FunctionJobList).validate_python(function_jobs), + user_id=logged_user["id"], + product_name=osparc_product_name, + ) ) # Find cached function jobs From 94a7052706a601b01290bc1393d76a1904428b43 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Mon, 20 Oct 2025 11:33:58 +0200 Subject: [PATCH 45/63] start using batch register function jobs in api-server --- .../rpc_interfaces/webserver/v1/functions.py | 2 +- .../_service_function_jobs.py | 30 +++++++++++-------- .../api/routes/function_jobs_routes.py | 7 ++--- .../services_rpc/wb_api_server.py | 20 +++++++++++-- ...function_job_collections_controller_rpc.py | 8 ++--- .../test_function_jobs_controller_rpc.py | 6 ++-- 6 files changed, 46 insertions(+), 27 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py index 4c62defa40b3..9a1d59e64957 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py @@ -333,7 +333,7 @@ async def register_function_job( ), ) - async def register_function_job_batch( + async def batch_register_function_jobs( self, *, product_name: ProductName, diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py index 2088e45e55c0..4604b0091108 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py @@ -163,13 +163,16 @@ async def pre_register_function_job( ) for input_ in job_inputs ] - jobs = await self._web_rpc_client.register_function_job( - function_jobs=TypeAdapter(FunctionJobList).validate_python( - function_jobs - ), - user_id=self.user_id, - product_name=self.product_name, + batch_registered_jobs = ( + await self._web_rpc_client.batch_register_function_jobs( + function_jobs=TypeAdapter(FunctionJobList).validate_python( + function_jobs + ), + user_id=self.user_id, + product_name=self.product_name, + ) ) + jobs = batch_registered_jobs.created_items elif function.function_class == FunctionClass.SOLVER: function_jobs = [ @@ -184,13 +187,16 @@ async def pre_register_function_job( ) for input_ in job_inputs ] - jobs = await self._web_rpc_client.register_function_job( - function_jobs=TypeAdapter(FunctionJobList).validate_python( - function_jobs - ), - user_id=self.user_id, - product_name=self.product_name, + batch_registered_jobs = ( + await self._web_rpc_client.batch_register_function_jobs( + function_jobs=TypeAdapter(FunctionJobList).validate_python( + function_jobs + ), + user_id=self.user_id, + product_name=self.product_name, + ) ) + jobs = batch_registered_jobs.created_items else: raise UnsupportedFunctionClassError( function_class=function.function_class, diff --git a/services/api-server/src/simcore_service_api_server/api/routes/function_jobs_routes.py b/services/api-server/src/simcore_service_api_server/api/routes/function_jobs_routes.py index 44b909c9d025..0ce21bfd5d9a 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/function_jobs_routes.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/function_jobs_routes.py @@ -172,11 +172,10 @@ async def register_function_job( user_id: Annotated[UserID, Depends(get_current_user_id)], product_name: Annotated[ProductName, Depends(get_product_name)], ) -> RegisteredFunctionJob: - registered_jobs = await wb_api_rpc.register_function_job( - function_jobs=[function_job], user_id=user_id, product_name=product_name + registered_job = await wb_api_rpc.register_function_job( + function_job=function_job, user_id=user_id, product_name=product_name ) - assert len(registered_jobs) == 1 # nosec - return registered_jobs[0] + return registered_job @function_job_router.get( diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index a4a38cbfd405..be7c024510e7 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -23,13 +23,14 @@ ) from models_library.api_schemas_webserver.licensed_items import LicensedItemRpcGetPage from models_library.functions import ( + BatchCreateRegisteredFunctionJobs, FunctionInputsList, + FunctionJob, FunctionJobList, FunctionJobStatus, FunctionOutputs, FunctionUserAccessRights, FunctionUserApiAccessRights, - RegisteredFunctionJobList, RegisteredFunctionJobWithStatus, RegisteredProjectFunctionJobPatchInputList, RegisteredSolverFunctionJobPatchInputList, @@ -487,10 +488,23 @@ async def register_function_job( self, *, user_id: UserID, - function_jobs: FunctionJobList, + function_job: FunctionJob, product_name: ProductName, - ) -> RegisteredFunctionJobList: + ) -> RegisteredFunctionJob: return await self._rpc_client.functions.register_function_job( + user_id=user_id, + product_name=product_name, + function_job=function_job, + ) + + async def batch_register_function_jobs( + self, + *, + user_id: UserID, + function_jobs: FunctionJobList, + product_name: ProductName, + ) -> BatchCreateRegisteredFunctionJobs: + return await self._rpc_client.functions.batch_register_function_jobs( user_id=user_id, product_name=product_name, function_jobs=function_jobs, diff --git a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_job_collections_controller_rpc.py b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_job_collections_controller_rpc.py index 58f7f4a3e22f..2cb6dfc9a0ad 100644 --- a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_job_collections_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_job_collections_controller_rpc.py @@ -70,7 +70,7 @@ async def test_function_job_collection( ] # Register the function jobs registered_jobs_batch_create = ( - await webserver_rpc_client.functions.register_function_job_batch( + await webserver_rpc_client.functions.batch_register_function_jobs( function_jobs=TypeAdapter(FunctionJobList).validate_python(function_jobs), user_id=logged_user["id"], product_name=osparc_product_name, @@ -201,7 +201,7 @@ async def test_create_function_job_collection_same_function_job_uuid( ) # Register the function job registered_jobs_batch_create = ( - await webserver_rpc_client.functions.register_function_job_batch( + await webserver_rpc_client.functions.batch_register_function_jobs( function_jobs=[registered_function_job], user_id=logged_user["id"], product_name=osparc_product_name, @@ -278,7 +278,7 @@ async def test_list_function_job_collections( ] # Register the function jobs registered_jobs_batch_create = ( - await webserver_rpc_client.functions.register_function_job_batch( + await webserver_rpc_client.functions.batch_register_function_jobs( function_jobs=TypeAdapter(FunctionJobList).validate_python(function_jobs), user_id=logged_user["id"], product_name=osparc_product_name, @@ -378,7 +378,7 @@ async def test_list_function_job_collections_filtered_function_id( ] # Register the function job registered_jobs_batch_create = ( - await webserver_rpc_client.functions.register_function_job_batch( + await webserver_rpc_client.functions.batch_register_function_jobs( function_jobs=TypeAdapter(FunctionJobList).validate_python( function_jobs ), diff --git a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py index 7a73599278a6..22125bd5a265 100644 --- a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py @@ -80,7 +80,7 @@ async def test_register_get_delete_function_job( # Register the function job registered_jobs_batch_create = ( - await webserver_rpc_client.functions.register_function_job_batch( + await webserver_rpc_client.functions.batch_register_function_jobs( function_jobs=[function_job], user_id=logged_user["id"], product_name=osparc_product_name, @@ -203,7 +203,7 @@ async def test_list_function_jobs( # Register the function job registered_jobs_batch_create = ( - await webserver_rpc_client.functions.register_function_job_batch( + await webserver_rpc_client.functions.batch_register_function_jobs( function_jobs=[function_job], user_id=logged_user["id"], product_name=osparc_product_name, @@ -451,7 +451,7 @@ async def test_find_cached_function_jobs( # Register the function job registered_jobs_batch_create = ( - await webserver_rpc_client.functions.register_function_job_batch( + await webserver_rpc_client.functions.batch_register_function_jobs( function_jobs=TypeAdapter(FunctionJobList).validate_python(function_jobs), user_id=logged_user["id"], product_name=osparc_product_name, From 809964be7f19697b4dabc11e0ce11ac0759ff241 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Mon, 20 Oct 2025 12:13:15 +0200 Subject: [PATCH 46/63] clean up --- .../rabbitmq/rpc_interfaces/webserver/v1/functions.py | 2 +- .../src/simcore_service_api_server/_service_function_jobs.py | 2 +- .../_service_function_jobs_task_client.py | 2 +- .../functions/_controller/_functions_rpc.py | 4 ++-- .../simcore_service_webserver/functions/_functions_service.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py index 9a1d59e64957..5df23995d9e8 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py @@ -343,7 +343,7 @@ async def batch_register_function_jobs( """Register a function job.""" return TypeAdapter(BatchCreateRegisteredFunctionJobs).validate_python( await self._request( - "register_function_job_batch", + "batch_register_function_jobs", product_name=product_name, user_id=user_id, function_jobs=function_jobs, diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py index 4604b0091108..e5cf1a37be58 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py @@ -134,7 +134,7 @@ def create_function_job_inputs( # pylint: disable=no-self-use values=joined_inputs or {}, ) - async def pre_register_function_job( + async def pre_register_function_jobs( self, *, function: RegisteredFunction, diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py index 2684c273686f..377ed00575a0 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py @@ -340,7 +340,7 @@ async def create_function_job_creation_tasks( ] pre_registered_function_job_data_list = ( - await self._function_job_service.pre_register_function_job( + await self._function_job_service.pre_register_function_jobs( function=function, job_inputs=uncached_inputs, ) diff --git a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py index eb5c880f6c06..e12084ba8265 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py @@ -109,14 +109,14 @@ async def register_function_job( FunctionJobsWriteApiAccessDeniedError, ) ) -async def register_function_job_batch( +async def batch_register_function_jobs( app: web.Application, *, user_id: UserID, product_name: ProductName, function_jobs: FunctionJobList, ) -> BatchCreateRegisteredFunctionJobs: - return await _functions_service.register_function_job_batch( + return await _functions_service.batch_register_function_jobs( app=app, user_id=user_id, product_name=product_name, function_jobs=function_jobs ) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py index dee7a2fb3ed6..f6ed325c18f3 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py @@ -107,7 +107,7 @@ async def register_function_job( return _decode_functionjob(created_items[0]) -async def register_function_job_batch( +async def batch_register_function_jobs( app: web.Application, *, user_id: UserID, From 7a68768be41ec9fc5165c3cd24389c901be8769e Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Mon, 20 Oct 2025 12:13:45 +0200 Subject: [PATCH 47/63] improve patch model schema --- .../src/models_library/functions.py | 21 ++----------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/packages/models-library/src/models_library/functions.py b/packages/models-library/src/models_library/functions.py index bdd24a6d4d38..e7b286a679a0 100644 --- a/packages/models-library/src/models_library/functions.py +++ b/packages/models-library/src/models_library/functions.py @@ -286,26 +286,9 @@ class BatchCreateRegisteredFunctionJobs(BatchCreateEnvelope[RegisteredFunctionJo ] -class RegisteredProjectFunctionJobPatchInput(BaseModel): +class FunctionJobPatchRequest(BaseModel): uid: FunctionJobID - patch: RegisteredProjectFunctionJobPatch - - -RegisteredProjectFunctionJobPatchInputList: TypeAlias = Annotated[ - list[RegisteredProjectFunctionJobPatchInput], - Field(max_length=_MAX_LIST_LENGTH, min_length=_MIN_LIST_LENGTH), -] - - -class RegisteredSolverFunctionJobPatchInput(BaseModel): - uid: FunctionJobID - patch: RegisteredSolverFunctionJobPatch - - -RegisteredSolverFunctionJobPatchInputList: TypeAlias = Annotated[ - list[RegisteredSolverFunctionJobPatchInput], - Field(max_length=_MAX_LIST_LENGTH, min_length=_MIN_LIST_LENGTH), -] + patch: RegisteredFunctionJobPatch class FunctionJobStatus(BaseModel): From 97993f2fa73296f169a43c396b5527afecce45bd Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Mon, 20 Oct 2025 15:07:54 +0200 Subject: [PATCH 48/63] refactor patch method --- .../src/models_library/batch_operations.py | 17 ++++++ .../src/models_library/functions.py | 19 ++++++- .../rpc_interfaces/webserver/v1/functions.py | 33 ++++++++---- .../functions/_controller/_functions_rpc.py | 37 ++++++++++--- .../functions/_function_jobs_repository.py | 40 ++++++-------- .../functions/_functions_service.py | 41 ++++++++++---- .../test_function_jobs_controller_rpc.py | 54 ++++--------------- 7 files changed, 144 insertions(+), 97 deletions(-) diff --git a/packages/models-library/src/models_library/batch_operations.py b/packages/models-library/src/models_library/batch_operations.py index b78c32a492f6..ff0b34b687a4 100644 --- a/packages/models-library/src/models_library/batch_operations.py +++ b/packages/models-library/src/models_library/batch_operations.py @@ -102,3 +102,20 @@ class BatchCreateEnvelope(BaseModel, Generic[SchemaT]): description="List of successfully created items", ), ] + + +class BatchUpdateEnvelope(BaseModel, Generic[SchemaT]): + """Generic envelope model for batch-update operations. + + This model represents the result of a strict batch update operation, + containing the list of updated items. The operation is expected to be "strict" + in the sense that it either updates all requested items or fails entirely. See https://google.aip.dev/234 + """ + + updated_items: Annotated[ + list[SchemaT], + Field( + min_length=1, + description="List of successfully updated items", + ), + ] diff --git a/packages/models-library/src/models_library/functions.py b/packages/models-library/src/models_library/functions.py index e7b286a679a0..cf25ca4c888d 100644 --- a/packages/models-library/src/models_library/functions.py +++ b/packages/models-library/src/models_library/functions.py @@ -15,6 +15,7 @@ from models_library.utils.enums import StrAutoEnum from pydantic import BaseModel, ConfigDict, Field +from .batch_operations import BatchUpdateEnvelope from .projects import ProjectID from .utils.change_case import snake_to_camel @@ -273,8 +274,10 @@ class RegisteredPythonCodeFunctionJob(PythonCodeFunctionJob, RegisteredFunctionJ class BatchCreateRegisteredFunctionJobs(BatchCreateEnvelope[RegisteredFunctionJob]): - """Envelope model for batch registering function jobs""" + pass + +class BatchUpdateRegisteredFunctionJobs(BatchUpdateEnvelope[RegisteredFunctionJob]): pass @@ -291,6 +294,16 @@ class FunctionJobPatchRequest(BaseModel): patch: RegisteredFunctionJobPatch +FunctionJobPatchRequestList: TypeAlias = Annotated[ + list[FunctionJobPatchRequest], + Field( + max_length=_MAX_LIST_LENGTH, + min_length=_MIN_LIST_LENGTH, + description="List of function job patch requests", + ), +] + + class FunctionJobStatus(BaseModel): status: str @@ -363,6 +376,10 @@ class BatchCreateRegisteredFunctionJobsDB(BatchCreateEnvelope[RegisteredFunction pass +class BatchUpdateRegisteredFunctionJobsDB(BatchUpdateEnvelope[RegisteredFunctionJobDB]): + pass + + class RegisteredFunctionJobWithStatusDB(FunctionJobDB): uuid: FunctionJobID created: datetime.datetime diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py index 5df23995d9e8..d12fdb5531ce 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py @@ -18,18 +18,19 @@ ) from models_library.functions import ( BatchCreateRegisteredFunctionJobs, + BatchUpdateRegisteredFunctionJobs, FunctionClass, FunctionGroupAccessRights, FunctionInputsList, FunctionJob, FunctionJobList, + FunctionJobPatchRequest, + FunctionJobPatchRequestList, FunctionJobStatus, FunctionOutputs, FunctionUserAccessRights, FunctionUserApiAccessRights, RegisteredFunctionJobWithStatus, - RegisteredProjectFunctionJobPatchInputList, - RegisteredSolverFunctionJobPatchInputList, ) from models_library.products import ProductName from models_library.rabbitmq_basic_types import RPCNamespace @@ -355,18 +356,32 @@ async def patch_registered_function_job( *, product_name: ProductName, user_id: UserID, - registered_function_job_patch_inputs: ( - RegisteredProjectFunctionJobPatchInputList - | RegisteredSolverFunctionJobPatchInputList - ), - ) -> list[RegisteredFunctionJob]: + function_job_patch_request: FunctionJobPatchRequest, + ) -> RegisteredFunctionJob: + """Patch a registered function job.""" + return TypeAdapter(RegisteredFunctionJob).validate_python( + await self._request( + "patch_registered_function_job", + product_name=product_name, + user_id=user_id, + function_job_patch_request=function_job_patch_request, + ), + ) + + async def batch_patch_registered_function_job( + self, + *, + product_name: ProductName, + user_id: UserID, + function_job_patch_requests: FunctionJobPatchRequestList, + ) -> BatchUpdateRegisteredFunctionJobs: """Patch a registered function job.""" - return TypeAdapter(list[RegisteredFunctionJob]).validate_python( + return BatchUpdateRegisteredFunctionJobs.model_validate( await self._request( "patch_registered_function_job", product_name=product_name, user_id=user_id, - registered_function_job_patch_inputs=registered_function_job_patch_inputs, + function_job_patch_requests=function_job_patch_requests, ), ) diff --git a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py index e12084ba8265..4ac1c419ea5f 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py @@ -3,6 +3,7 @@ from aiohttp import web from models_library.functions import ( BatchCreateRegisteredFunctionJobs, + BatchUpdateRegisteredFunctionJobs, Function, FunctionAccessRights, FunctionClass, @@ -16,6 +17,8 @@ FunctionJobCollectionsListFilters, FunctionJobID, FunctionJobList, + FunctionJobPatchRequest, + FunctionJobPatchRequestList, FunctionJobStatus, FunctionOutputs, FunctionOutputSchema, @@ -25,8 +28,6 @@ RegisteredFunctionJob, RegisteredFunctionJobCollection, RegisteredFunctionJobWithStatus, - RegisteredProjectFunctionJobPatchInputList, - RegisteredSolverFunctionJobPatchInputList, ) from models_library.functions_errors import ( FunctionIDNotFoundError, @@ -133,17 +134,37 @@ async def patch_registered_function_job( *, user_id: UserID, product_name: ProductName, - registered_function_job_patch_inputs: ( - RegisteredProjectFunctionJobPatchInputList - | RegisteredSolverFunctionJobPatchInputList - ), -) -> list[RegisteredFunctionJob]: + function_job_patch_request: FunctionJobPatchRequest, +) -> RegisteredFunctionJob: return await _functions_service.patch_registered_function_job( app=app, user_id=user_id, product_name=product_name, - registered_function_job_patch_inputs=registered_function_job_patch_inputs, + function_job_patch_request=function_job_patch_request, + ) + + +@router.expose( + reraise_if_error_type=( + UnsupportedFunctionJobClassError, + FunctionJobsWriteApiAccessDeniedError, + FunctionJobPatchModelIncompatibleError, + ) +) +async def batch_patch_registered_function_jobs( + app: web.Application, + *, + user_id: UserID, + product_name: ProductName, + function_job_patch_requests: FunctionJobPatchRequestList, +) -> BatchUpdateRegisteredFunctionJobs: + + return await _functions_service.batch_patch_registered_function_jobs( + app=app, + user_id=user_id, + product_name=product_name, + function_job_patch_requests=function_job_patch_requests, ) diff --git a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py index 6f6fa7b87e43..7a674fb5092e 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py @@ -7,6 +7,7 @@ from aiohttp import web from models_library.functions import ( BatchCreateRegisteredFunctionJobsDB, + BatchUpdateRegisteredFunctionJobsDB, FunctionClass, FunctionClassSpecificData, FunctionID, @@ -14,14 +15,13 @@ FunctionJobCollectionID, FunctionJobDB, FunctionJobID, + FunctionJobPatchRequest, FunctionJobStatus, FunctionOutputs, FunctionsApiAccessRights, RegisteredFunctionJobDB, RegisteredFunctionJobPatch, RegisteredFunctionJobWithStatusDB, - RegisteredProjectFunctionJobPatchInputList, - RegisteredSolverFunctionJobPatchInputList, ) from models_library.functions_errors import ( FunctionJobIDNotFoundError, @@ -130,24 +130,14 @@ async def create_function_jobs( # noqa: PLR0913 return BatchCreateRegisteredFunctionJobsDB(created_items=created_jobs) -async def patch_function_job( +async def patch_function_jobs( app: web.Application, connection: AsyncConnection | None = None, *, user_id: UserID, product_name: ProductName, - registered_function_job_patch_inputs: ( - RegisteredProjectFunctionJobPatchInputList - | RegisteredSolverFunctionJobPatchInputList - ), -) -> list[RegisteredFunctionJobDB]: - - # check only a single function class is used - TypeAdapter( - RegisteredProjectFunctionJobPatchInputList - | RegisteredSolverFunctionJobPatchInputList - ).validate_python(registered_function_job_patch_inputs) - used_function_class = registered_function_job_patch_inputs[0].patch.function_class + function_job_patch_requests: list[FunctionJobPatchRequest], +) -> BatchUpdateRegisteredFunctionJobsDB: async with transaction_context(get_asyncpg_engine(app), connection) as transaction: await check_user_api_access_rights( @@ -160,41 +150,41 @@ async def patch_function_job( ], ) updated_jobs = [] - for patch_input in registered_function_job_patch_inputs: + for patch_request in function_job_patch_requests: job = await get_function_job( app, connection=transaction, user_id=user_id, product_name=product_name, - function_job_id=patch_input.uid, + function_job_id=patch_request.uid, ) - if job.function_class != used_function_class: + if job.function_class != patch_request.patch.function_class: raise FunctionJobPatchModelIncompatibleError( function_id=job.function_uuid, product_name=product_name ) class_specific_data = _update_class_specific_data( - class_specific_data=job.class_specific_data, patch=patch_input.patch + class_specific_data=job.class_specific_data, patch=patch_request.patch ) update_values = { - "inputs": patch_input.patch.inputs, - "outputs": patch_input.patch.outputs, + "inputs": patch_request.patch.inputs, + "outputs": patch_request.patch.outputs, "class_specific_data": class_specific_data, - "title": patch_input.patch.title, - "description": patch_input.patch.description, + "title": patch_request.patch.title, + "description": patch_request.patch.description, "status": "created", } result = await transaction.execute( function_jobs_table.update() - .where(function_jobs_table.c.uuid == f"{patch_input.uid}") + .where(function_jobs_table.c.uuid == f"{patch_request.uid}") .values(**{k: v for k, v in update_values.items() if v is not None}) .returning(*_FUNCTION_JOBS_TABLE_COLS) ) updated_jobs.append(RegisteredFunctionJobDB.model_validate(result.one())) - return updated_jobs + return BatchUpdateRegisteredFunctionJobsDB(updated_items=updated_jobs) async def list_function_jobs_with_status( diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py index f6ed325c18f3..52747022be08 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py @@ -4,6 +4,7 @@ from models_library.basic_types import IDStr from models_library.functions import ( BatchCreateRegisteredFunctionJobs, + BatchUpdateRegisteredFunctionJobs, Function, FunctionClass, FunctionClassSpecificData, @@ -20,6 +21,8 @@ FunctionJobDB, FunctionJobID, FunctionJobList, + FunctionJobPatchRequest, + FunctionJobPatchRequestList, FunctionJobStatus, FunctionOutputs, FunctionOutputSchema, @@ -35,11 +38,9 @@ RegisteredFunctionJobWithStatusDB, RegisteredProjectFunction, RegisteredProjectFunctionJob, - RegisteredProjectFunctionJobPatchInputList, RegisteredProjectFunctionJobWithStatus, RegisteredSolverFunction, RegisteredSolverFunctionJob, - RegisteredSolverFunctionJobPatchInputList, RegisteredSolverFunctionJobWithStatus, ) from models_library.functions_errors import ( @@ -134,19 +135,39 @@ async def patch_registered_function_job( *, user_id: UserID, product_name: ProductName, - registered_function_job_patch_inputs: ( - RegisteredProjectFunctionJobPatchInputList - | RegisteredSolverFunctionJobPatchInputList - ), -) -> list[RegisteredFunctionJob]: + function_job_patch_request: FunctionJobPatchRequest, +) -> RegisteredFunctionJob: + + result = await _function_jobs_repository.patch_function_jobs( + app=app, + user_id=user_id, + product_name=product_name, + function_job_patch_requests=[function_job_patch_request], + ) + assert len(result.updated_items) == 1 # nosec + return _decode_functionjob(result.updated_items[0]) + - result = await _function_jobs_repository.patch_function_job( +async def batch_patch_registered_function_jobs( + app: web.Application, + *, + user_id: UserID, + product_name: ProductName, + function_job_patch_requests: FunctionJobPatchRequestList, +) -> BatchUpdateRegisteredFunctionJobs: + TypeAdapter(FunctionJobPatchRequestList).validate_python( + function_job_patch_requests + ) + + result = await _function_jobs_repository.patch_function_jobs( app=app, user_id=user_id, product_name=product_name, - registered_function_job_patch_inputs=registered_function_job_patch_inputs, + function_job_patch_requests=function_job_patch_requests, + ) + return BatchUpdateRegisteredFunctionJobs( + updated_items=[_decode_functionjob(job) for job in result.updated_items] ) - return [_decode_functionjob(job) for job in result] async def register_function_job_collection( diff --git a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py index 22125bd5a265..c33c178163d3 100644 --- a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py @@ -16,15 +16,12 @@ FunctionClass, FunctionJobCollection, FunctionJobList, + FunctionJobPatchRequest, FunctionJobStatus, RegisteredFunctionJob, RegisteredFunctionJobPatch, RegisteredProjectFunctionJobPatch, - RegisteredProjectFunctionJobPatchInput, - RegisteredProjectFunctionJobPatchInputList, RegisteredSolverFunctionJobPatch, - RegisteredSolverFunctionJobPatchInput, - RegisteredSolverFunctionJobPatchInputList, SolverFunctionJob, ) from models_library.functions_errors import ( @@ -630,32 +627,14 @@ async def test_patch_registered_function_jobs( user_id=logged_user["id"], product_name=osparc_product_name, ) - patch_inputs: ( - RegisteredProjectFunctionJobPatchInputList - | RegisteredSolverFunctionJobPatchInputList - ) - if function.function_class == FunctionClass.PROJECT: - assert isinstance(patch, RegisteredProjectFunctionJobPatch) - patch_inputs = [ - RegisteredProjectFunctionJobPatchInput(uid=registered_job.uid, patch=patch) - ] - elif function.function_class == FunctionClass.SOLVER: - assert isinstance(patch, RegisteredSolverFunctionJobPatch) - patch_inputs = [ - RegisteredSolverFunctionJobPatchInput(uid=registered_job.uid, patch=patch) - ] - else: - pytest.fail("Unsupported function class") - registered_jobs = ( - await webserver_rpc_client.functions.patch_registered_function_job( - user_id=logged_user["id"], - product_name=osparc_product_name, - registered_function_job_patch_inputs=patch_inputs, - ) + registered_job = await webserver_rpc_client.functions.patch_registered_function_job( + user_id=logged_user["id"], + product_name=osparc_product_name, + function_job_patch_request=FunctionJobPatchRequest( + uid=registered_job.uid, patch=patch + ), ) - assert len(registered_jobs) == 1 - registered_job = registered_jobs[0] assert registered_job.title == patch.title assert registered_job.description == patch.description assert registered_job.inputs == patch.inputs @@ -723,27 +702,14 @@ async def test_incompatible_patch_model_error( user_id=logged_user["id"], product_name=osparc_product_name, ) - patch_input = None - if function.function_class == FunctionClass.PROJECT: - assert isinstance(patch, RegisteredSolverFunctionJobPatch) - patch_input = RegisteredSolverFunctionJobPatchInput( - uid=registered_job.uid, patch=patch - ) - if function.function_class == FunctionClass.SOLVER: - assert isinstance(patch, RegisteredProjectFunctionJobPatch) - patch_input = RegisteredProjectFunctionJobPatchInput( - uid=registered_job.uid, patch=patch - ) - assert patch_input is not None with pytest.raises(FunctionJobPatchModelIncompatibleError): registered_job = ( await webserver_rpc_client.functions.patch_registered_function_job( user_id=logged_user["id"], product_name=osparc_product_name, - registered_function_job_patch_inputs=TypeAdapter( - RegisteredSolverFunctionJobPatchInputList - | RegisteredSolverFunctionJobPatchInputList - ).validate_python([patch_input]), + function_job_patch_request=FunctionJobPatchRequest( + uid=registered_job.uid, patch=patch + ), ) ) From 0ebd5008b144f1c74cc9692d82ebb9264150e09b Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Mon, 20 Oct 2025 15:33:40 +0200 Subject: [PATCH 49/63] use patch batch endpoint --- .../_service_function_jobs.py | 83 +++++++++---------- .../_service_function_jobs_task_client.py | 6 +- .../services_rpc/wb_api_server.py | 27 ++++-- 3 files changed, 59 insertions(+), 57 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py index e5cf1a37be58..2a294f64b1d0 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py @@ -1,25 +1,23 @@ from dataclasses import dataclass -from typing import Annotated import jsonschema from common_library.exclude import as_dict_exclude_none from models_library.functions import ( + BatchUpdateRegisteredFunctionJobs, FunctionClass, FunctionID, FunctionInputs, FunctionJobCollectionID, FunctionJobID, FunctionJobList, + FunctionJobPatchRequest, + FunctionJobPatchRequestList, FunctionSchemaClass, ProjectFunctionJob, RegisteredFunction, RegisteredFunctionJob, RegisteredProjectFunctionJobPatch, - RegisteredProjectFunctionJobPatchInput, - RegisteredProjectFunctionJobPatchInputList, RegisteredSolverFunctionJobPatch, - RegisteredSolverFunctionJobPatchInput, - RegisteredSolverFunctionJobPatchInputList, SolverFunctionJob, ) from models_library.functions_errors import ( @@ -31,7 +29,7 @@ from models_library.rest_pagination import PageMetaInfoLimitOffset, PageOffsetInt from models_library.rpc_pagination import PageLimitInt from models_library.users import UserID -from pydantic import Field, TypeAdapter, ValidationError, validate_call +from pydantic import TypeAdapter, ValidationError from simcore_service_api_server._service_functions import FunctionService from simcore_service_api_server.services_rpc.storage import StorageService @@ -210,25 +208,18 @@ async def pre_register_function_jobs( for job, input_ in zip(jobs, job_inputs) ] - @validate_call - async def patch_registered_function_job( + async def batch_patch_registered_function_job( self, *, user_id: UserID, product_name: ProductName, - patches: Annotated[ - list[FunctionJobPatch], - Field(max_length=50, min_length=1), - ], - ) -> list[RegisteredFunctionJob]: - patch_inputs: list[ - RegisteredProjectFunctionJobPatchInput - | RegisteredSolverFunctionJobPatchInput - ] = [] - for patch in patches: + function_job_patches: list[FunctionJobPatch], + ) -> BatchUpdateRegisteredFunctionJobs: + patch_inputs: FunctionJobPatchRequestList = [] + for patch in function_job_patches: if patch.function_class == FunctionClass.PROJECT: patch_inputs.append( - RegisteredProjectFunctionJobPatchInput( + FunctionJobPatchRequest( uid=patch.function_job_id, patch=RegisteredProjectFunctionJobPatch( title=None, @@ -236,13 +227,13 @@ async def patch_registered_function_job( inputs=None, outputs=None, job_creation_task_id=patch.job_creation_task_id, - project_job_id=patch.project_job_id, + project_job_id=None, ), ) ) elif patch.function_class == FunctionClass.SOLVER: patch_inputs.append( - RegisteredSolverFunctionJobPatchInput( + FunctionJobPatchRequest( uid=patch.function_job_id, patch=RegisteredSolverFunctionJobPatch( title=None, @@ -250,7 +241,7 @@ async def patch_registered_function_job( inputs=None, outputs=None, job_creation_task_id=patch.job_creation_task_id, - solver_job_id=patch.solver_job_id, + solver_job_id=None, ), ) ) @@ -258,14 +249,10 @@ async def patch_registered_function_job( raise UnsupportedFunctionClassError( function_class=patch.function_class, ) - - return await self._web_rpc_client.patch_registered_function_job( + return await self._web_rpc_client.batch_patch_registered_function_job( user_id=user_id, product_name=product_name, - registered_function_job_patch_inputs=TypeAdapter( - RegisteredProjectFunctionJobPatchInputList - | RegisteredSolverFunctionJobPatchInputList - ).validate_python(patch_inputs), + function_job_patch_requests=patch_inputs, ) async def run_function( @@ -294,20 +281,22 @@ async def run_function( job_id=study_job.id, pricing_spec=pricing_spec, ) - registered_jobs = await self.patch_registered_function_job( + registered_job = await self._web_rpc_client.patch_registered_function_job( user_id=self.user_id, product_name=self.product_name, - patches=[ - FunctionJobPatch( - function_class=FunctionClass.PROJECT, - function_job_id=pre_registered_function_job_data.function_job_id, + function_job_patch_request=FunctionJobPatchRequest( + uid=pre_registered_function_job_data.function_job_id, + patch=RegisteredProjectFunctionJobPatch( + title=None, + description=None, + inputs=None, + outputs=None, job_creation_task_id=None, project_job_id=study_job.id, - ) - ], + ), + ), ) - assert len(registered_jobs) == 1 - return registered_jobs[0] + return registered_job if function.function_class == FunctionClass.SOLVER: solver_job = await self._job_service.create_solver_job( @@ -325,20 +314,22 @@ async def run_function( job_id=solver_job.id, pricing_spec=pricing_spec, ) - registered_jobs = await self.patch_registered_function_job( + registered_job = await self._web_rpc_client.patch_registered_function_job( user_id=self.user_id, product_name=self.product_name, - patches=[ - FunctionJobPatch( - function_class=FunctionClass.SOLVER, - function_job_id=pre_registered_function_job_data.function_job_id, + function_job_patch_request=FunctionJobPatchRequest( + uid=pre_registered_function_job_data.function_job_id, + patch=RegisteredSolverFunctionJobPatch( + title=None, + description=None, + inputs=None, + outputs=None, job_creation_task_id=None, solver_job_id=solver_job.id, - ) - ], + ), + ), ) - assert len(registered_jobs) == 1 - return registered_jobs[0] + return registered_job raise UnsupportedFunctionClassError( function_class=function.function_class, diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py index 377ed00575a0..2464cea15b83 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py @@ -371,10 +371,10 @@ async def create_function_job_creation_tasks( for pre_registered_function_job_data in pre_registered_function_job_data_list ] - patched_jobs = await self._function_job_service.patch_registered_function_job( + patched_jobs = await self._function_job_service.batch_patch_registered_function_job( user_id=user_identity.user_id, product_name=user_identity.product_name, - patches=[ + function_job_patches=[ FunctionJobPatch( function_class=function.function_class, function_job_id=pre_registered_function_job_data.function_job_id, @@ -387,6 +387,6 @@ async def create_function_job_creation_tasks( ) ], ) - patched_jobs_iter = iter(patched_jobs) + patched_jobs_iter = iter(patched_jobs.updated_items) _ = lambda job: job if job is not None else next(patched_jobs_iter) return [_(job) for job in cached_jobs] diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index be7c024510e7..93be96c99139 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -24,16 +24,17 @@ from models_library.api_schemas_webserver.licensed_items import LicensedItemRpcGetPage from models_library.functions import ( BatchCreateRegisteredFunctionJobs, + BatchUpdateRegisteredFunctionJobs, FunctionInputsList, FunctionJob, FunctionJobList, + FunctionJobPatchRequest, + FunctionJobPatchRequestList, FunctionJobStatus, FunctionOutputs, FunctionUserAccessRights, FunctionUserApiAccessRights, RegisteredFunctionJobWithStatus, - RegisteredProjectFunctionJobPatchInputList, - RegisteredSolverFunctionJobPatchInputList, ) from models_library.licenses import LicensedItemID from models_library.products import ProductName @@ -515,15 +516,25 @@ async def patch_registered_function_job( *, user_id: UserID, product_name: ProductName, - registered_function_job_patch_inputs: ( - RegisteredProjectFunctionJobPatchInputList - | RegisteredSolverFunctionJobPatchInputList - ), - ) -> list[RegisteredFunctionJob]: + function_job_patch_request: FunctionJobPatchRequest, + ) -> RegisteredFunctionJob: return await self._rpc_client.functions.patch_registered_function_job( user_id=user_id, product_name=product_name, - registered_function_job_patch_inputs=registered_function_job_patch_inputs, + function_job_patch_request=function_job_patch_request, + ) + + async def batch_patch_registered_function_job( + self, + *, + product_name: ProductName, + user_id: UserID, + function_job_patch_requests: FunctionJobPatchRequestList, + ) -> BatchUpdateRegisteredFunctionJobs: + return await self._rpc_client.functions.batch_patch_registered_function_job( + product_name=product_name, + user_id=user_id, + function_job_patch_requests=function_job_patch_requests, ) async def get_function_input_schema( From 650986dfec8df3118a1f77b4cbb1fa7fbd8d6789 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Mon, 20 Oct 2025 16:45:53 +0200 Subject: [PATCH 50/63] start fixing find_cache method --- .../functions/_controller/_functions_rpc.py | 29 ++++++++----- .../functions/_function_jobs_repository.py | 17 ++++++-- .../functions/_functions_service.py | 41 ++++++++++++------- .../test_function_jobs_controller_rpc.py | 17 ++++---- 4 files changed, 66 insertions(+), 38 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py index 4ac1c419ea5f..26a2e6af6542 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py @@ -3,6 +3,7 @@ from aiohttp import web from models_library.functions import ( BatchCreateRegisteredFunctionJobs, + BatchGetCachedRegisteredFunctionJobs, BatchUpdateRegisteredFunctionJobs, Function, FunctionAccessRights, @@ -490,17 +491,23 @@ async def find_cached_function_jobs( function_id: FunctionID, inputs: FunctionInputsList, status_filter: list[FunctionJobStatus] | None, -) -> list[RegisteredFunctionJob | None]: - jobs = await _functions_service.find_cached_function_jobs( - app=app, - user_id=user_id, - product_name=product_name, - function_id=function_id, - inputs=inputs, - status_filter=status_filter, - ) - assert len(jobs) == len(inputs) # nosec - return jobs +) -> BatchGetCachedRegisteredFunctionJobs: + retrieved_cached_function_jobs = ( + await _functions_service.batch_find_cached_function_jobs( + app=app, + user_id=user_id, + product_name=product_name, + function_id=function_id, + inputs=inputs, + status_filter=status_filter, + ) + ) + assert len(retrieved_cached_function_jobs.found_items) + len( + retrieved_cached_function_jobs.missing_identifiers + ) == len( + inputs + ) # nosec + return retrieved_cached_function_jobs @router.expose(reraise_if_error_type=(FunctionIDNotFoundError,)) diff --git a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py index 7a674fb5092e..0afba0c090dd 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py @@ -7,6 +7,7 @@ from aiohttp import web from models_library.functions import ( BatchCreateRegisteredFunctionJobsDB, + BatchGetCachedRegisteredFunctionJobsDB, BatchUpdateRegisteredFunctionJobsDB, FunctionClass, FunctionClassSpecificData, @@ -318,7 +319,7 @@ async def delete_function_job( ) -async def find_cached_function_jobs( +async def batch_find_cached_function_jobs( app: web.Application, connection: AsyncConnection | None = None, *, @@ -327,7 +328,7 @@ async def find_cached_function_jobs( product_name: ProductName, inputs: FunctionInputsList, status_filter: list[FunctionJobStatus] | None = None, -) -> list[RegisteredFunctionJobDB | None]: +) -> BatchGetCachedRegisteredFunctionJobsDB: async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: # Get user groups for access check user_groups = await list_all_user_groups_ids(app, user_id=user_id) @@ -378,8 +379,16 @@ async def find_cached_function_jobs( for row in results } - # Return results in the same order as inputs, with None for missing jobs - return [jobs_by_input.get(json_input, None) for json_input in json_inputs] + return BatchGetCachedRegisteredFunctionJobsDB( + found_items=[ + jobs_by_input[input_] + for input_ in json_inputs + if input_ in jobs_by_input + ], + missing_identifiers=[ + input_ for input_ in inputs if json.dumps(input_) not in jobs_by_input + ], + ) async def get_function_job_status( diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py index 52747022be08..637e524dd84f 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py @@ -4,6 +4,7 @@ from models_library.basic_types import IDStr from models_library.functions import ( BatchCreateRegisteredFunctionJobs, + BatchGetCachedRegisteredFunctionJobs, BatchUpdateRegisteredFunctionJobs, Function, FunctionClass, @@ -448,7 +449,7 @@ async def update_function( return _decode_function(updated_function) -async def find_cached_function_jobs( +async def batch_find_cached_function_jobs( app: web.Application, *, user_id: UserID, @@ -456,22 +457,26 @@ async def find_cached_function_jobs( function_id: FunctionID, inputs: FunctionInputsList, status_filter: list[FunctionJobStatus] | None = None, -) -> list[RegisteredFunctionJob | None]: - returned_function_jobs = await _function_jobs_repository.find_cached_function_jobs( - app=app, - user_id=user_id, - product_name=product_name, - function_id=function_id, - inputs=inputs, - status_filter=status_filter, +) -> BatchGetCachedRegisteredFunctionJobs: + returned_function_jobs = ( + await _function_jobs_repository.batch_find_cached_function_jobs( + app=app, + user_id=user_id, + product_name=product_name, + function_id=function_id, + inputs=inputs, + status_filter=status_filter, + ) ) - assert len(returned_function_jobs) == len(inputs) # nosec + assert len(returned_function_jobs.found_items) + len( + returned_function_jobs.missing_identifiers + ) == len( + inputs + ) # nosec def _map_db_model_to_domain_model( - job: RegisteredFunctionJobDB | None, - ) -> RegisteredFunctionJob | None: - if job is None: - return None + job: RegisteredFunctionJobDB, + ) -> RegisteredFunctionJob: if job.function_class == FunctionClass.PROJECT: return RegisteredProjectFunctionJob( uid=job.uuid, @@ -502,7 +507,13 @@ def _map_db_model_to_domain_model( ) raise UnsupportedFunctionJobClassError(function_job_class=job.function_class) - return [_map_db_model_to_domain_model(job) for job in returned_function_jobs] + return BatchGetCachedRegisteredFunctionJobs( + found_items=[ + _map_db_model_to_domain_model(job) + for job in returned_function_jobs.found_items + ], + missing_identifiers=returned_function_jobs.missing_identifiers, + ) async def get_function_input_schema( diff --git a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py index c33c178163d3..0d5145d6aeb7 100644 --- a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py @@ -464,12 +464,13 @@ async def test_find_cached_function_jobs( ) # Assert the cached jobs contain the registered job - assert cached_jobs is not None - assert len(cached_jobs) == 2 - job0 = cached_jobs[0] + assert cached_jobs.found_items is not None + assert len(cached_jobs.found_items) == 1 + job0 = cached_jobs.found_items[0] assert job0 is not None assert job0.inputs == {"input1": 1} - assert cached_jobs[1] is None + assert len(cached_jobs.missing_identifiers) == 1 + assert cached_jobs.missing_identifiers[0] == {"input1": 10} cached_jobs = await webserver_rpc_client.functions.find_cached_function_jobs( function_id=registered_function.uid, @@ -479,8 +480,8 @@ async def test_find_cached_function_jobs( ) # Assert the cached jobs does not contain the registered job for the other user - assert len(cached_jobs) == 2 - assert all(elm is None for elm in cached_jobs) + assert len(cached_jobs.missing_identifiers) == 2 + assert len(cached_jobs.found_items) == 0 @pytest.mark.parametrize( @@ -541,8 +542,8 @@ async def test_find_cached_function_jobs_with_status( inputs=[input_], status_filter=[status], ) - assert len(cached_jobs) == 1 - cached_job = cached_jobs[0] + assert len(cached_jobs.found_items) == 1 + cached_job = cached_jobs.found_items[0] assert cached_job is not None assert cached_job.inputs == input_ cached_job_status = await webserver_rpc_client.functions.get_function_job_status( From c3f27551faf2c12132730b639fb22199453c916c Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 21 Oct 2025 09:15:09 +0200 Subject: [PATCH 51/63] batch cache endpoints --- .../models-library/src/models_library/functions.py | 14 +++++++++++++- .../rpc_interfaces/webserver/v1/functions.py | 5 +++-- .../_service_function_jobs.py | 2 +- .../_service_function_jobs_task_client.py | 2 +- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/packages/models-library/src/models_library/functions.py b/packages/models-library/src/models_library/functions.py index cf25ca4c888d..21d9a24fda3d 100644 --- a/packages/models-library/src/models_library/functions.py +++ b/packages/models-library/src/models_library/functions.py @@ -15,7 +15,7 @@ from models_library.utils.enums import StrAutoEnum from pydantic import BaseModel, ConfigDict, Field -from .batch_operations import BatchUpdateEnvelope +from .batch_operations import BatchGetEnvelope, BatchUpdateEnvelope from .projects import ProjectID from .utils.change_case import snake_to_camel @@ -281,6 +281,12 @@ class BatchUpdateRegisteredFunctionJobs(BatchUpdateEnvelope[RegisteredFunctionJo pass +class BatchGetCachedRegisteredFunctionJobs( + BatchGetEnvelope[RegisteredFunctionJob, FunctionInputs] +): + pass + + RegisteredFunctionJobPatch = Annotated[ RegisteredProjectFunctionJobPatch | RegisteredPythonCodeFunctionJobPatch @@ -372,6 +378,12 @@ class RegisteredFunctionJobDB(FunctionJobDB): created: datetime.datetime +class BatchGetCachedRegisteredFunctionJobsDB( + BatchGetEnvelope[RegisteredFunctionJobDB, FunctionInputs] +): + pass + + class BatchCreateRegisteredFunctionJobsDB(BatchCreateEnvelope[RegisteredFunctionJobDB]): pass diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py index d12fdb5531ce..02ab1a4cf31a 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py @@ -18,6 +18,7 @@ ) from models_library.functions import ( BatchCreateRegisteredFunctionJobs, + BatchGetCachedRegisteredFunctionJobs, BatchUpdateRegisteredFunctionJobs, FunctionClass, FunctionGroupAccessRights, @@ -501,9 +502,9 @@ async def find_cached_function_jobs( function_id: FunctionID, inputs: FunctionInputsList, status_filter: list[FunctionJobStatus] | None = None, - ) -> list[RegisteredFunctionJob | None]: + ) -> BatchGetCachedRegisteredFunctionJobs: """Find cached function jobs.""" - return TypeAdapter(list[RegisteredFunctionJob | None]).validate_python( + return TypeAdapter(BatchGetCachedRegisteredFunctionJobs).validate_python( await self._request( "find_cached_function_jobs", product_name=product_name, diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py index 2a294f64b1d0..fd823a078d1a 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py @@ -132,7 +132,7 @@ def create_function_job_inputs( # pylint: disable=no-self-use values=joined_inputs or {}, ) - async def pre_register_function_jobs( + async def batch_pre_register_function_jobs( self, *, function: RegisteredFunction, diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py index 2464cea15b83..b08e4d929605 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py @@ -340,7 +340,7 @@ async def create_function_job_creation_tasks( ] pre_registered_function_job_data_list = ( - await self._function_job_service.pre_register_function_jobs( + await self._function_job_service.batch_pre_register_function_jobs( function=function, job_inputs=uncached_inputs, ) From 1e7486aee087874ea09a81d5b2ac3c75365d5fdb Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 21 Oct 2025 09:28:18 +0200 Subject: [PATCH 52/63] change requirement for bach get --- .../src/models_library/batch_operations.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/models-library/src/models_library/batch_operations.py b/packages/models-library/src/models_library/batch_operations.py index ff0b34b687a4..a96b9e972807 100644 --- a/packages/models-library/src/models_library/batch_operations.py +++ b/packages/models-library/src/models_library/batch_operations.py @@ -23,10 +23,10 @@ - https://google.aip.dev/231 """ -from typing import Annotated, Generic, TypeVar +from typing import Annotated, Generic, Self, TypeVar from common_library.basic_types import DEFAULT_FACTORY -from pydantic import BaseModel, BeforeValidator, Field, TypeAdapter +from pydantic import BaseModel, BeforeValidator, Field, TypeAdapter, model_validator ResourceT = TypeVar("ResourceT") IdentifierT = TypeVar("IdentifierT") @@ -74,7 +74,6 @@ class BatchGetEnvelope(BaseModel, Generic[ResourceT, IdentifierT]): found_items: Annotated[ list[ResourceT], Field( - min_length=1, description="List of successfully retrieved items. Must contain at least one item.", ), ] @@ -86,6 +85,14 @@ class BatchGetEnvelope(BaseModel, Generic[ResourceT, IdentifierT]): ), ] = DEFAULT_FACTORY + @model_validator(mode="after") + def check_found_items_not_empty(self) -> Self: + if len(self.found_items) + len(self.missing_identifiers) == 0: + raise ValueError( + "At least one item must be found or missing in a batch-get operation." + ) + return self + class BatchCreateEnvelope(BaseModel, Generic[SchemaT]): """Generic envelope model for batch-create operations. From d5786aa0223271d04e8444c023105b358d3f8724 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 21 Oct 2025 09:29:11 +0200 Subject: [PATCH 53/63] name change --- packages/models-library/src/models_library/batch_operations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/models-library/src/models_library/batch_operations.py b/packages/models-library/src/models_library/batch_operations.py index a96b9e972807..ce033bdff147 100644 --- a/packages/models-library/src/models_library/batch_operations.py +++ b/packages/models-library/src/models_library/batch_operations.py @@ -86,7 +86,7 @@ class BatchGetEnvelope(BaseModel, Generic[ResourceT, IdentifierT]): ] = DEFAULT_FACTORY @model_validator(mode="after") - def check_found_items_not_empty(self) -> Self: + def check_items_not_empty(self) -> Self: if len(self.found_items) + len(self.missing_identifiers) == 0: raise ValueError( "At least one item must be found or missing in a batch-get operation." From 77b4a5fe4d92da1650392a4528f44a5197db8f0c Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 21 Oct 2025 10:16:13 +0200 Subject: [PATCH 54/63] use custom signature for batch cache endpoint --- .../src/models_library/batch_operations.py | 13 ++---- .../rpc_interfaces/webserver/v1/functions.py | 5 +-- .../_service_function_jobs_task_client.py | 15 ++----- .../functions/_controller/_functions_rpc.py | 27 +++++------- .../functions/_function_jobs_repository.py | 16 ++------ .../functions/_functions_service.py | 41 +++++++------------ 6 files changed, 37 insertions(+), 80 deletions(-) diff --git a/packages/models-library/src/models_library/batch_operations.py b/packages/models-library/src/models_library/batch_operations.py index ce033bdff147..ff0b34b687a4 100644 --- a/packages/models-library/src/models_library/batch_operations.py +++ b/packages/models-library/src/models_library/batch_operations.py @@ -23,10 +23,10 @@ - https://google.aip.dev/231 """ -from typing import Annotated, Generic, Self, TypeVar +from typing import Annotated, Generic, TypeVar from common_library.basic_types import DEFAULT_FACTORY -from pydantic import BaseModel, BeforeValidator, Field, TypeAdapter, model_validator +from pydantic import BaseModel, BeforeValidator, Field, TypeAdapter ResourceT = TypeVar("ResourceT") IdentifierT = TypeVar("IdentifierT") @@ -74,6 +74,7 @@ class BatchGetEnvelope(BaseModel, Generic[ResourceT, IdentifierT]): found_items: Annotated[ list[ResourceT], Field( + min_length=1, description="List of successfully retrieved items. Must contain at least one item.", ), ] @@ -85,14 +86,6 @@ class BatchGetEnvelope(BaseModel, Generic[ResourceT, IdentifierT]): ), ] = DEFAULT_FACTORY - @model_validator(mode="after") - def check_items_not_empty(self) -> Self: - if len(self.found_items) + len(self.missing_identifiers) == 0: - raise ValueError( - "At least one item must be found or missing in a batch-get operation." - ) - return self - class BatchCreateEnvelope(BaseModel, Generic[SchemaT]): """Generic envelope model for batch-create operations. diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py index 02ab1a4cf31a..d12fdb5531ce 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py @@ -18,7 +18,6 @@ ) from models_library.functions import ( BatchCreateRegisteredFunctionJobs, - BatchGetCachedRegisteredFunctionJobs, BatchUpdateRegisteredFunctionJobs, FunctionClass, FunctionGroupAccessRights, @@ -502,9 +501,9 @@ async def find_cached_function_jobs( function_id: FunctionID, inputs: FunctionInputsList, status_filter: list[FunctionJobStatus] | None = None, - ) -> BatchGetCachedRegisteredFunctionJobs: + ) -> list[RegisteredFunctionJob | None]: """Find cached function jobs.""" - return TypeAdapter(BatchGetCachedRegisteredFunctionJobs).validate_python( + return TypeAdapter(list[RegisteredFunctionJob | None]).validate_python( await self._request( "find_cached_function_jobs", product_name=product_name, diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py index b08e4d929605..b684ae2182fd 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py @@ -51,7 +51,7 @@ from .models.domain.celery_models import ApiServerOwnerMetadata from .models.domain.functions import FunctionJobPatch from .models.schemas.functions import FunctionJobCreationTaskStatus -from .models.schemas.jobs import JobPricingSpecification +from .models.schemas.jobs import JobInputs, JobPricingSpecification from .services_http.webserver import AuthSession from .services_rpc.storage import StorageService from .services_rpc.wb_api_server import WbApiRpcClient @@ -317,24 +317,17 @@ async def create_function_job_creation_tasks( parent_node_id: NodeID | None = None, ) -> list[RegisteredFunctionJob]: inputs = [ - self._function_job_service.create_function_job_inputs( - function=function, function_inputs=input_ - ) - for input_ in function_inputs + join_inputs(function.default_inputs, input_) for input_ in function_inputs ] cached_jobs = await self._web_rpc_client.find_cached_function_jobs( user_id=user_identity.user_id, product_name=user_identity.product_name, function_id=function.uid, - inputs=TypeAdapter(FunctionInputsList).validate_python( - [input_.values for input_ in inputs] - ), + inputs=TypeAdapter(FunctionInputsList).validate_python(inputs), status_filter=[FunctionJobStatus(status=RunningState.SUCCESS)], ) - assert len(cached_jobs) == len(inputs) # nosec - uncached_inputs = [ input_ for input_, job in zip(inputs, cached_jobs) if job is None ] @@ -342,7 +335,7 @@ async def create_function_job_creation_tasks( pre_registered_function_job_data_list = ( await self._function_job_service.batch_pre_register_function_jobs( function=function, - job_inputs=uncached_inputs, + job_inputs=[JobInputs(values=_ or {}) for _ in uncached_inputs], ) ) diff --git a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py index 26a2e6af6542..9a5b9580754a 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py @@ -3,7 +3,6 @@ from aiohttp import web from models_library.functions import ( BatchCreateRegisteredFunctionJobs, - BatchGetCachedRegisteredFunctionJobs, BatchUpdateRegisteredFunctionJobs, Function, FunctionAccessRights, @@ -491,22 +490,16 @@ async def find_cached_function_jobs( function_id: FunctionID, inputs: FunctionInputsList, status_filter: list[FunctionJobStatus] | None, -) -> BatchGetCachedRegisteredFunctionJobs: - retrieved_cached_function_jobs = ( - await _functions_service.batch_find_cached_function_jobs( - app=app, - user_id=user_id, - product_name=product_name, - function_id=function_id, - inputs=inputs, - status_filter=status_filter, - ) - ) - assert len(retrieved_cached_function_jobs.found_items) + len( - retrieved_cached_function_jobs.missing_identifiers - ) == len( - inputs - ) # nosec +) -> list[RegisteredFunctionJob | None]: + retrieved_cached_function_jobs = await _functions_service.find_cached_function_jobs( + app=app, + user_id=user_id, + product_name=product_name, + function_id=function_id, + inputs=inputs, + status_filter=status_filter, + ) + assert len(retrieved_cached_function_jobs) == len(inputs) # nosec return retrieved_cached_function_jobs diff --git a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py index 0afba0c090dd..8be65087fb54 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py @@ -7,7 +7,6 @@ from aiohttp import web from models_library.functions import ( BatchCreateRegisteredFunctionJobsDB, - BatchGetCachedRegisteredFunctionJobsDB, BatchUpdateRegisteredFunctionJobsDB, FunctionClass, FunctionClassSpecificData, @@ -319,7 +318,7 @@ async def delete_function_job( ) -async def batch_find_cached_function_jobs( +async def find_cached_function_jobs( app: web.Application, connection: AsyncConnection | None = None, *, @@ -328,7 +327,7 @@ async def batch_find_cached_function_jobs( product_name: ProductName, inputs: FunctionInputsList, status_filter: list[FunctionJobStatus] | None = None, -) -> BatchGetCachedRegisteredFunctionJobsDB: +) -> list[RegisteredFunctionJobDB | None]: async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: # Get user groups for access check user_groups = await list_all_user_groups_ids(app, user_id=user_id) @@ -379,16 +378,7 @@ async def batch_find_cached_function_jobs( for row in results } - return BatchGetCachedRegisteredFunctionJobsDB( - found_items=[ - jobs_by_input[input_] - for input_ in json_inputs - if input_ in jobs_by_input - ], - missing_identifiers=[ - input_ for input_ in inputs if json.dumps(input_) not in jobs_by_input - ], - ) + return [jobs_by_input.get(input_, None) for input_ in json_inputs] async def get_function_job_status( diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py index 637e524dd84f..52747022be08 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py @@ -4,7 +4,6 @@ from models_library.basic_types import IDStr from models_library.functions import ( BatchCreateRegisteredFunctionJobs, - BatchGetCachedRegisteredFunctionJobs, BatchUpdateRegisteredFunctionJobs, Function, FunctionClass, @@ -449,7 +448,7 @@ async def update_function( return _decode_function(updated_function) -async def batch_find_cached_function_jobs( +async def find_cached_function_jobs( app: web.Application, *, user_id: UserID, @@ -457,26 +456,22 @@ async def batch_find_cached_function_jobs( function_id: FunctionID, inputs: FunctionInputsList, status_filter: list[FunctionJobStatus] | None = None, -) -> BatchGetCachedRegisteredFunctionJobs: - returned_function_jobs = ( - await _function_jobs_repository.batch_find_cached_function_jobs( - app=app, - user_id=user_id, - product_name=product_name, - function_id=function_id, - inputs=inputs, - status_filter=status_filter, - ) +) -> list[RegisteredFunctionJob | None]: + returned_function_jobs = await _function_jobs_repository.find_cached_function_jobs( + app=app, + user_id=user_id, + product_name=product_name, + function_id=function_id, + inputs=inputs, + status_filter=status_filter, ) - assert len(returned_function_jobs.found_items) + len( - returned_function_jobs.missing_identifiers - ) == len( - inputs - ) # nosec + assert len(returned_function_jobs) == len(inputs) # nosec def _map_db_model_to_domain_model( - job: RegisteredFunctionJobDB, - ) -> RegisteredFunctionJob: + job: RegisteredFunctionJobDB | None, + ) -> RegisteredFunctionJob | None: + if job is None: + return None if job.function_class == FunctionClass.PROJECT: return RegisteredProjectFunctionJob( uid=job.uuid, @@ -507,13 +502,7 @@ def _map_db_model_to_domain_model( ) raise UnsupportedFunctionJobClassError(function_job_class=job.function_class) - return BatchGetCachedRegisteredFunctionJobs( - found_items=[ - _map_db_model_to_domain_model(job) - for job in returned_function_jobs.found_items - ], - missing_identifiers=returned_function_jobs.missing_identifiers, - ) + return [_map_db_model_to_domain_model(job) for job in returned_function_jobs] async def get_function_input_schema( From d0f31429168949196533fec567bcb9929a56cb64 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 21 Oct 2025 10:24:19 +0200 Subject: [PATCH 55/63] fix tests --- .../test_function_jobs_controller_rpc.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py index 0d5145d6aeb7..9f138bf8a17a 100644 --- a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py @@ -464,13 +464,11 @@ async def test_find_cached_function_jobs( ) # Assert the cached jobs contain the registered job - assert cached_jobs.found_items is not None - assert len(cached_jobs.found_items) == 1 - job0 = cached_jobs.found_items[0] + assert len(cached_jobs) == 2 + job0 = cached_jobs[0] assert job0 is not None assert job0.inputs == {"input1": 1} - assert len(cached_jobs.missing_identifiers) == 1 - assert cached_jobs.missing_identifiers[0] == {"input1": 10} + assert cached_jobs[1] is None cached_jobs = await webserver_rpc_client.functions.find_cached_function_jobs( function_id=registered_function.uid, @@ -480,8 +478,8 @@ async def test_find_cached_function_jobs( ) # Assert the cached jobs does not contain the registered job for the other user - assert len(cached_jobs.missing_identifiers) == 2 - assert len(cached_jobs.found_items) == 0 + assert len(cached_jobs) == 2 + assert all(job is None for job in cached_jobs) @pytest.mark.parametrize( @@ -542,8 +540,8 @@ async def test_find_cached_function_jobs_with_status( inputs=[input_], status_filter=[status], ) - assert len(cached_jobs.found_items) == 1 - cached_job = cached_jobs.found_items[0] + assert len(cached_jobs) == 1 + cached_job = cached_jobs[0] assert cached_job is not None assert cached_job.inputs == input_ cached_job_status = await webserver_rpc_client.functions.get_function_job_status( From e6f289da17c0d7e59167044821245dc88a4e49e4 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 21 Oct 2025 10:29:24 +0200 Subject: [PATCH 56/63] remove fcn --- .../functions/functions_rpc_interface.py | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py index 43ab02fa657f..6a6650067bfc 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py @@ -27,8 +27,6 @@ FunctionUserAccessRights, FunctionUserApiAccessRights, RegisteredFunctionJobWithStatus, - RegisteredProjectFunctionJobPatchInputList, - RegisteredSolverFunctionJobPatchInputList, ) from models_library.products import ProductName from models_library.rabbitmq_basic_types import RPCMethodName @@ -361,30 +359,6 @@ async def register_function_job( ) # Validates the result as a RegisteredFunctionJob -@log_decorator(_logger, level=logging.DEBUG) -async def patch_registered_function_job( - rabbitmq_rpc_client: RabbitMQRPCClient, - *, - user_id: UserID, - product_name: ProductName, - registered_function_job_patch_inputs: ( - RegisteredProjectFunctionJobPatchInputList - | RegisteredSolverFunctionJobPatchInputList - ), -) -> list[RegisteredFunctionJob]: - result = await rabbitmq_rpc_client.request( - DEFAULT_WEBSERVER_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("patch_registered_function_job"), - user_id=user_id, - product_name=product_name, - registered_function_job_patch_inputs=registered_function_job_patch_inputs, - timeout_s=_FUNCTION_RPC_TIMEOUT_SEC, - ) - return TypeAdapter(list[RegisteredFunctionJob]).validate_python( - result - ) # Validates the result as a RegisteredFunctionJob - - @log_decorator(_logger, level=logging.DEBUG) async def get_function_job( rabbitmq_rpc_client: RabbitMQRPCClient, From 5d99508f31d8fe06bb8621ddf2b61e1e2e13b93d Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 21 Oct 2025 11:53:26 +0200 Subject: [PATCH 57/63] fix api-server tests --- .../rpc_interfaces/webserver/v1/functions.py | 2 +- .../celery/test_functions_celery.py | 129 ++++++++++++------ .../test_api_routers_function_jobs.py | 7 +- .../test_api_routers_functions.py | 34 ++--- 4 files changed, 110 insertions(+), 62 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py index d12fdb5531ce..aa9693d9b459 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py @@ -378,7 +378,7 @@ async def batch_patch_registered_function_job( """Patch a registered function job.""" return BatchUpdateRegisteredFunctionJobs.model_validate( await self._request( - "patch_registered_function_job", + "batch_patch_registered_function_jobs", product_name=product_name, user_id=user_id, function_job_patch_requests=function_job_patch_requests, diff --git a/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py b/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py index 50eb1a3986c8..3d28c0efa15b 100644 --- a/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py +++ b/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py @@ -24,12 +24,16 @@ from httpx import AsyncClient, BasicAuth, HTTPStatusError from models_library.api_schemas_long_running_tasks.tasks import TaskResult, TaskStatus from models_library.functions import ( + BatchCreateRegisteredFunctionJobs, + BatchUpdateRegisteredFunctionJobs, FunctionClass, FunctionID, FunctionInputsList, FunctionJobCollection, FunctionJobID, FunctionJobList, + FunctionJobPatchRequest, + FunctionJobPatchRequestList, FunctionJobStatus, FunctionUserAccessRights, FunctionUserApiAccessRights, @@ -38,8 +42,8 @@ RegisteredFunctionJobCollection, RegisteredProjectFunction, RegisteredProjectFunctionJob, - RegisteredProjectFunctionJobPatchInputList, - RegisteredSolverFunctionJobPatchInputList, + RegisteredProjectFunctionJobPatch, + RegisteredSolverFunctionJobPatch, ) from models_library.products import ProductName from models_library.projects import ProjectID @@ -155,24 +159,46 @@ def _(celery_app: Celery) -> None: return _ -async def _patch_registered_function_job_side_effect( +async def _batch_patch_registered_function_job( mock_registered_project_function_job: RegisteredFunctionJob, product_name: ProductName, user_id: UserID, - registered_function_job_patch_inputs: ( - RegisteredProjectFunctionJobPatchInputList - | RegisteredSolverFunctionJobPatchInputList - ), + function_job_patch_requests: FunctionJobPatchRequestList, ): - return [ - mock_registered_project_function_job.model_copy( - update={ - "job_creation_task_id": patch.patch.job_creation_task_id, - "uid": patch.uid, - } + jobs = [] + for patch_request in function_job_patch_requests: + patch = patch_request.patch + assert isinstance(patch, RegisteredProjectFunctionJobPatch) or isinstance( + patch, RegisteredSolverFunctionJobPatch + ) + jobs.append( + mock_registered_project_function_job.model_copy( + update={ + "job_creation_task_id": patch.job_creation_task_id, + "uid": patch_request.uid, + } + ) ) - for patch in registered_function_job_patch_inputs - ] + return BatchUpdateRegisteredFunctionJobs(updated_items=jobs) + + +async def _patch_registered_function_job( + mock_registered_project_function_job: RegisteredFunctionJob, + product_name: ProductName, + user_id: UserID, + function_job_patch_request: FunctionJobPatchRequest, +): + patch = function_job_patch_request.patch + assert isinstance(patch, RegisteredProjectFunctionJobPatch) or isinstance( + patch, RegisteredSolverFunctionJobPatch + ) + job = mock_registered_project_function_job.model_copy( + update={ + "job_creation_task_id": patch.job_creation_task_id, + "uid": function_job_patch_request.uid, + } + ) + return job async def _find_cached_function_jobs_side_effect( @@ -186,18 +212,20 @@ async def _find_cached_function_jobs_side_effect( return [None] * len(inputs) -async def _register_function_job_side_effect( +async def _batch_register_function_jobs( registered_function_job: RegisteredFunctionJob, user_id: UserID, function_jobs: FunctionJobList, product_name: ProductName, ): - return [ - registered_function_job.model_copy( - update={"uid": FunctionJobID(_faker.uuid4())} - ) - for _ in function_jobs - ] + return BatchCreateRegisteredFunctionJobs( + created_items=[ + registered_function_job.model_copy( + update={"uid": FunctionJobID(_faker.uuid4())} + ) + for _ in function_jobs + ] + ) @pytest.mark.parametrize("register_celery_tasks", [_register_fake_run_function_task()]) @@ -257,17 +285,17 @@ async def test_with_fake_run_function( "find_cached_function_jobs", side_effect=_find_cached_function_jobs_side_effect ) mock_handler_in_functions_rpc_interface( - "register_function_job", + "batch_register_function_jobs", side_effect=partial( - _register_function_job_side_effect, + _batch_register_function_jobs, fake_registered_project_function_job, ), ) mock_handler_in_functions_rpc_interface( - "patch_registered_function_job", + "batch_patch_registered_function_job", side_effect=partial( - _patch_registered_function_job_side_effect, + _batch_patch_registered_function_job, fake_registered_project_function_job, ), ) @@ -422,9 +450,9 @@ def _default_side_effect( "find_cached_function_jobs", side_effect=_find_cached_function_jobs_side_effect ) mock_handler_in_functions_rpc_interface( - "register_function_job", + "batch_register_function_jobs", side_effect=partial( - _register_function_job_side_effect, + _batch_register_function_jobs, fake_registered_project_function_job, ), ) @@ -437,10 +465,17 @@ def _default_side_effect( read_functions=True, ), ) + mock_handler_in_functions_rpc_interface( + "batch_patch_registered_function_job", + side_effect=partial( + _batch_patch_registered_function_job, + fake_registered_project_function_job, + ), + ) mock_handler_in_functions_rpc_interface( "patch_registered_function_job", side_effect=partial( - _patch_registered_function_job_side_effect, + _patch_registered_function_job, fake_registered_project_function_job, ), ) @@ -546,9 +581,9 @@ def _default_side_effect( "find_cached_function_jobs", side_effect=_find_cached_function_jobs_side_effect ) mock_handler_in_functions_rpc_interface( - "register_function_job", + "batch_register_function_jobs", side_effect=partial( - _register_function_job_side_effect, + _batch_register_function_jobs, fake_registered_project_function_job, ), ) @@ -573,13 +608,19 @@ def _default_side_effect( ) patch_mock = mock_handler_in_functions_rpc_interface( + "batch_patch_registered_function_job", + side_effect=partial( + _batch_patch_registered_function_job, + fake_registered_project_function_job, + ), + ) + mock_handler_in_functions_rpc_interface( "patch_registered_function_job", side_effect=partial( - _patch_registered_function_job_side_effect, + _patch_registered_function_job, fake_registered_project_function_job, ), ) - mock_handler_in_projects_rpc_interface("mark_project_as_job", return_value=None) # ACT @@ -601,7 +642,7 @@ def _default_side_effect( if expected_status_code == status.HTTP_200_OK: FunctionJobCollection.model_validate(response.json()) - task_id = patch_mock.call_args.kwargs["registered_function_job_patch_inputs"][ + task_id = patch_mock.call_args.kwargs["function_job_patch_requests"][ 0 ].patch.job_creation_task_id assert task_id is not None @@ -662,7 +703,7 @@ def _default_side_effect( _generated_function_job_ids: list[FunctionJobID] = [] - async def _register_function_job_side_effect( + async def _batch_register_function_jobs_side_effect( generated_function_job_ids: list, user_id: UserID, function_jobs: FunctionJobList, @@ -675,12 +716,12 @@ async def _register_function_job_side_effect( registered_jobs.append( fake_registered_project_function_job.model_copy(update={"uid": uid}) ) - return registered_jobs + return BatchCreateRegisteredFunctionJobs(created_items=registered_jobs) mock_handler_in_functions_rpc_interface( - "register_function_job", + "batch_register_function_jobs", side_effect=partial( - _register_function_job_side_effect, _generated_function_job_ids + _batch_register_function_jobs_side_effect, _generated_function_job_ids ), ) mock_handler_in_functions_rpc_interface( @@ -709,9 +750,17 @@ async def _register_function_job_collection_side_effect(*args, **kwargs): ) patch_mock = mock_handler_in_functions_rpc_interface( + "batch_patch_registered_function_job", + side_effect=partial( + _batch_patch_registered_function_job, + fake_registered_project_function_job, + ), + ) + + mock_handler_in_functions_rpc_interface( "patch_registered_function_job", side_effect=partial( - _patch_registered_function_job_side_effect, + _patch_registered_function_job, fake_registered_project_function_job, ), ) @@ -737,7 +786,7 @@ async def _register_function_job_collection_side_effect(*args, **kwargs): celery_task_ids = set() for args in patch_mock.call_args_list: - inputs = args.kwargs["registered_function_job_patch_inputs"] + inputs = args.kwargs["function_job_patch_requests"] celery_task_ids = celery_task_ids.union( { input_.patch.job_creation_task_id diff --git a/services/api-server/tests/unit/api_functions/test_api_routers_function_jobs.py b/services/api-server/tests/unit/api_functions/test_api_routers_function_jobs.py index 99fea7684af1..8b4dcea44ea0 100644 --- a/services/api-server/tests/unit/api_functions/test_api_routers_function_jobs.py +++ b/services/api-server/tests/unit/api_functions/test_api_routers_function_jobs.py @@ -20,7 +20,7 @@ RegisteredProjectFunctionJob, ) from models_library.functions import ( - FunctionJobList, + FunctionJob, FunctionJobStatus, RegisteredProjectFunction, RegisteredProjectFunctionJobWithStatus, @@ -75,11 +75,10 @@ async def test_register_function_job( async def _register_function_job_side_effect( user_id: UserID, - function_jobs: FunctionJobList, + function_job: FunctionJob, product_name: ProductName, ): - assert len(function_jobs) == 1 - return [fake_registered_project_function_job] + return fake_registered_project_function_job mock_handler_in_functions_rpc_interface( "register_function_job", side_effect=_register_function_job_side_effect diff --git a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py index 4725559ee3a0..4970430f3cd0 100644 --- a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py +++ b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py @@ -20,6 +20,7 @@ from models_library.api_schemas_long_running_tasks.tasks import TaskGet from models_library.functions import ( FunctionJobList, + FunctionJobPatchRequest, FunctionUserAccessRights, FunctionUserApiAccessRights, ProjectFunction, @@ -27,8 +28,8 @@ RegisteredFunctionJob, RegisteredProjectFunction, RegisteredProjectFunctionJob, - RegisteredProjectFunctionJobPatchInputList, - RegisteredSolverFunctionJobPatchInputList, + RegisteredProjectFunctionJobPatch, + RegisteredSolverFunctionJobPatch, ) from models_library.functions_errors import ( FunctionIDNotFoundError, @@ -496,27 +497,26 @@ async def _register_function_job_side_effect( ), ) - async def _patch_registered_function_job_side_effect( + async def _patch_registered_function_job( product_name: ProductName, user_id: UserID, - registered_function_job_patch_inputs: ( - RegisteredProjectFunctionJobPatchInputList - | RegisteredSolverFunctionJobPatchInputList - ), + function_job_patch_request: FunctionJobPatchRequest, ): - return [ - fake_registered_project_function_job.model_copy( - update={ - "job_creation_task_id": patch.patch.job_creation_task_id, - "uid": patch.uid, - } - ) - for patch in registered_function_job_patch_inputs - ] + patch = function_job_patch_request.patch + assert isinstance(patch, RegisteredProjectFunctionJobPatch) or isinstance( + patch, RegisteredSolverFunctionJobPatch + ) + job = fake_registered_project_function_job.model_copy( + update={ + "job_creation_task_id": patch.job_creation_task_id, + "uid": function_job_patch_request.uid, + } + ) + return job mock_handler_in_functions_rpc_interface( "patch_registered_function_job", - side_effect=_patch_registered_function_job_side_effect, + side_effect=_patch_registered_function_job, ) pre_registered_function_job_data = PreRegisteredFunctionJobData( From 74052c5584905f83991d98663129b02172890b96 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 21 Oct 2025 11:59:59 +0200 Subject: [PATCH 58/63] pylint --- .../unit/api_functions/celery/test_functions_celery.py | 8 ++++---- .../unit/api_functions/test_api_routers_functions.py | 4 ++-- .../wb-api-server/test_function_jobs_controller_rpc.py | 10 ++++------ 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py b/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py index 3d28c0efa15b..4659668812ba 100644 --- a/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py +++ b/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py @@ -168,8 +168,8 @@ async def _batch_patch_registered_function_job( jobs = [] for patch_request in function_job_patch_requests: patch = patch_request.patch - assert isinstance(patch, RegisteredProjectFunctionJobPatch) or isinstance( - patch, RegisteredSolverFunctionJobPatch + assert isinstance( + patch, (RegisteredProjectFunctionJobPatch, RegisteredSolverFunctionJobPatch) ) jobs.append( mock_registered_project_function_job.model_copy( @@ -189,8 +189,8 @@ async def _patch_registered_function_job( function_job_patch_request: FunctionJobPatchRequest, ): patch = function_job_patch_request.patch - assert isinstance(patch, RegisteredProjectFunctionJobPatch) or isinstance( - patch, RegisteredSolverFunctionJobPatch + assert isinstance( + patch, (RegisteredProjectFunctionJobPatch, RegisteredSolverFunctionJobPatch) ) job = mock_registered_project_function_job.model_copy( update={ diff --git a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py index 4970430f3cd0..52d228debb93 100644 --- a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py +++ b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py @@ -503,8 +503,8 @@ async def _patch_registered_function_job( function_job_patch_request: FunctionJobPatchRequest, ): patch = function_job_patch_request.patch - assert isinstance(patch, RegisteredProjectFunctionJobPatch) or isinstance( - patch, RegisteredSolverFunctionJobPatch + assert isinstance( + patch, (RegisteredProjectFunctionJobPatch, RegisteredSolverFunctionJobPatch) ) job = fake_registered_project_function_job.model_copy( update={ diff --git a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py index 9f138bf8a17a..71498f157aba 100644 --- a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py @@ -447,12 +447,10 @@ async def test_find_cached_function_jobs( ] # Register the function job - registered_jobs_batch_create = ( - await webserver_rpc_client.functions.batch_register_function_jobs( - function_jobs=TypeAdapter(FunctionJobList).validate_python(function_jobs), - user_id=logged_user["id"], - product_name=osparc_product_name, - ) + await webserver_rpc_client.functions.batch_register_function_jobs( + function_jobs=TypeAdapter(FunctionJobList).validate_python(function_jobs), + user_id=logged_user["id"], + product_name=osparc_product_name, ) # Find cached function jobs From ad85bd9a6b27c57af867ed78ed5557fb37c49355 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 21 Oct 2025 13:36:30 +0200 Subject: [PATCH 59/63] remove min list length constraint and update openapi specs --- packages/models-library/src/models_library/functions.py | 6 ++---- services/api-server/openapi.json | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/models-library/src/models_library/functions.py b/packages/models-library/src/models_library/functions.py index 21d9a24fda3d..317a9df5b2bf 100644 --- a/packages/models-library/src/models_library/functions.py +++ b/packages/models-library/src/models_library/functions.py @@ -25,7 +25,6 @@ FileID: TypeAlias = UUID InputTypes: TypeAlias = FileID | float | int | bool | str | list -_MIN_LIST_LENGTH: Final[int] = 1 _MAX_LIST_LENGTH: Final[int] = 50 @@ -84,7 +83,7 @@ class FunctionClass(str, Enum): FunctionInputsList: TypeAlias = Annotated[ list[FunctionInputs], - Field(max_length=_MAX_LIST_LENGTH, min_length=_MIN_LIST_LENGTH), + Field(max_length=_MAX_LIST_LENGTH), ] @@ -244,7 +243,7 @@ class RegisteredPythonCodeFunctionJobPatch(BaseModel): Field(discriminator="function_class"), ] FunctionJobList: TypeAlias = Annotated[ - list[FunctionJob], Field(max_length=_MAX_LIST_LENGTH, min_length=_MIN_LIST_LENGTH) + list[FunctionJob], Field(max_length=_MAX_LIST_LENGTH) ] @@ -304,7 +303,6 @@ class FunctionJobPatchRequest(BaseModel): list[FunctionJobPatchRequest], Field( max_length=_MAX_LIST_LENGTH, - min_length=_MIN_LIST_LENGTH, description="List of function job patch requests", ), ] diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index c755705a1a74..c6aa0416b5b1 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -8240,7 +8240,6 @@ } ] }, - "minItems": 1, "maxItems": 50, "title": "Function Inputs List" } From 43059b8aca1be877d9e4b1bd7715e03070b24a93 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 21 Oct 2025 13:42:47 +0200 Subject: [PATCH 60/63] rename lambda --- .../_service_function_jobs_task_client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py index b684ae2182fd..fe3f5ec387c4 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py @@ -381,5 +381,7 @@ async def create_function_job_creation_tasks( ], ) patched_jobs_iter = iter(patched_jobs.updated_items) - _ = lambda job: job if job is not None else next(patched_jobs_iter) - return [_(job) for job in cached_jobs] + resolve_cached_jobs = lambda job: ( + job if job is not None else next(patched_jobs_iter) + ) + return [resolve_cached_jobs(job) for job in cached_jobs] From 3dce8b17e4e1c8717bd8ccc187bd68359e14f3e9 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 21 Oct 2025 13:44:11 +0200 Subject: [PATCH 61/63] remove double FunctionJobCreationTaskStatus import --- .../_service_function_jobs_task_client.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py index fe3f5ec387c4..8dd7e862896b 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py @@ -33,9 +33,6 @@ from pydantic import TypeAdapter from servicelib.celery.models import ExecutionMetadata, TasksQueue, TaskUUID from servicelib.celery.task_manager import TaskManager -from simcore_service_api_server.models.schemas.functions import ( - FunctionJobCreationTaskStatus, -) from sqlalchemy.ext.asyncio import AsyncEngine from ._meta import APP_NAME From ade3efe6afa792b1f37fb456ad0dea684cccf41b Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 21 Oct 2025 13:46:24 +0200 Subject: [PATCH 62/63] don't update status when patching job --- .../functions/_function_jobs_repository.py | 1 - 1 file changed, 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py index 8be65087fb54..c8c5dcec9d01 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py @@ -173,7 +173,6 @@ async def patch_function_jobs( "class_specific_data": class_specific_data, "title": patch_request.patch.title, "description": patch_request.patch.description, - "status": "created", } result = await transaction.execute( From ee7bb375d7e461cd27e13e35d404032100629698 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 21 Oct 2025 13:59:09 +0200 Subject: [PATCH 63/63] status_filter -> cached_job_statuses --- .../rabbitmq/rpc_interfaces/webserver/v1/functions.py | 4 ++-- .../_service_function_jobs.py | 10 +++++----- .../_service_function_jobs_task_client.py | 2 +- .../services_rpc/wb_api_server.py | 2 +- .../functions/_controller/_functions_rpc.py | 4 ++-- .../functions/_function_jobs_repository.py | 6 +++--- .../functions/_functions_service.py | 4 ++-- .../wb-api-server/test_function_jobs_controller_rpc.py | 2 +- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py index aa9693d9b459..4fd249c2ec82 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/v1/functions.py @@ -500,7 +500,7 @@ async def find_cached_function_jobs( user_id: UserID, function_id: FunctionID, inputs: FunctionInputsList, - status_filter: list[FunctionJobStatus] | None = None, + cached_job_statuses: list[FunctionJobStatus] | None = None, ) -> list[RegisteredFunctionJob | None]: """Find cached function jobs.""" return TypeAdapter(list[RegisteredFunctionJob | None]).validate_python( @@ -510,7 +510,7 @@ async def find_cached_function_jobs( user_id=user_id, function_id=function_id, inputs=inputs, - status_filter=status_filter, + cached_job_statuses=cached_job_statuses, ), ) diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py index fd823a078d1a..e4a7e99317dd 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs.py @@ -136,13 +136,13 @@ async def batch_pre_register_function_jobs( self, *, function: RegisteredFunction, - job_inputs: list[JobInputs], + job_input_list: list[JobInputs], ) -> list[PreRegisteredFunctionJobData]: if function.input_schema is not None: is_valid, validation_str = await self.validate_function_inputs( function=function, - job_inputs=job_inputs, + job_inputs=job_input_list, ) if not is_valid: raise FunctionInputsValidationError(error=validation_str) @@ -159,7 +159,7 @@ async def batch_pre_register_function_jobs( project_job_id=None, job_creation_task_id=None, ) - for input_ in job_inputs + for input_ in job_input_list ] batch_registered_jobs = ( await self._web_rpc_client.batch_register_function_jobs( @@ -183,7 +183,7 @@ async def batch_pre_register_function_jobs( solver_job_id=None, job_creation_task_id=None, ) - for input_ in job_inputs + for input_ in job_input_list ] batch_registered_jobs = ( await self._web_rpc_client.batch_register_function_jobs( @@ -205,7 +205,7 @@ async def batch_pre_register_function_jobs( function_job_id=job.uid, job_inputs=input_, ) - for job, input_ in zip(jobs, job_inputs) + for job, input_ in zip(jobs, job_input_list) ] async def batch_patch_registered_function_job( diff --git a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py index 8dd7e862896b..79af528deeb5 100644 --- a/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py +++ b/services/api-server/src/simcore_service_api_server/_service_function_jobs_task_client.py @@ -332,7 +332,7 @@ async def create_function_job_creation_tasks( pre_registered_function_job_data_list = ( await self._function_job_service.batch_pre_register_function_jobs( function=function, - job_inputs=[JobInputs(values=_ or {}) for _ in uncached_inputs], + job_input_list=[JobInputs(values=_ or {}) for _ in uncached_inputs], ) ) diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index 93be96c99139..16b4262ebaeb 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -629,7 +629,7 @@ async def find_cached_function_jobs( product_name=product_name, function_id=function_id, inputs=inputs, - status_filter=status_filter, + cached_job_statuses=status_filter, ) async def get_function_job_collection( diff --git a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py index 9a5b9580754a..bfdc9528c19c 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py +++ b/services/web/server/src/simcore_service_webserver/functions/_controller/_functions_rpc.py @@ -489,7 +489,7 @@ async def find_cached_function_jobs( product_name: ProductName, function_id: FunctionID, inputs: FunctionInputsList, - status_filter: list[FunctionJobStatus] | None, + cached_job_statuses: list[FunctionJobStatus] | None, ) -> list[RegisteredFunctionJob | None]: retrieved_cached_function_jobs = await _functions_service.find_cached_function_jobs( app=app, @@ -497,7 +497,7 @@ async def find_cached_function_jobs( product_name=product_name, function_id=function_id, inputs=inputs, - status_filter=status_filter, + cached_job_statuses=cached_job_statuses, ) assert len(retrieved_cached_function_jobs) == len(inputs) # nosec return retrieved_cached_function_jobs diff --git a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py index c8c5dcec9d01..155a344efcd2 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/functions/_function_jobs_repository.py @@ -325,7 +325,7 @@ async def find_cached_function_jobs( function_id: FunctionID, product_name: ProductName, inputs: FunctionInputsList, - status_filter: list[FunctionJobStatus] | None = None, + cached_job_statuses: list[FunctionJobStatus] | None = None, ) -> list[RegisteredFunctionJobDB | None]: async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: # Get user groups for access check @@ -352,9 +352,9 @@ async def find_cached_function_jobs( function_jobs_table.c.uuid.in_(access_subquery), ( function_jobs_table.c.status.in_( - [status.status for status in status_filter] + [status.status for status in cached_job_statuses] ) - if status_filter is not None + if cached_job_statuses is not None else sqlalchemy.sql.true() ), ) diff --git a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py index 52747022be08..35ddbc54cc6b 100644 --- a/services/web/server/src/simcore_service_webserver/functions/_functions_service.py +++ b/services/web/server/src/simcore_service_webserver/functions/_functions_service.py @@ -455,7 +455,7 @@ async def find_cached_function_jobs( product_name: ProductName, function_id: FunctionID, inputs: FunctionInputsList, - status_filter: list[FunctionJobStatus] | None = None, + cached_job_statuses: list[FunctionJobStatus] | None = None, ) -> list[RegisteredFunctionJob | None]: returned_function_jobs = await _function_jobs_repository.find_cached_function_jobs( app=app, @@ -463,7 +463,7 @@ async def find_cached_function_jobs( product_name=product_name, function_id=function_id, inputs=inputs, - status_filter=status_filter, + cached_job_statuses=cached_job_statuses, ) assert len(returned_function_jobs) == len(inputs) # nosec diff --git a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py index 71498f157aba..5c411b9d4305 100644 --- a/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py +++ b/services/web/server/tests/unit/with_dbs/04/functions/wb-api-server/test_function_jobs_controller_rpc.py @@ -536,7 +536,7 @@ async def test_find_cached_function_jobs_with_status( product_name=osparc_product_name, user_id=logged_user["id"], inputs=[input_], - status_filter=[status], + cached_job_statuses=[status], ) assert len(cached_jobs) == 1 cached_job = cached_jobs[0]