Merge branch 'master' into 2025/fix/director-v0-url-type

Author: mrnicegyu11

.env-devel

@@ -49,6 +49,8 @@ CATALOG_SERVICES_DEFAULT_RESOURCES='{"CPU": {"limit": 0.1, "reservation": 0.1},
 CATALOG_SERVICES_DEFAULT_SPECIFICATIONS='{}'
 CATALOG_TRACING=null
 
+CELERY_RESULT_EXPIRES=P7D
+
 CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}'
 CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG=master-github-latest
 CLUSTERS_KEEPER_DASK_NTHREADS=0

.github/workflows/ci-testing-pull-request.yml

@@ -10,6 +10,20 @@ on:
     branches:
       - "master"
 
+  workflow_dispatch:
+    inputs:
+      target_repo:
+        description: full repository name (e.g. 'ITISFoundation/osparc-simcore')
+        required: true
+        default: "ITISFoundation/osparc-simcore"
+        type: environment
+      target_branch:
+        description: Check backwards compatibility against target_branch in target_repo
+        required: true
+        default: "master"
+        type: environment
+
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -39,7 +53,7 @@ jobs:
       - name: Check openapi specs are up to date
         run: |
           if ! ./ci/github/helpers/openapi-specs-diff.bash diff \
-            https://raw.githubusercontent.com/${{ github.event.pull_request.head.repo.full_name }}/${{ github.event.pull_request.head.sha }} \
+            "https://raw.githubusercontent.com/$GITHUB_REPOSITORY/$GITHUB_SHA" \
             .; then \
             echo "::error:: OAS are not up to date. Run 'make openapi-specs' to update them"; exit 1; \
           fi
@@ -56,10 +70,19 @@ jobs:
           python-version: "3.11"
       - name: checkout
         uses: actions/checkout@v4
+      - name: Set environment variables based on event type
+        run: |
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            echo "REPO=${{ inputs.target_repo }}" >> $GITHUB_ENV
+            echo "BRANCH=${{ inputs.target_branch }}" >> $GITHUB_ENV
+          else
+            echo "REPO=${{ github.event.pull_request.base.repo.full_name }}" >> $GITHUB_ENV
+            echo "BRANCH=${{ github.base_ref }}" >> $GITHUB_ENV
+          fi
       - name: check api-server backwards compatibility
         run: |
           ./scripts/openapi-diff.bash breaking --fail-on ERR\
-            https://raw.githubusercontent.com/${{ github.event.pull_request.base.repo.full_name }}/refs/heads/${{ github.base_ref }}/services/api-server/openapi.json \
+            "https://raw.githubusercontent.com/$REPO/refs/heads/$BRANCH/services/api-server/openapi.json" \
             /specs/services/api-server/openapi.json
 
   all-oas-breaking:
@@ -75,8 +98,17 @@ jobs:
           python-version: "3.11"
       - name: checkout
         uses: actions/checkout@v4
+      - name: Set environment variables based on event type
+        run: |
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            echo "REPO=${{ inputs.target_repo }}" >> $GITHUB_ENV
+            echo "BRANCH=${{ inputs.target_branch }}" >> $GITHUB_ENV
+          else
+            echo "REPO=${{ github.event.pull_request.base.repo.full_name }}" >> $GITHUB_ENV
+            echo "BRANCH=${{ github.base_ref }}" >> $GITHUB_ENV
+          fi
       - name: Check openapi-specs backwards compatibility
         run: |
           ./ci/github/helpers/openapi-specs-diff.bash breaking \
-            https://raw.githubusercontent.com/${{ github.event.pull_request.base.repo.full_name }}/refs/heads/${{ github.base_ref }} \
+            "https://raw.githubusercontent.com/$REPO/refs/heads/$BRANCH" \
             .

api/specs/web-server/_auth_api_keys.py

@@ -10,7 +10,9 @@
 from models_library.rest_error import EnvelopedError
 from simcore_service_webserver._meta import API_VTAG
 from simcore_service_webserver.api_keys._controller_rest import ApiKeysPathParams
-from simcore_service_webserver.api_keys._exceptions_handlers import _TO_HTTP_ERROR_MAP
+from simcore_service_webserver.api_keys._controller_rest_exceptions import (
+    _TO_HTTP_ERROR_MAP,
+)
 
 router = APIRouter(
     prefix=f"/{API_VTAG}",

api/specs/web-server/_catalog.py

@@ -3,8 +3,8 @@
 from fastapi import APIRouter, Depends
 from models_library.api_schemas_api_server.pricing_plans import ServicePricingPlanGet
 from models_library.api_schemas_webserver.catalog import (
+    CatalogLatestServiceGet,
     CatalogServiceGet,
-    CatalogServiceListItem,
     CatalogServiceUpdate,
     ServiceInputGet,
     ServiceInputKey,
@@ -15,13 +15,13 @@
 from models_library.generics import Envelope
 from models_library.rest_pagination import Page
 from simcore_service_webserver._meta import API_VTAG
-from simcore_service_webserver.catalog._rest_controller import (
+from simcore_service_webserver.catalog._controller_rest_schemas import (
+    FromServiceOutputQueryParams,
     ListServiceParams,
+    ServiceInputsPathParams,
+    ServiceOutputsPathParams,
     ServicePathParams,
-    _FromServiceOutputParams,
-    _ServiceInputsPathParams,
-    _ServiceOutputsPathParams,
-    _ToServiceInputsParams,
+    ToServiceInputsQueryParams,
 )
 
 router = APIRouter(
@@ -34,10 +34,9 @@
 
 @router.get(
     "/catalog/services/-/latest",
-    response_model=Page[CatalogServiceListItem],
+    response_model=Page[CatalogLatestServiceGet],
 )
-def list_services_latest(_query: Annotated[ListServiceParams, Depends()]):
-    pass
+def list_services_latest(_query: Annotated[ListServiceParams, Depends()]): ...
 
 
 @router.get(
@@ -71,7 +70,7 @@ def list_service_inputs(
     response_model=Envelope[ServiceInputGet],
 )
 def get_service_input(
-    _path: Annotated[_ServiceInputsPathParams, Depends()],
+    _path: Annotated[ServiceInputsPathParams, Depends()],
 ): ...
 
 
@@ -81,7 +80,7 @@ def get_service_input(
 )
 def get_compatible_inputs_given_source_output(
     _path: Annotated[ServicePathParams, Depends()],
-    _query: Annotated[_FromServiceOutputParams, Depends()],
+    _query: Annotated[FromServiceOutputQueryParams, Depends()],
 ): ...
 
 
@@ -99,7 +98,7 @@ def list_service_outputs(
     response_model=Envelope[list[ServiceOutputGet]],
 )
 def get_service_output(
-    _path: Annotated[_ServiceOutputsPathParams, Depends()],
+    _path: Annotated[ServiceOutputsPathParams, Depends()],
 ): ...
 
 
@@ -109,7 +108,7 @@ def get_service_output(
 )
 def get_compatible_outputs_given_target_input(
     _path: Annotated[ServicePathParams, Depends()],
-    _query: Annotated[_ToServiceInputsParams, Depends()],
+    _query: Annotated[ToServiceInputsQueryParams, Depends()],
 ): ...
 
 

api/specs/web-server/_catalog_tags.py

@@ -10,7 +10,7 @@
 from models_library.api_schemas_webserver.catalog import CatalogServiceGet
 from models_library.generics import Envelope
 from simcore_service_webserver._meta import API_VTAG
-from simcore_service_webserver.catalog._rest_tags_controller import (
+from simcore_service_webserver.catalog._controller_rest_schemas import (
     ServicePathParams,
     ServiceTagPathParams,
 )

api/specs/web-server/_storage.py

@@ -210,15 +210,15 @@ async def export_data(data_export: DataExportPost, location_id: LocationID):
     response_model=Envelope[StorageAsyncJobStatus],
     name="get_async_job_status",
 )
-async def get_async_job_status(storage_async_job_get: StorageAsyncJobGet, job_id: UUID):
+async def get_async_job_status(job_id: UUID):
     """Get async job status"""
 
 
 @router.post(
     "/storage/async-jobs/{job_id}:abort",
     name="abort_async_job",
 )
-async def abort_async_job(storage_async_job_get: StorageAsyncJobGet, job_id: UUID):
+async def abort_async_job(job_id: UUID):
     """aborts execution of an async job"""
 
 
@@ -227,7 +227,7 @@ async def abort_async_job(storage_async_job_get: StorageAsyncJobGet, job_id: UUI
     response_model=Envelope[StorageAsyncJobResult],
     name="get_async_job_result",
 )
-async def get_async_job_result(storage_async_job_get: StorageAsyncJobGet, job_id: UUID):
+async def get_async_job_result(job_id: UUID):
     """Get the result of the async job"""
 
 

packages/common-library/src/common_library/async_tools.py

@@ -0,0 +1,24 @@
+import asyncio
+import functools
+from collections.abc import Awaitable, Callable
+from concurrent.futures import Executor
+from typing import ParamSpec, TypeVar
+
+R = TypeVar("R")
+P = ParamSpec("P")
+
+
+def make_async(
+    executor: Executor | None = None,
+) -> Callable[[Callable[P, R]], Callable[P, Awaitable[R]]]:
+    def decorator(func: Callable[P, R]) -> Callable[P, Awaitable[R]]:
+        @functools.wraps(func)
+        async def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+            loop = asyncio.get_running_loop()
+            return await loop.run_in_executor(
+                executor, functools.partial(func, *args, **kwargs)
+            )
+
+        return wrapper
+
+    return decorator

packages/common-library/tests/test_async_tools.py

@@ -0,0 +1,45 @@
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
+
+import pytest
+from common_library.async_tools import make_async
+
+
+@make_async()
+def sync_function(x: int, y: int) -> int:
+    return x + y
+
+
+@make_async()
+def sync_function_with_exception() -> None:
+    raise ValueError("This is an error!")
+
+
+@pytest.mark.asyncio
+async def test_make_async_returns_coroutine():
+    result = sync_function(2, 3)
+    assert asyncio.iscoroutine(result), "Function should return a coroutine"
+
+
+@pytest.mark.asyncio
+async def test_make_async_execution():
+    result = await sync_function(2, 3)
+    assert result == 5, "Function should return 5"
+
+
+@pytest.mark.asyncio
+async def test_make_async_exception():
+    with pytest.raises(ValueError, match="This is an error!"):
+        await sync_function_with_exception()
+
+
+@pytest.mark.asyncio
+async def test_make_async_with_executor():
+    executor = ThreadPoolExecutor()
+
+    @make_async(executor)
+    def heavy_computation(x: int) -> int:
+        return x * x
+
+    result = await heavy_computation(4)
+    assert result == 16, "Function should return 16"

packages/models-library/src/models_library/api_schemas_catalog/services.py

@@ -202,8 +202,15 @@ class _BaseServiceGetV2(CatalogOutputSchema):
 
     access_rights: dict[GroupID, ServiceGroupAccessRightsV2] | None
 
-    classifiers: list[str] | None = []
-    quality: dict[str, Any] = {}
+    classifiers: Annotated[
+        list[str] | None,
+        Field(default_factory=list),
+    ] = DEFAULT_FACTORY
+
+    quality: Annotated[
+        dict[str, Any],
+        Field(default_factory=dict),
+    ] = DEFAULT_FACTORY
 
     model_config = ConfigDict(
         extra="forbid",
@@ -212,6 +219,34 @@ class _BaseServiceGetV2(CatalogOutputSchema):
     )
 
 
+class LatestServiceGet(_BaseServiceGetV2):
+    release: Annotated[
+        ServiceRelease,
+        Field(description="release information of current (latest) service"),
+    ]
+
+    @staticmethod
+    def _update_json_schema_extra(schema: JsonDict) -> None:
+        schema.update(
+            {
+                "examples": [
+                    {
+                        **_EXAMPLE_SLEEPER,  # v2.2.1  (latest)
+                        "release": {
+                            "version": _EXAMPLE_SLEEPER["version"],
+                            "version_display": "Summer Release",
+                            "released": "2025-07-20T15:00:00",
+                        },
+                    }
+                ]
+            }
+        )
+
+    model_config = ConfigDict(
+        json_schema_extra=_update_json_schema_extra,
+    )
+
+
 class ServiceGetV2(_BaseServiceGetV2):
     # Model used in catalog's rpc and rest interfaces
     history: Annotated[
@@ -235,7 +270,7 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
                             {
                                 "version": _EXAMPLE_SLEEPER["version"],
                                 "version_display": "Summer Release",
-                                "released": "2024-07-20T15:00:00",
+                                "released": "2024-07-21T15:00:00",
                             },
                             {
                                 "version": "2.0.0",
@@ -263,7 +298,7 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
                             },
                             {
                                 "version": "0.9.0",
-                                "retired": "2024-07-20T15:00:00",
+                                "retired": "2024-07-20T16:00:00",
                             },
                             {"version": "0.8.0"},
                             {"version": "0.1.0"},
@@ -288,21 +323,9 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
     )
 
 
-class ServiceListItem(_BaseServiceGetV2):
-    history: Annotated[
-        list[ServiceRelease],
-        Field(
-            default_factory=list,
-            deprecated=True,
-            description="History will be replaced by current 'release' instead",
-            json_schema_extra={"default": []},
-        ),
-    ] = DEFAULT_FACTORY
-
-
 PageRpcServicesGetV2: TypeAlias = PageRpc[
     # WARNING: keep this definition in models_library and not in the RPC interface
-    ServiceListItem
+    LatestServiceGet
 ]
 
 ServiceResourcesGet: TypeAlias = ServiceResourcesDict

packages/models-library/src/models_library/api_schemas_rpc_async_jobs/async_jobs.py

@@ -1,10 +1,8 @@
-from datetime import datetime
 from typing import Any, TypeAlias
 from uuid import UUID
 
 from models_library.users import UserID
-from pydantic import BaseModel, model_validator
-from typing_extensions import Self
+from pydantic import BaseModel
 
 from ..progress_bar import ProgressReport
 
@@ -15,18 +13,6 @@ class AsyncJobStatus(BaseModel):
     job_id: AsyncJobId
     progress: ProgressReport
     done: bool
-    started: datetime
-    stopped: datetime | None
-
-    @model_validator(mode="after")
-    def _check_consistency(self) -> Self:
-        is_done = self.done
-        is_stopped = self.stopped is not None
-
-        if is_done != is_stopped:
-            msg = f"Inconsistent data: {self.done=}, {self.stopped=}"
-            raise ValueError(msg)
-        return self
 
 
 class AsyncJobResult(BaseModel):