✨ Create custom generate json schema for resolving references (#8295)

bisgaard-itis · web-flow · commit 9ebb8305d208 · 2025-09-10T09:13:44.000+02:00
diff --git a/packages/aws-library/requirements/_base.txt b/packages/aws-library/requirements/_base.txt
@@ -118,6 +118,10 @@ jmespath==1.0.1
     #   aiobotocore
     #   boto3
     #   botocore
+jsonref==1.1.0
+    # via
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
 jsonschema==4.23.0
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
diff --git a/packages/celery-library/requirements/_base.txt b/packages/celery-library/requirements/_base.txt
@@ -109,6 +109,10 @@ idna==3.10
     #   yarl
 importlib-metadata==8.6.1
     # via opentelemetry-api
+jsonref==1.1.0
+    # via
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
 jsonschema==4.23.0
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
diff --git a/packages/dask-task-models-library/requirements/_base.txt b/packages/dask-task-models-library/requirements/_base.txt
@@ -40,6 +40,8 @@ jinja2==3.1.6
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   distributed
+jsonref==1.1.0
+    # via -r requirements/../../../packages/models-library/requirements/_base.in
 jsonschema==4.23.0
     # via -r requirements/../../../packages/models-library/requirements/_base.in
 jsonschema-specifications==2025.4.1
diff --git a/packages/models-library/requirements/_base.in b/packages/models-library/requirements/_base.in
@@ -5,6 +5,7 @@
 --requirement ../../../packages/common-library/requirements/_base.in
 
 arrow
+jsonref
 jsonschema
 orjson
 pydantic-extra-types
diff --git a/packages/models-library/requirements/_base.txt b/packages/models-library/requirements/_base.txt
@@ -12,6 +12,8 @@ email-validator==2.2.0
     # via pydantic
 idna==3.10
     # via email-validator
+jsonref==1.1.0
+    # via -r requirements/_base.in
 jsonschema==4.23.0
     # via -r requirements/_base.in
 jsonschema-specifications==2024.10.1
diff --git a/packages/models-library/src/models_library/progress_bar.py b/packages/models-library/src/models_library/progress_bar.py
@@ -1,5 +1,6 @@
 from typing import Literal, TypeAlias
 
+from models_library.utils.json_schema import GenerateResolvedJsonSchema
 from pydantic import BaseModel, ConfigDict
 from pydantic.config import JsonDict
 
@@ -99,17 +100,9 @@ def composed_message(self) -> str:
                 {
                     "actual_value": 0.3,
                     "total": 1.0,
-                    "message": {
-                        "description": "downloading",
-                        "current": 2.0,
-                        "total": 5,
-                        "sub": {
-                            "description": "port 2",
-                            "current": 12.2,
-                            "total": 123,
-                            "unit": "Byte",
-                        },
-                    },
+                    "message": ProgressStructuredMessage.model_json_schema(
+                        schema_generator=GenerateResolvedJsonSchema
+                    )["examples"][2],
                 },
             ]
         },
diff --git a/packages/models-library/src/models_library/utils/json_schema.py b/packages/models-library/src/models_library/utils/json_schema.py
@@ -5,6 +5,7 @@
 
 See how is used to validate input/output content-schemas of service models
 """
+
 # SEE possible enhancements in https://github.com/ITISFoundation/osparc-simcore/issues/3008
 
 
@@ -13,8 +14,11 @@
 from copy import deepcopy
 from typing import Any
 
+import jsonref  # type: ignore[import-untyped]
 import jsonschema
 from jsonschema import validators
+from pydantic.json_schema import GenerateJsonSchema, JsonSchemaMode, JsonSchemaValue
+from pydantic_core import CoreSchema
 
 # ERRORS
 
@@ -92,6 +96,19 @@ def any_ref_key(obj):
     return False
 
 
+class GenerateResolvedJsonSchema(GenerateJsonSchema):
+    """Generates a json-schema with all $ref resolved
+    Usage: pydantic_base_model.model_json_schema(schema_generator=GenerateResolvedJsonSchema) returns a json schema where it is guaranteed that all json references are resolved.
+    """
+
+    def generate(
+        self, schema: CoreSchema, mode: JsonSchemaMode = "validation"
+    ) -> JsonSchemaValue:
+        schema_value = super().generate(schema=schema, mode=mode)
+        schema_value = jsonref.replace_refs(schema_value, jsonschema=True)
+        return JsonSchemaValue(schema_value)
+
+
 __all__: tuple[str, ...] = (
     "any_ref_key",
     "InvalidJsonSchema",
diff --git a/packages/notifications-library/requirements/_base.txt b/packages/notifications-library/requirements/_base.txt
@@ -37,6 +37,8 @@ jinja2==3.1.6
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/_base.in
+jsonref==1.1.0
+    # via -r requirements/../../../packages/models-library/requirements/_base.in
 jsonschema==4.23.0
     # via -r requirements/../../../packages/models-library/requirements/_base.in
 jsonschema-specifications==2024.10.1
diff --git a/packages/service-integration/requirements/_base.txt b/packages/service-integration/requirements/_base.txt
@@ -52,6 +52,8 @@ jinja2==3.1.6
     #   jinja2-time
 jinja2-time==0.2.0
     # via -r requirements/_base.in
+jsonref==1.1.0
+    # via -r requirements/../../../packages/models-library/requirements/_base.in
 jsonschema==4.23.0
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
diff --git a/packages/service-library/requirements/_base.txt b/packages/service-library/requirements/_base.txt
@@ -79,6 +79,8 @@ idna==3.10
     #   yarl
 importlib-metadata==8.5.0
     # via opentelemetry-api
+jsonref==1.1.0
+    # via -r requirements/../../../packages/models-library/requirements/_base.in
 jsonschema==4.23.0
     # via -r requirements/../../../packages/models-library/requirements/_base.in
 jsonschema-specifications==2024.10.1
diff --git a/packages/simcore-sdk/requirements/_base.txt b/packages/simcore-sdk/requirements/_base.txt
@@ -136,6 +136,10 @@ idna==3.10
     #   yarl
 importlib-metadata==8.5.0
     # via opentelemetry-api
+jsonref==1.1.0
+    # via
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
 jsonschema==4.23.0
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
diff --git a/services/agent/requirements/_base.txt b/services/agent/requirements/_base.txt
@@ -166,6 +166,10 @@ jinja2==3.1.6
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   fastapi
+jsonref==1.1.0
+    # via
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
 jsonschema==4.23.0
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
diff --git a/services/api-server/requirements/_base.txt b/services/api-server/requirements/_base.txt
@@ -381,6 +381,14 @@ jinja2==3.1.6
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   fastapi
+jsonref==1.1.0
+    # via
+    #   -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
 jsonschema==3.2.0
     # via
     #   -c requirements/./constraints.txt
diff --git a/services/api-server/requirements/_test.txt b/services/api-server/requirements/_test.txt
@@ -206,7 +206,9 @@ jsonpickle==4.0.2
 jsonpointer==3.0.0
     # via jsonpatch
 jsonref==1.1.0
-    # via -r requirements/_test.in
+    # via
+    #   -c requirements/_base.txt
+    #   -r requirements/_test.in
 jsonschema==3.2.0
     # via
     #   -c requirements/_base.txt
diff --git a/services/api-server/tests/unit/api_functions/test_api_routers_function_jobs.py b/services/api-server/tests/unit/api_functions/test_api_routers_function_jobs.py
@@ -31,6 +31,7 @@
 from models_library.projects_state import RunningState
 from models_library.rest_pagination import PageMetaInfoLimitOffset
 from models_library.users import UserID
+from models_library.utils.json_schema import GenerateResolvedJsonSchema
 from pytest_mock import MockerFixture, MockType
 from servicelib.celery.models import TaskFilter, TaskState, TaskStatus, TaskUUID
 from simcore_service_api_server._meta import API_VTAG
@@ -303,9 +304,9 @@ async def _get_task_status(
                     attempt=1,
                     unit=None,
                     message=ProgressStructuredMessage.model_validate(
-                        ProgressStructuredMessage.model_json_schema()["$defs"][
-                            "ProgressStructuredMessage"
-                        ]["examples"][0]
+                        ProgressStructuredMessage.model_json_schema(
+                            schema_generator=GenerateResolvedJsonSchema
+                        )["examples"][0]
                     ),
                 ),
             )
diff --git a/services/api-server/tests/unit/test_tasks.py b/services/api-server/tests/unit/test_tasks.py
@@ -11,6 +11,7 @@
 from httpx import AsyncClient, BasicAuth
 from models_library.api_schemas_long_running_tasks.tasks import TaskGet, TaskStatus
 from models_library.progress_bar import ProgressReport, ProgressStructuredMessage
+from models_library.utils.json_schema import GenerateResolvedJsonSchema
 from pytest_mock import MockerFixture, MockType
 from servicelib.celery.models import TaskState
 from servicelib.celery.models import TaskStatus as CeleryTaskStatus
@@ -143,11 +144,9 @@ async def test_get_task_result(
                     total=1.0,
                     unit="Byte",
                     message=ProgressStructuredMessage.model_validate(
-                        {
-                            "description": "some description",
-                            "current": 12.2,
-                            "total": 123,
-                        }
+                        ProgressStructuredMessage.model_json_schema(
+                            schema_generator=GenerateResolvedJsonSchema
+                        )["examples"][0]
                     ),
                 ),
             ),
@@ -167,11 +166,9 @@ async def test_get_task_result(
                     total=1.0,
                     unit="Byte",
                     message=ProgressStructuredMessage.model_validate(
-                        {
-                            "description": "some description",
-                            "current": 12.2,
-                            "total": 123,
-                        }
+                        ProgressStructuredMessage.model_json_schema(
+                            schema_generator=GenerateResolvedJsonSchema
+                        )["examples"][0]
                     ),
                 ),
             ),
diff --git a/services/autoscaling/requirements/_base.txt b/services/autoscaling/requirements/_base.txt
@@ -279,6 +279,12 @@ jmespath==1.0.1
     #   aiobotocore
     #   boto3
     #   botocore
+jsonref==1.1.0
+    # via
+    #   -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
 jsonschema==4.23.0
     # via
     #   -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in
diff --git a/services/catalog/requirements/_base.txt b/services/catalog/requirements/_base.txt
@@ -182,6 +182,10 @@ jinja2==3.1.6
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   fastapi
+jsonref==1.1.0
+    # via
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
 jsonschema==4.23.0
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
diff --git a/services/clusters-keeper/requirements/_base.txt b/services/clusters-keeper/requirements/_base.txt
@@ -277,6 +277,12 @@ jmespath==1.0.1
     #   aiobotocore
     #   boto3
     #   botocore
+jsonref==1.1.0
+    # via
+    #   -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
 jsonschema==4.23.0
     # via
     #   -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in
diff --git a/services/dask-sidecar/requirements/_base.txt b/services/dask-sidecar/requirements/_base.txt
@@ -173,6 +173,11 @@ jmespath==1.0.1
     # via
     #   aiobotocore
     #   botocore
+jsonref==1.1.0
+    # via
+    #   -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
 jsonschema==4.23.0
     # via
     #   -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in
diff --git a/services/datcore-adapter/requirements/_base.txt b/services/datcore-adapter/requirements/_base.txt
@@ -179,6 +179,10 @@ jmespath==1.0.1
     # via
     #   boto3
     #   botocore
+jsonref==1.1.0
+    # via
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
 jsonschema==4.21.1
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
diff --git a/services/director-v2/requirements/_base.txt b/services/director-v2/requirements/_base.txt
@@ -318,6 +318,13 @@ jinja2==3.1.6
     #   -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
     #   distributed
     #   fastapi
+jsonref==1.1.0
+    # via
+    #   -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
 jsonschema==4.23.0
     # via
     #   -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in
diff --git a/services/director/requirements/_base.txt b/services/director/requirements/_base.txt
@@ -167,6 +167,10 @@ jinja2==3.1.6
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   fastapi
+jsonref==1.1.0
+    # via
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
 jsonschema==4.23.0
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
diff --git a/services/director/requirements/_test.txt b/services/director/requirements/_test.txt
@@ -71,7 +71,9 @@ idna==3.10
 iniconfig==2.0.0
     # via pytest
 jsonref==1.1.0
-    # via -r requirements/_test.in
+    # via
+    #   -c requirements/_base.txt
+    #   -r requirements/_test.in
 multidict==6.1.0
     # via
     #   -c requirements/_base.txt
diff --git a/services/docker-api-proxy/requirements/_test.txt b/services/docker-api-proxy/requirements/_test.txt
@@ -114,6 +114,10 @@ importlib-metadata==8.5.0
     # via opentelemetry-api
 iniconfig==2.0.0
     # via pytest
+jsonref==1.1.0
+    # via
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
 jsonschema==4.23.0
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
diff --git a/services/dynamic-scheduler/requirements/_base.txt b/services/dynamic-scheduler/requirements/_base.txt
@@ -196,6 +196,10 @@ jinja2==3.1.6
     #   -c requirements/../../../requirements/constraints.txt
     #   fastapi
     #   nicegui
+jsonref==1.1.0
+    # via
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
 jsonschema==4.23.0
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
diff --git a/services/dynamic-sidecar/requirements/_base.txt b/services/dynamic-sidecar/requirements/_base.txt
@@ -267,6 +267,12 @@ jinja2==3.1.6
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   fastapi
+jsonref==1.1.0
+    # via
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
 jsonschema==4.23.0
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
diff --git a/services/efs-guardian/requirements/_base.txt b/services/efs-guardian/requirements/_base.txt
@@ -265,6 +265,12 @@ jmespath==1.0.1
     #   aiobotocore
     #   boto3
     #   botocore
+jsonref==1.1.0
+    # via
+    #   -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
 jsonschema==4.24.0
     # via
     #   -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in
diff --git a/services/invitations/requirements/_base.txt b/services/invitations/requirements/_base.txt
diff --git a/services/notifications/requirements/_base.txt b/services/notifications/requirements/_base.txt
diff --git a/services/payments/requirements/_base.txt b/services/payments/requirements/_base.txt
diff --git a/services/payments/requirements/_test.txt b/services/payments/requirements/_test.txt
diff --git a/services/resource-usage-tracker/requirements/_base.txt b/services/resource-usage-tracker/requirements/_base.txt
diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt
diff --git a/services/storage/requirements/_test.txt b/services/storage/requirements/_test.txt
diff --git a/services/web/server/requirements/_base.txt b/services/web/server/requirements/_base.txt
diff --git a/services/web/server/requirements/_test.txt b/services/web/server/requirements/_test.txt
diff --git a/tests/swarm-deploy/requirements/_test.txt b/tests/swarm-deploy/requirements/_test.txt
