From 93d810a8c849e8807ef8dd3ca28352998a99ab8b Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Tue, 15 Apr 2025 14:28:48 +0200 Subject: [PATCH 1/9] fix typo --- .../modules/long_running_tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py index 83458fd9d1e0..a1c0dfead7ae 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py @@ -328,7 +328,7 @@ async def _send_resource_tracking_stop(platform_status: SimcorePlatformStatus): progress.update(message="done", percent=ProgressPercent(0.99)) -def _get_satate_folders_size(paths: list[Path]) -> int: +def _get_state_folders_size(paths: list[Path]) -> int: total_size: int = 0 for path in paths: for file in path.rglob("*"): @@ -406,7 +406,7 @@ async def task_restore_state( ) progress.update(message="state restored", percent=ProgressPercent(0.99)) - return _get_satate_folders_size(state_paths) + return _get_state_folders_size(state_paths) async def _save_state_folder( @@ -471,7 +471,7 @@ async def task_save_state( await post_sidecar_log_message(app, "Finished state saving", log_level=logging.INFO) progress.update(message="finished state saving", percent=ProgressPercent(0.99)) - return _get_satate_folders_size(state_paths) + return _get_state_folders_size(state_paths) async def task_ports_inputs_pull( From fe8526b9be2c9b6ef7b14f4fb2e38d27233a0b58 Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Tue, 15 Apr 2025 16:35:32 +0200 Subject: [PATCH 2/9] init --- .../services_metadata_published.py | 12 +++++++ ...ntroduce_legacy_service_identification_.py | 36 +++++++++++++++++++ .../models/services.py | 7 ++++ .../core/background_tasks.py | 9 ++++- .../models/services_db.py | 1 + .../registry_proxy.py | 4 ++- 6 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/354b5d921312_introduce_legacy_service_identification_.py diff --git a/packages/models-library/src/models_library/services_metadata_published.py b/packages/models-library/src/models_library/services_metadata_published.py index 51fba05b7f42..5a3c09d95d94 100644 --- a/packages/models-library/src/models_library/services_metadata_published.py +++ b/packages/models-library/src/models_library/services_metadata_published.py @@ -149,6 +149,18 @@ class ServiceMetaDataPublished(ServiceKeyVersion, ServiceBaseDisplay): alias="progress_regexp", description="regexp pattern for detecting computational service's progress", ) + inputs_path: str | None = Field( + None, + description="if this is present, the service is a modern style dv2 dynamic service", + ) + outputs_path: str | None = Field( + None, + description="if this is present, the service is a modern style dv2 dynamic service", + ) + state_paths: str | None = Field( + None, + description="if this is present, the service is a modern style dv2 dynamic service", + ) # SEE https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys image_digest: str | None = Field( diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/354b5d921312_introduce_legacy_service_identification_.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/354b5d921312_introduce_legacy_service_identification_.py new file mode 100644 index 000000000000..3da3e04896e8 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/354b5d921312_introduce_legacy_service_identification_.py @@ -0,0 +1,36 @@ +"""Introduce legacy service identification column + +Revision ID: 354b5d921312 +Revises: cf8f743fd0b7 +Create Date: 2025-04-15 13:53:44.404695+00:00 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "354b5d921312" +down_revision = "cf8f743fd0b7" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "services_meta_data", + sa.Column( + "is_legacy_dynamic_service", + sa.Boolean(), + server_default=sa.text("false"), + nullable=False, + ), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("services_meta_data", "is_legacy_dynamic_service") + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/services.py b/packages/postgres-database/src/simcore_postgres_database/models/services.py index ec12f0f3ca87..53d37a0c67b3 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/services.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/services.py @@ -119,6 +119,13 @@ doc="Timestamp when the service is retired (editable)." "A fixed time before this date, service is marked as deprecated.", ), + sa.Column( + "is_legacy_dynamic_service", + sa.Boolean, + nullable=False, + server_default=sa.false(), + doc="Is the service a legacy-style dynamic service? This is false for comp-services and special items like filepickers.", + ), sa.PrimaryKeyConstraint("key", "version", name="services_meta_data_pk"), ) diff --git a/services/catalog/src/simcore_service_catalog/core/background_tasks.py b/services/catalog/src/simcore_service_catalog/core/background_tasks.py index fea4c6537d8a..d4ffb05a92da 100644 --- a/services/catalog/src/simcore_service_catalog/core/background_tasks.py +++ b/services/catalog/src/simcore_service_catalog/core/background_tasks.py @@ -89,9 +89,16 @@ def _by_version(t: tuple[ServiceKey, ServiceVersion]) -> Version: ) # set the service in the DB + _is_legacy_dynamic_service: bool = all( + service_metadata.inputs_path is not None, + service_metadata.outputs_path is not None, + service_metadata.state_paths is not None, + ) await services_repo.create_or_update_service( ServiceMetaDataDBCreate( - **service_metadata.model_dump(exclude_unset=True), owner=owner_gid + **service_metadata.model_dump(exclude_unset=True, exclude={""}), + owner=owner_gid, + is_legacy_dynamic_service=_is_legacy_dynamic_service, ), service_access_rights, ) diff --git a/services/catalog/src/simcore_service_catalog/models/services_db.py b/services/catalog/src/simcore_service_catalog/models/services_db.py index 2ad800d2b44d..bc7d2a3c1ecd 100644 --- a/services/catalog/src/simcore_service_catalog/models/services_db.py +++ b/services/catalog/src/simcore_service_catalog/models/services_db.py @@ -118,6 +118,7 @@ class ServiceMetaDataDBCreate(BaseModel): # lifecycle deprecated: datetime | None = None + is_legacy_dynamic_service: bool = False @staticmethod def _update_json_schema_extra(schema: JsonDict) -> None: diff --git a/services/director/src/simcore_service_director/registry_proxy.py b/services/director/src/simcore_service_director/registry_proxy.py index f90373bb2f18..4dccc6ef13a8 100644 --- a/services/director/src/simcore_service_director/registry_proxy.py +++ b/services/director/src/simcore_service_director/registry_proxy.py @@ -410,7 +410,9 @@ async def get_image_details( if not labels: return image_details for key in labels: - if not key.startswith("io.simcore."): + if not key.startswith("io.simcore.") and not key.startswith( + "simcore.service." + ): # This adds input_paths, output_paths, state_paths continue try: label_data = json.loads(labels[key]) From 0dcca0c07492a45c3a1cc284d210986b8f89a96d Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Tue, 15 Apr 2025 17:54:52 +0200 Subject: [PATCH 3/9] Bugfix --- .../src/simcore_service_catalog/core/background_tasks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/catalog/src/simcore_service_catalog/core/background_tasks.py b/services/catalog/src/simcore_service_catalog/core/background_tasks.py index d4ffb05a92da..304798479fa1 100644 --- a/services/catalog/src/simcore_service_catalog/core/background_tasks.py +++ b/services/catalog/src/simcore_service_catalog/core/background_tasks.py @@ -89,10 +89,10 @@ def _by_version(t: tuple[ServiceKey, ServiceVersion]) -> Version: ) # set the service in the DB - _is_legacy_dynamic_service: bool = all( - service_metadata.inputs_path is not None, - service_metadata.outputs_path is not None, - service_metadata.state_paths is not None, + _is_legacy_dynamic_service: bool = ( + service_metadata.inputs_path is not None + and ervice_metadata.outputs_path is not None + and service_metadata.state_paths is not None ) await services_repo.create_or_update_service( ServiceMetaDataDBCreate( From 7f26d3b0c785f79bb27702e358d074ebe62a5163 Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Tue, 15 Apr 2025 18:36:28 +0200 Subject: [PATCH 4/9] Adjust comment --- .../src/simcore_service_director/api/rest/_services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/director/src/simcore_service_director/api/rest/_services.py b/services/director/src/simcore_service_director/api/rest/_services.py index a82699331ce0..3d04a04ebd2c 100644 --- a/services/director/src/simcore_service_director/api/rest/_services.py +++ b/services/director/src/simcore_service_director/api/rest/_services.py @@ -57,7 +57,7 @@ async def list_services( the_app, registry_proxy.ServiceType.DYNAMIC ) # NOTE: the validation is done in the catalog. This entrypoint IS and MUST BE only used by the catalog!! - # NOTE2: the catalog will directly talk to the registry see case #2165 [https://github.com/ITISFoundation/osparc-simcore/issues/2165] + # NOTE2: the catalog might eventually directly talk to the registry see case #2165 [https://github.com/ITISFoundation/osparc-simcore/issues/2165] # services = node_validator.validate_nodes(services) return Envelope[list[dict[str, Any]]](data=services) except RegistryConnectionError as err: @@ -96,7 +96,7 @@ async def list_service_labels( ) from err -@router.get("/services/{service_key:path}/{service_version}") +@router.get("/services/{service_key:path}/{service_version}") # <-- MD async def get_service( the_app: Annotated[FastAPI, Depends(get_app)], service_key: ServiceKey, From 31c5082d6b98ebafbc176f3d373c574d0d10fedb Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Wed, 16 Apr 2025 14:00:36 +0200 Subject: [PATCH 5/9] Bugfixes --- .../src/simcore_service_catalog/core/background_tasks.py | 6 +++--- .../director/src/simcore_service_director/registry_proxy.py | 6 +++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/services/catalog/src/simcore_service_catalog/core/background_tasks.py b/services/catalog/src/simcore_service_catalog/core/background_tasks.py index 304798479fa1..24f100eea198 100644 --- a/services/catalog/src/simcore_service_catalog/core/background_tasks.py +++ b/services/catalog/src/simcore_service_catalog/core/background_tasks.py @@ -90,9 +90,9 @@ def _by_version(t: tuple[ServiceKey, ServiceVersion]) -> Version: # set the service in the DB _is_legacy_dynamic_service: bool = ( - service_metadata.inputs_path is not None - and ervice_metadata.outputs_path is not None - and service_metadata.state_paths is not None + service_metadata.inputs_path is None + and service_metadata.outputs_path is None + and service_metadata.state_paths is None ) await services_repo.create_or_update_service( ServiceMetaDataDBCreate( diff --git a/services/director/src/simcore_service_director/registry_proxy.py b/services/director/src/simcore_service_director/registry_proxy.py index 4dccc6ef13a8..01f651d7eb53 100644 --- a/services/director/src/simcore_service_director/registry_proxy.py +++ b/services/director/src/simcore_service_director/registry_proxy.py @@ -412,11 +412,15 @@ async def get_image_details( for key in labels: if not key.startswith("io.simcore.") and not key.startswith( "simcore.service." - ): # This adds input_paths, output_paths, state_paths + ): # Keeping "simcore.service." adds additonally input_paths, output_paths, state_paths continue try: label_data = json.loads(labels[key]) for label_key in label_data: + if isinstance( + label_key, dict + ): # Dicts from "simcore.service." docker image labels are omitted. + continue image_details[label_key] = label_data[label_key] except json.decoder.JSONDecodeError: logging.exception( From f3896b1ca9f1a46ed8690d24774c7c6c3722c993 Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Wed, 16 Apr 2025 14:05:54 +0200 Subject: [PATCH 6/9] Rename legacy --> classic --- .../354b5d921312_introduce_legacy_service_identification_.py | 4 ++-- .../src/simcore_postgres_database/models/services.py | 4 ++-- .../src/simcore_service_catalog/core/background_tasks.py | 4 ++-- .../catalog/src/simcore_service_catalog/models/services_db.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/354b5d921312_introduce_legacy_service_identification_.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/354b5d921312_introduce_legacy_service_identification_.py index 3da3e04896e8..5726cfd88e2e 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/354b5d921312_introduce_legacy_service_identification_.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/354b5d921312_introduce_legacy_service_identification_.py @@ -21,7 +21,7 @@ def upgrade(): op.add_column( "services_meta_data", sa.Column( - "is_legacy_dynamic_service", + "is_classic_dynamic_service", sa.Boolean(), server_default=sa.text("false"), nullable=False, @@ -32,5 +32,5 @@ def upgrade(): def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.drop_column("services_meta_data", "is_legacy_dynamic_service") + op.drop_column("services_meta_data", "is_classic_dynamic_service") # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/services.py b/packages/postgres-database/src/simcore_postgres_database/models/services.py index 53d37a0c67b3..1759dc6e9d93 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/services.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/services.py @@ -120,11 +120,11 @@ "A fixed time before this date, service is marked as deprecated.", ), sa.Column( - "is_legacy_dynamic_service", + "is_classic_dynamic_service", sa.Boolean, nullable=False, server_default=sa.false(), - doc="Is the service a legacy-style dynamic service? This is false for comp-services and special items like filepickers.", + doc="Is the service a legacy-style classic dynamic service? This is false for comp-services and special items like filepickers.", ), sa.PrimaryKeyConstraint("key", "version", name="services_meta_data_pk"), ) diff --git a/services/catalog/src/simcore_service_catalog/core/background_tasks.py b/services/catalog/src/simcore_service_catalog/core/background_tasks.py index 24f100eea198..5cdff6b8129e 100644 --- a/services/catalog/src/simcore_service_catalog/core/background_tasks.py +++ b/services/catalog/src/simcore_service_catalog/core/background_tasks.py @@ -89,7 +89,7 @@ def _by_version(t: tuple[ServiceKey, ServiceVersion]) -> Version: ) # set the service in the DB - _is_legacy_dynamic_service: bool = ( + _is_classic_dynamic_service: bool = ( service_metadata.inputs_path is None and service_metadata.outputs_path is None and service_metadata.state_paths is None @@ -98,7 +98,7 @@ def _by_version(t: tuple[ServiceKey, ServiceVersion]) -> Version: ServiceMetaDataDBCreate( **service_metadata.model_dump(exclude_unset=True, exclude={""}), owner=owner_gid, - is_legacy_dynamic_service=_is_legacy_dynamic_service, + _is_classic_dynamic_service=_is_classic_dynamic_service, ), service_access_rights, ) diff --git a/services/catalog/src/simcore_service_catalog/models/services_db.py b/services/catalog/src/simcore_service_catalog/models/services_db.py index bc7d2a3c1ecd..f01688ad116e 100644 --- a/services/catalog/src/simcore_service_catalog/models/services_db.py +++ b/services/catalog/src/simcore_service_catalog/models/services_db.py @@ -118,7 +118,7 @@ class ServiceMetaDataDBCreate(BaseModel): # lifecycle deprecated: datetime | None = None - is_legacy_dynamic_service: bool = False + _is_classic_dynamic_service: bool = False @staticmethod def _update_json_schema_extra(schema: JsonDict) -> None: From 50a72a0df6b3df9b6315d4395551794a4593a593 Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Wed, 16 Apr 2025 14:11:32 +0200 Subject: [PATCH 7/9] Fix remove comment --- .../director/src/simcore_service_director/api/rest/_services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/director/src/simcore_service_director/api/rest/_services.py b/services/director/src/simcore_service_director/api/rest/_services.py index 3d04a04ebd2c..b5794e24af99 100644 --- a/services/director/src/simcore_service_director/api/rest/_services.py +++ b/services/director/src/simcore_service_director/api/rest/_services.py @@ -96,7 +96,7 @@ async def list_service_labels( ) from err -@router.get("/services/{service_key:path}/{service_version}") # <-- MD +@router.get("/services/{service_key:path}/{service_version}") async def get_service( the_app: Annotated[FastAPI, Depends(get_app)], service_key: ServiceKey, From 30e2cd9ccec1bedbf8183ba52195bb288af4d9a8 Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Wed, 16 Apr 2025 15:23:45 +0200 Subject: [PATCH 8/9] Bugfixes --- .../src/models_library/services_metadata_published.py | 6 +++++- .../src/simcore_service_director/registry_proxy.py | 7 +++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/models-library/src/models_library/services_metadata_published.py b/packages/models-library/src/models_library/services_metadata_published.py index 5a3c09d95d94..ebbc4a30a262 100644 --- a/packages/models-library/src/models_library/services_metadata_published.py +++ b/packages/models-library/src/models_library/services_metadata_published.py @@ -149,6 +149,10 @@ class ServiceMetaDataPublished(ServiceKeyVersion, ServiceBaseDisplay): alias="progress_regexp", description="regexp pattern for detecting computational service's progress", ) + strip_path: str | None = Field( + None, + description="??? Mystery Pokemon Missingo", + ) inputs_path: str | None = Field( None, description="if this is present, the service is a modern style dv2 dynamic service", @@ -157,7 +161,7 @@ class ServiceMetaDataPublished(ServiceKeyVersion, ServiceBaseDisplay): None, description="if this is present, the service is a modern style dv2 dynamic service", ) - state_paths: str | None = Field( + state_paths: list[str] = Field( None, description="if this is present, the service is a modern style dv2 dynamic service", ) diff --git a/services/director/src/simcore_service_director/registry_proxy.py b/services/director/src/simcore_service_director/registry_proxy.py index 01f651d7eb53..69e1a0a57dc1 100644 --- a/services/director/src/simcore_service_director/registry_proxy.py +++ b/services/director/src/simcore_service_director/registry_proxy.py @@ -417,11 +417,14 @@ async def get_image_details( try: label_data = json.loads(labels[key]) for label_key in label_data: - if isinstance( - label_key, dict + if isinstance(label_key, dict) or isinstance( + label_data, list ): # Dicts from "simcore.service." docker image labels are omitted. continue image_details[label_key] = label_data[label_key] + except json.JSONDecodeError: + label_data = [] # Set to empty list if the value is not JSON + pass except json.decoder.JSONDecodeError: logging.exception( "Error while decoding json formatted data from %s:%s", From 97b267a9d3e056b365a8dc9df661a1559ed0a3dd Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Wed, 16 Apr 2025 15:26:54 +0200 Subject: [PATCH 9/9] Bugfixes --- .../director/src/simcore_service_director/registry_proxy.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/services/director/src/simcore_service_director/registry_proxy.py b/services/director/src/simcore_service_director/registry_proxy.py index 69e1a0a57dc1..01bf7d681c58 100644 --- a/services/director/src/simcore_service_director/registry_proxy.py +++ b/services/director/src/simcore_service_director/registry_proxy.py @@ -417,8 +417,10 @@ async def get_image_details( try: label_data = json.loads(labels[key]) for label_key in label_data: - if isinstance(label_key, dict) or isinstance( - label_data, list + if ( + isinstance(label_key, dict) + or isinstance(label_data, list) + or isinstance(label_data, str) ): # Dicts from "simcore.service." docker image labels are omitted. continue image_details[label_key] = label_data[label_key]