diff --git a/.env-devel b/.env-devel index e2f8baef3ca..2505f3cd3f0 100644 --- a/.env-devel +++ b/.env-devel @@ -133,6 +133,10 @@ DYNAMIC_SCHEDULER_UI_STORAGE_SECRET=adminadmin FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "unknown@osparc.io", "affiliation": "unknown"}}' + +ITIS_VIP_API_URL=https://itis.swiss/PD_DirectDownload/getDownloadableItems/{category} +ITIS_VIP_CATEGORIES='{"HumanWholeBody": "Humans", "HumanBodyRegion": "Humans (Region)", "AnimalWholeBody": "Animal"}' + # Can use 'docker run -it itisfoundation/invitations:latest simcore-service-invitations generate-dotenv --auto-password' INVITATIONS_DEFAULT_PRODUCT=osparc INVITATIONS_HOST=invitations diff --git a/packages/models-library/src/models_library/licenses.py b/packages/models-library/src/models_library/licenses.py new file mode 100644 index 00000000000..a59daea4733 --- /dev/null +++ b/packages/models-library/src/models_library/licenses.py @@ -0,0 +1,51 @@ +from datetime import datetime +from typing import TypeAlias +from uuid import UUID + +from pydantic import BaseModel, ConfigDict + +from .licensed_items import LicensedResourceType +from .products import ProductName +from .resource_tracker import PricingPlanId + +LicenseID: TypeAlias = UUID + + +# +# DB +# + + +class LicenseDB(BaseModel): + license_id: LicenseID + display_name: str + licensed_resource_type: LicensedResourceType + pricing_plan_id: PricingPlanId + product_name: ProductName + + # states + created: datetime + modified: datetime + + model_config = ConfigDict(from_attributes=True) + + +class LicenseUpdateDB(BaseModel): + display_name: str | None = None + pricing_plan_id: PricingPlanId | None = None + + +# +# License Domain +# + + +class License(BaseModel): + license_id: LicenseID + display_name: str + licensed_resource_type: LicensedResourceType + resources: list[dict] + pricing_plan_id: PricingPlanId + product_name: ProductName + + model_config = ConfigDict(from_attributes=True) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/2215301c2496_add_licenses_table.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/2215301c2496_add_licenses_table.py new file mode 100644 index 00000000000..5c0d8443614 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/2215301c2496_add_licenses_table.py @@ -0,0 +1,235 @@ +"""add licenses table + +Revision ID: 2215301c2496 +Revises: e71ea59858f4 +Create Date: 2025-02-04 15:26:27.325429+00:00 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "2215301c2496" +down_revision = "e71ea59858f4" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "licensed_resources", + sa.Column( + "licensed_item_id", + postgresql.UUID(as_uuid=True), + server_default=sa.text("gen_random_uuid()"), + nullable=False, + ), + sa.Column("display_name", sa.String(), nullable=False), + sa.Column("licensed_resource_name", sa.String(), nullable=False), + sa.Column( + "licensed_resource_type", + sa.Enum("VIP_MODEL", name="licensedresourcetype"), + nullable=False, + ), + sa.Column( + "licensed_resource_data", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + ), + sa.Column("pricing_plan_id", sa.BigInteger(), nullable=True), + sa.Column("product_name", sa.String(), nullable=True), + sa.Column( + "created", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "trashed", + sa.DateTime(timezone=True), + nullable=True, + comment="The date and time when the licensed_item was marked as trashed. Null if the licensed_item has not been trashed [default].", + ), + sa.ForeignKeyConstraint( + ["pricing_plan_id"], + ["resource_tracker_pricing_plans.pricing_plan_id"], + name="fk_resource_tracker_license_packages_pricing_plan_id", + onupdate="CASCADE", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["product_name"], + ["products.name"], + name="fk_resource_tracker_license_packages_product_name", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("licensed_item_id"), + # sa.UniqueConstraint( + # "licensed_resource_name", + # "licensed_resource_type", + # name="uq_licensed_resource_name_type", + # ), + ) + op.create_table( + "licenses", + sa.Column( + "license_id", + postgresql.UUID(as_uuid=True), + server_default=sa.text("gen_random_uuid()"), + nullable=False, + ), + sa.Column("display_name", sa.String(), nullable=False), + sa.Column( + "licensed_resource_type", + sa.Enum("VIP_MODEL", name="licensedresourcetype"), + nullable=False, + ), + sa.Column("pricing_plan_id", sa.BigInteger(), nullable=False), + sa.Column("product_name", sa.String(), nullable=False), + sa.Column( + "created", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["pricing_plan_id"], + ["resource_tracker_pricing_plans.pricing_plan_id"], + name="fk_resource_tracker_license_packages_pricing_plan_id", + onupdate="CASCADE", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["product_name"], + ["products.name"], + name="fk_resource_tracker_license_packages_product_name", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("license_id"), + ) + op.create_table( + "license_to_resource", + sa.Column("license_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("licensed_item_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column( + "created", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["license_id"], + ["licenses.license_id"], + name="fk_license_to_resource_license_id", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["licensed_item_id"], + ["licensed_resources.licensed_item_id"], + name="fk_license_to_resource_licensed_item_id", + onupdate="CASCADE", + ondelete="CASCADE", + ), + ) + op.drop_table("licensed_items") + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "licensed_items", + sa.Column( + "licensed_item_id", + postgresql.UUID(), + server_default=sa.text("gen_random_uuid()"), + autoincrement=False, + nullable=False, + ), + sa.Column( + "licensed_resource_name", sa.VARCHAR(), autoincrement=False, nullable=False + ), + sa.Column( + "licensed_resource_type", + postgresql.ENUM("VIP_MODEL", name="licensedresourcetype"), + autoincrement=False, + nullable=False, + ), + sa.Column("pricing_plan_id", sa.BIGINT(), autoincrement=False, nullable=True), + sa.Column("product_name", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column( + "created", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column( + "modified", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column( + "licensed_resource_data", + postgresql.JSONB(astext_type=sa.Text()), + autoincrement=False, + nullable=True, + ), + sa.Column( + "trashed", + postgresql.TIMESTAMP(timezone=True), + autoincrement=False, + nullable=True, + comment="The date and time when the licensed_item was marked as trashed. Null if the licensed_item has not been trashed [default].", + ), + sa.Column("display_name", sa.VARCHAR(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint( + ["pricing_plan_id"], + ["resource_tracker_pricing_plans.pricing_plan_id"], + name="fk_resource_tracker_license_packages_pricing_plan_id", + onupdate="CASCADE", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["product_name"], + ["products.name"], + name="fk_resource_tracker_license_packages_product_name", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("licensed_item_id", name="licensed_items_pkey"), + sa.UniqueConstraint( + "licensed_resource_name", + "licensed_resource_type", + name="uq_licensed_resource_name_type", + ), + ) + op.drop_table("license_to_resource") + op.drop_table("licenses") + op.drop_table("licensed_resources") + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/e71ea59858f4_add_uniqu_constraint_in_licensed_items.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/e71ea59858f4_add_uniqu_constraint_in_licensed_items.py new file mode 100644 index 00000000000..3af7ff911f8 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/e71ea59858f4_add_uniqu_constraint_in_licensed_items.py @@ -0,0 +1,32 @@ +"""add uniqu constraint in licensed_items + +Revision ID: e71ea59858f4 +Revises: 7d1c6425a51d" +Create Date: 2025-01-30 18:42:15.192968+00:00 + +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = "e71ea59858f4" +down_revision = "7d1c6425a51d" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_unique_constraint( + "uq_licensed_resource_name_type", + "licensed_items", + ["licensed_resource_name", "licensed_resource_type"], + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + "uq_licensed_resource_name_type", "licensed_items", type_="unique" + ) + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/license_to_resource.py b/packages/postgres-database/src/simcore_postgres_database/models/license_to_resource.py new file mode 100644 index 00000000000..09813c5816c --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/license_to_resource.py @@ -0,0 +1,34 @@ +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from ._common import RefActions, column_created_datetime, column_modified_datetime +from .base import metadata + +license_to_resource = sa.Table( + "license_to_resource", + metadata, + sa.Column( + "license_id", + postgresql.UUID(as_uuid=True), + sa.ForeignKey( + "licenses.license_id", + name="fk_license_to_resource_license_id", + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + ), + nullable=False, + ), + sa.Column( + "licensed_item_id", # <-- This will be renamed to "licensed_resource_id" + postgresql.UUID(as_uuid=True), + sa.ForeignKey( + "licensed_resources.licensed_item_id", # <-- This will be renamed to "licensed_resource_id" + name="fk_license_to_resource_licensed_item_id", + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + ), + nullable=False, + ), + column_created_datetime(timezone=True), + column_modified_datetime(timezone=True), +) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py b/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py index 5286a7eee5a..149abd2e681 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py @@ -1,8 +1,3 @@ -""" resource_tracker_service_runs table -""" - -import enum - import sqlalchemy as sa from sqlalchemy.dialects import postgresql @@ -13,17 +8,13 @@ column_trashed_datetime, ) from .base import metadata +from .licenses import LicensedResourceType - -class LicensedResourceType(str, enum.Enum): - VIP_MODEL = "VIP_MODEL" - - -licensed_items = sa.Table( - "licensed_items", +licensed_resources = sa.Table( + "licensed_resources", # <-- This will be renamed to "licensed_resources" metadata, sa.Column( - "licensed_item_id", + "licensed_item_id", # <-- This will be renamed to "licensed_resource_id" postgresql.UUID(as_uuid=True), nullable=False, primary_key=True, @@ -79,4 +70,9 @@ class LicensedResourceType(str, enum.Enum): column_created_datetime(timezone=True), column_modified_datetime(timezone=True), column_trashed_datetime("licensed_item"), + sa.UniqueConstraint( + "licensed_resource_name", + "licensed_resource_type", + name="uq_licensed_resource_name_type", + ), ) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/licenses.py b/packages/postgres-database/src/simcore_postgres_database/models/licenses.py new file mode 100644 index 00000000000..14af962dc09 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/licenses.py @@ -0,0 +1,60 @@ +import enum + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from ._common import RefActions, column_created_datetime, column_modified_datetime +from .base import metadata + + +class LicensedResourceType(str, enum.Enum): + VIP_MODEL = "VIP_MODEL" + + +licenses = sa.Table( + "licenses", + metadata, + sa.Column( + "license_id", + postgresql.UUID(as_uuid=True), + nullable=False, + primary_key=True, + server_default=sa.text("gen_random_uuid()"), + ), + sa.Column( + "display_name", + sa.String, + nullable=False, + doc="Display name for front-end", + ), + sa.Column( + "licensed_resource_type", + sa.Enum(LicensedResourceType), + nullable=False, + doc="Resource type, ex. VIP_MODEL", + ), + sa.Column( + "pricing_plan_id", + sa.BigInteger, + sa.ForeignKey( + "resource_tracker_pricing_plans.pricing_plan_id", + name="fk_resource_tracker_license_packages_pricing_plan_id", + onupdate=RefActions.CASCADE, + ondelete=RefActions.RESTRICT, + ), + nullable=False, + ), + sa.Column( + "product_name", + sa.String, + sa.ForeignKey( + "products.name", + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_resource_tracker_license_packages_product_name", + ), + nullable=False, + ), + column_created_datetime(timezone=True), + column_modified_datetime(timezone=True), +) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py index ce7c5685e94..9e7ac7da9ec 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py @@ -437,3 +437,35 @@ def random_service_access_rights( data.update(**overrides) return data + + +def random_itis_vip_available_download_item( + identifier: int, + fake: Faker = DEFAULT_FAKER, + features_functionality: str = "Posable", + **overrides, +): + features_str = ( + "{" + f"name: {fake.name()} Right Hand," # w/o spaces + f" version: V{fake.pyint()}.0, " # w/ x2 spaces + f"sex: Male, age: 8 years," # w/o spaces + f"date: {fake.date()}, " # w/ x1 spaces + f"ethnicity: Caucasian, functionality: {features_functionality} " + "}" + ) + + data = { + "ID": identifier, + "Description": fake.sentence(), + "Thumbnail": fake.image_url(), + "Features": features_str, + "DOI": fake.bothify(text="10.####/ViP#####-##-#"), + "LicenseKey": fake.bothify(text="MODEL_????_V#"), + "LicenseVersion": fake.bothify(text="V#.0"), + "Protection": fake.random_element(elements=["Code", "PayPal"]), + "AvailableFromURL": fake.random_element(elements=[None, fake.url()]), + } + + data.update(**overrides) + return data diff --git a/packages/service-library/src/servicelib/background_task.py b/packages/service-library/src/servicelib/background_task.py index 793d05b1f9b..feeb06ff475 100644 --- a/packages/service-library/src/servicelib/background_task.py +++ b/packages/service-library/src/servicelib/background_task.py @@ -60,6 +60,11 @@ def periodic( def _decorator( func: Callable[P, Coroutine[Any, Any, None]], ) -> Callable[P, Coroutine[Any, Any, None]]: + class _InternalTryAgain(TryAgain): + # Local exception to prevent reacting to similarTryAgain exceptions raised by the wrapped func + # e.g. when this decorators is used twice on the same function + ... + nap = ( asyncio.sleep if early_wake_up_event is None @@ -71,7 +76,7 @@ def _decorator( wait=wait_fixed(interval.total_seconds()), reraise=True, retry=( - retry_if_exception_type(TryAgain) + retry_if_exception_type(_InternalTryAgain) if raise_on_error else retry_if_exception_type() ), @@ -80,7 +85,7 @@ def _decorator( @functools.wraps(func) async def _wrapper(*args: P.args, **kwargs: P.kwargs) -> None: await func(*args, **kwargs) - raise TryAgain + raise _InternalTryAgain return _wrapper diff --git a/services/api-server/src/simcore_service_api_server/main.py b/services/api-server/src/simcore_service_api_server/main.py index 493874ee6eb..8b636ac4315 100644 --- a/services/api-server/src/simcore_service_api_server/main.py +++ b/services/api-server/src/simcore_service_api_server/main.py @@ -1,8 +1,7 @@ """Main application to be deployed in for example uvicorn. """ from fastapi import FastAPI - -from .core.application import init_app +from simcore_service_api_server.core.application import init_app # SINGLETON FastAPI app the_app: FastAPI = init_app() diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 555f47c4c9d..6a7e8bc966f 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -705,6 +705,8 @@ services: INVITATIONS_USERNAME: ${INVITATIONS_USERNAME} WEBSERVER_LICENSES: ${WEBSERVER_LICENSES} + ITIS_VIP_API_URL: ${ITIS_VIP_API_URL} + ITIS_VIP_CATEGORIES: ${ITIS_VIP_CATEGORIES} WEBSERVER_LOGIN: ${WEBSERVER_LOGIN} LOGIN_ACCOUNT_DELETION_RETENTION_DAYS: ${LOGIN_ACCOUNT_DELETION_RETENTION_DAYS} diff --git a/services/resource-usage-tracker/docker/boot.sh b/services/resource-usage-tracker/docker/boot.sh index 28854b7b2b5..2b3c1bebc5f 100755 --- a/services/resource-usage-tracker/docker/boot.sh +++ b/services/resource-usage-tracker/docker/boot.sh @@ -48,7 +48,7 @@ if [ "${SC_BOOT_MODE}" = "debug" ]; then exec sh -c " cd services/resource-usage-tracker/src/simcore_service_resource_usage_tracker && \ - python -m debugpy --listen 0.0.0.0:${RESOURCE_USAGE_TRACKER_REMOTE_DEBUGGING_PORT} -m uvicorn web_main:the_app \ + python -m debugpy --listen 0.0.0.0:${RESOURCE_USAGE_TRACKER_REMOTE_DEBUGGING_PORT} -m uvicorn main:the_app \ --host 0.0.0.0 \ --reload \ $reload_dir_packages diff --git a/services/web/server/requirements/_base.in b/services/web/server/requirements/_base.in index caf883fc166..f8e72f62ca6 100644 --- a/services/web/server/requirements/_base.in +++ b/services/web/server/requirements/_base.in @@ -32,8 +32,10 @@ aiosmtplib # email asyncpg # db captcha cryptography # security +deepdiff[optimize] # diffs data-structures faker # Only used in dev-mode for proof-of-concepts gunicorn[setproctitle] +httpx jinja_app_loader # email json2html jsondiff diff --git a/services/web/server/requirements/_base.txt b/services/web/server/requirements/_base.txt index f35f5ab6212..562b0eab736 100644 --- a/services/web/server/requirements/_base.txt +++ b/services/web/server/requirements/_base.txt @@ -88,6 +88,7 @@ anyio==4.3.0 # via # fast-depends # faststream + # httpx appdirs==1.4.4 # via pint arrow==1.2.3 @@ -145,6 +146,8 @@ certifi==2023.7.22 # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../../requirements/constraints.txt + # httpcore + # httpx # requests cffi==1.17.1 # via cryptography @@ -186,6 +189,8 @@ cryptography==41.0.7 # -c requirements/../../../../requirements/constraints.txt # -r requirements/_base.in # aiohttp-session +deepdiff==8.1.1 + # via -r requirements/_base.in deprecated==1.2.14 # via # opentelemetry-api @@ -225,10 +230,46 @@ grpcio==1.66.0 # via opentelemetry-exporter-otlp-proto-grpc gunicorn==23.0.0 # via -r requirements/_base.in +h11==0.14.0 + # via httpcore +httpcore==1.0.7 + # via httpx +httpx==0.28.1 + # via + # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../requirements/constraints.txt + # -r requirements/_base.in idna==3.3 # via # anyio # email-validator + # httpx # requests # yarl importlib-metadata==8.0.0 @@ -416,6 +457,8 @@ opentelemetry-util-http==0.48b0 # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-aiohttp-server # opentelemetry-instrumentation-requests +orderly-set==5.2.3 + # via deepdiff orjson==3.10.0 # via # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -465,6 +508,7 @@ orjson==3.10.0 # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/_base.in + # deepdiff packaging==24.1 # via # -r requirements/../../../../packages/simcore-sdk/requirements/_base.in diff --git a/services/web/server/requirements/_test.in b/services/web/server/requirements/_test.in index 22ed8e423bc..0644e604010 100644 --- a/services/web/server/requirements/_test.in +++ b/services/web/server/requirements/_test.in @@ -38,6 +38,7 @@ pytest-sugar pytest-xdist python-dotenv redis +respx sqlalchemy[mypy] # adds Mypy / Pep-484 Support for ORM Mappings SEE https://docs.sqlalchemy.org/en/20/orm/extensions/mypy.html tenacity types-aiofiles diff --git a/services/web/server/requirements/_test.txt b/services/web/server/requirements/_test.txt index 6beb543d3f8..08cf7390c84 100644 --- a/services/web/server/requirements/_test.txt +++ b/services/web/server/requirements/_test.txt @@ -14,6 +14,10 @@ alembic==1.8.1 # via # -c requirements/_base.txt # -r requirements/_test.in +anyio==4.3.0 + # via + # -c requirements/_base.txt + # httpx async-timeout==4.0.3 # via # -c requirements/_base.txt @@ -35,6 +39,8 @@ certifi==2023.7.22 # via # -c requirements/../../../../requirements/constraints.txt # -c requirements/_base.txt + # httpcore + # httpx # requests charset-normalizer==2.0.12 # via @@ -68,6 +74,19 @@ greenlet==2.0.2 # via # -c requirements/_base.txt # sqlalchemy +h11==0.14.0 + # via + # -c requirements/_base.txt + # httpcore +httpcore==1.0.7 + # via + # -c requirements/_base.txt + # httpx +httpx==0.28.1 + # via + # -c requirements/../../../../requirements/constraints.txt + # -c requirements/_base.txt + # respx hypothesis==6.91.0 # via -r requirements/_test.in icdiff==2.0.7 @@ -75,6 +94,8 @@ icdiff==2.0.7 idna==3.3 # via # -c requirements/_base.txt + # anyio + # httpx # requests # yarl iniconfig==2.0.0 @@ -185,6 +206,8 @@ requests==2.32.2 # via # -c requirements/_base.txt # docker +respx==0.22.0 + # via -r requirements/_test.in setuptools==69.1.1 # via # -c requirements/_base.txt @@ -195,6 +218,10 @@ six==1.16.0 # -c requirements/_base.txt # jsonschema # python-dateutil +sniffio==1.3.1 + # via + # -c requirements/_base.txt + # anyio sortedcontainers==2.4.0 # via hypothesis sqlalchemy==1.4.47 diff --git a/services/web/server/src/simcore_service_webserver/licenses/_common/models.py b/services/web/server/src/simcore_service_webserver/licenses/_common/models.py index 887a6db6f59..0207cd6b99e 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_common/models.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_common/models.py @@ -8,6 +8,7 @@ LicensedItemID, LicensedResourceType, ) +from models_library.licenses import License from models_library.resource_tracker import PricingPlanId, PricingUnitId from models_library.resource_tracker_licensed_items_purchases import ( LicensedItemPurchaseID, @@ -67,6 +68,11 @@ class LicensedItemPage(NamedTuple): total: PositiveInt +class LicensePage(NamedTuple): + items: list[License] + total: PositiveInt + + class LicensedItemsRequestContext(RequestParameters): user_id: UserID = Field(..., alias=RQT_USERID_KEY) # type: ignore[literal-required] product_name: str = Field(..., alias=RQ_PRODUCT_KEY) # type: ignore[literal-required] diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py new file mode 100644 index 00000000000..8e0d83378b6 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py @@ -0,0 +1,101 @@ +import re +from typing import Annotated, Any, Literal, NamedTuple, NotRequired, Self, TypeAlias + +from models_library.basic_types import IDStr +from pydantic import ( + BaseModel, + BeforeValidator, + Field, + HttpUrl, + StringConstraints, + TypeAdapter, +) +from typing_extensions import TypedDict + +_max_str_adapter = TypeAdapter( + Annotated[str, StringConstraints(strip_whitespace=True, max_length=1_000)] +) + + +def _feature_descriptor_to_dict(descriptor: str) -> dict[str, Any]: + # NOTE: this is manually added in the server side so be more robust to errors + descriptor = _max_str_adapter.validate_python(descriptor.strip("{}")) + pattern = r"(\w{1,100}): ([^,]{1,100})" + matches = re.findall(pattern, descriptor) + return dict(matches) + + +# +# ITIS-VIP API Schema +# + + +class FeaturesDict(TypedDict): + name: str + version: str + sex: NotRequired[str] + age: NotRequired[str] + weight: NotRequired[str] + height: NotRequired[str] + date: NotRequired[str] + ethnicity: NotRequired[str] + functionality: NotRequired[str] + + +class ItisVipData(BaseModel): + id: Annotated[int, Field(alias="ID")] + description: Annotated[str, Field(alias="Description")] + thumbnail: Annotated[str, Field(alias="Thumbnail")] + features: Annotated[ + dict[str, Any], # NOTE: for the moment FeaturesDict is NOT used + BeforeValidator(_feature_descriptor_to_dict), + Field(alias="Features"), + ] + doi: Annotated[str | None, Field(alias="DOI")] + license_key: Annotated[str | None, Field(alias="LicenseKey")] + license_version: Annotated[str | None, Field(alias="LicenseVersion")] + protection: Annotated[Literal["Code", "PayPal"], Field(alias="Protection")] + available_from_url: Annotated[HttpUrl | None, Field(alias="AvailableFromURL")] + + +class ItisVipApiResponse(BaseModel): + msg: int | None = None # still not used + available_downloads: Annotated[list[ItisVipData], Field(alias="availableDownloads")] + + +# +# RESOURCE +# + + +class ItisVipResourceData(BaseModel): + category_id: IDStr + category_display: str + source: Annotated[ + dict[str, Any], Field(description="Original published data in the api") + ] + + @classmethod + def create( + cls, category_id: IDStr, category_display: str, source: ItisVipData + ) -> Self: + return cls( + category_id=category_id, + category_display=category_display, + # NOTE: ensures source data is the same as the one in the original VIP API model + source=source.model_dump(mode="json", by_alias=True), + ) + + +# +# INTERNAL +# + +CategoryID: TypeAlias = IDStr +CategoryDisplay: TypeAlias = str + + +class CategoryTuple(NamedTuple): + url: HttpUrl + id: CategoryID + display: CategoryDisplay diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py new file mode 100644 index 00000000000..2f3b9deab1e --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py @@ -0,0 +1,32 @@ +import httpx +from pydantic import HttpUrl +from tenacity import ( + retry, + retry_if_exception_cause_type, + stop_after_attempt, + wait_exponential, +) + +from ._itis_vip_models import ItisVipApiResponse, ItisVipData + + +@retry( + wait=wait_exponential(multiplier=1, min=4, max=10), + stop=stop_after_attempt(5), + retry=retry_if_exception_cause_type(httpx.RequestError), +) +async def get_category_items( + client: httpx.AsyncClient, url: HttpUrl +) -> list[ItisVipData]: + """ + + Raises: + httpx.HTTPStatusError + pydantic.ValidationError + """ + response = await client.post(f"{url}") + response.raise_for_status() + + validated_data = ItisVipApiResponse.model_validate(response.json()) + + return validated_data.available_downloads diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_settings.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_settings.py new file mode 100644 index 00000000000..a961472afbc --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_settings.py @@ -0,0 +1,34 @@ +from typing import Annotated + +from pydantic import AfterValidator, HttpUrl +from settings_library.base import BaseCustomSettings + +from ._itis_vip_models import CategoryDisplay, CategoryID, CategoryTuple + + +def _validate_url_contains_category(url: str) -> str: + if "{category}" not in url: + msg = "URL must contain '{category}'" + raise ValueError(msg) + return url + + +class ItisVipSettings(BaseCustomSettings): + ITIS_VIP_API_URL: Annotated[str, AfterValidator(_validate_url_contains_category)] + ITIS_VIP_CATEGORIES: dict[CategoryID, CategoryDisplay] + + def get_urls(self) -> list[HttpUrl]: + return [ + HttpUrl(self.ITIS_VIP_API_URL.format(category=category)) + for category in self.ITIS_VIP_CATEGORIES + ] + + def to_categories(self) -> list[CategoryTuple]: + return [ + CategoryTuple( + url=HttpUrl(self.ITIS_VIP_API_URL.format(category=category_id)), + id=category_id, + display=category_display, + ) + for category_id, category_display in self.ITIS_VIP_CATEGORIES.items() + ] diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py new file mode 100644 index 00000000000..f5214b53d8a --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py @@ -0,0 +1,134 @@ +import asyncio +import logging +from datetime import timedelta + +from aiohttp import web +from httpx import AsyncClient +from models_library.licensed_items import LicensedResourceType +from pydantic import ValidationError +from servicelib.async_utils import cancel_wait_task +from servicelib.background_task_utils import exclusive_periodic +from servicelib.logging_utils import log_catch, log_context +from simcore_service_webserver.licenses import ( + _itis_vip_service, + _licensed_items_service, +) +from simcore_service_webserver.licenses._itis_vip_settings import ItisVipSettings + +from ..redis import get_redis_lock_manager_client_sdk, setup_redis +from ._itis_vip_models import CategoryTuple, ItisVipData, ItisVipResourceData +from ._licensed_items_service import RegistrationState + +_logger = logging.getLogger(__name__) + + +async def sync_resources_with_licensed_items( + app: web.Application, categories: list[CategoryTuple] +): + async with AsyncClient() as http_client: + for category_url, category_id, category_display in categories: + assert f"{category_url}".endswith(category_id) # nosec + + # FETCH & VALIDATION + with log_context( + _logger, logging.INFO, "Fetching %s and validating", category_url + ), log_catch(_logger, reraise=True): + vip_data_items: list[ + ItisVipData + # TODO: handle errors to avoid disrupting other categories? + ] = await _itis_vip_service.get_category_items( + http_client, category_url + ) + + # REGISTRATION + for vip_data in vip_data_items: + + licensed_resource_name = f"{category_id}/{vip_data.id}" + + with log_context( + _logger, logging.INFO, "Registering %s", licensed_resource_name + ), log_catch(_logger, reraise=False): + # TODO: handle error to avoid disrupting other vip_data_items? + result = await _licensed_items_service.register_resource_as_licensed_item( + app, + licensed_item_display_name=f"{vip_data.features.get('name', 'UNNAMED!!')} " + f"{vip_data.features.get('version', 'UNVERSIONED!!')}", + # RESOURCE unique identifiers + licensed_resource_name=licensed_resource_name, + licensed_resource_type=LicensedResourceType.VIP_MODEL, + # RESOURCE extended data + licensed_resource_data=ItisVipResourceData.create( + category_id=category_id, + category_display=category_display, + source=vip_data, + ), + ) + + if result.state == RegistrationState.ALREADY_REGISTERED: + # NOTE: not really interesting + _logger.debug(result.message) + + elif result.state == RegistrationState.DIFFERENT_RESOURCE: + # NOTE: notify since need human decision + _logger.warning(result.message) + + else: + assert ( + result.state == RegistrationState.NEWLY_REGISTERED + ) # nosec + # NOTE: inform since needs curation + _logger.info( + "%s . New licensed_item_id=%s pending for activation.", + result.message, + result.registered.licensed_item_id, + ) + + # TODO: check delete!? + + +_BACKGROUND_TASK_NAME = f"{__name__}.itis_vip_syncer_cleanup_ctx._periodic_sync" + + +def setup_itis_vip_syncer(app: web.Application): + categories = [] + + try: + settings = ItisVipSettings.create_from_envs() + categories = settings.to_categories() + + except ValidationError as err: + _logger.warning("IT'IS VIP syncer disabled. Skipping. %s", err) + return + + if categories: + + async def _cleanup_ctx(app_: web.Application): + with ( + log_context( + _logger, + logging.INFO, + f"IT'IS VIP syncing {len(categories)} categories", + ), + log_catch(_logger, reraise=False), + ): + + @exclusive_periodic( + get_redis_lock_manager_client_sdk(app_), + task_interval=timedelta(minutes=1), + retry_after=timedelta(minutes=2), + ) + async def _periodic_sync() -> None: + await sync_resources_with_licensed_items( + app_, categories=categories + ) + + background_task = asyncio.create_task( + _periodic_sync(), name=_BACKGROUND_TASK_NAME + ) + + yield + + await cancel_wait_task(background_task) + + setup_redis(app) + app.cleanup_ctx.append(_cleanup_ctx) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py index b63c9e8c58f..a6c107778b7 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py @@ -5,7 +5,7 @@ """ import logging -from typing import Literal, cast +from typing import Any, Literal, cast from aiohttp import web from models_library.licensed_items import ( @@ -18,13 +18,14 @@ from models_library.resource_tracker import PricingPlanId from models_library.rest_ordering import OrderBy, OrderDirection from pydantic import NonNegativeInt -from simcore_postgres_database.models.licensed_items import licensed_items +from simcore_postgres_database.models.licensed_items import licensed_resources from simcore_postgres_database.utils_repos import ( get_columns_from_db_model, pass_or_acquire_connection, transaction_context, ) from sqlalchemy import asc, desc, func +from sqlalchemy.dialects import postgresql from sqlalchemy.ext.asyncio import AsyncConnection from sqlalchemy.sql import select @@ -34,7 +35,31 @@ _logger = logging.getLogger(__name__) -_SELECTION_ARGS = get_columns_from_db_model(licensed_items, LicensedItemDB) +_SELECTION_ARGS = get_columns_from_db_model(licensed_resources, LicensedItemDB) + + +def _create_insert_query( + display_name: str, + licensed_resource_name: str, + licensed_resource_type: LicensedResourceType, + licensed_resource_data: dict[str, Any] | None, + product_name: ProductName | None, + pricing_plan_id: PricingPlanId | None, +) -> postgresql.Insert: + return ( + postgresql.insert(licensed_resources) + .values( + licensed_resource_name=licensed_resource_name, + licensed_resource_type=licensed_resource_type, + licensed_resource_data=licensed_resource_data, + display_name=display_name, + pricing_plan_id=pricing_plan_id, + product_name=product_name, + created=func.now(), + modified=func.now(), + ) + .returning(*_SELECTION_ARGS) + ) async def create( @@ -44,29 +69,66 @@ async def create( display_name: str, licensed_resource_name: str, licensed_resource_type: LicensedResourceType, - licensed_resource_data: dict | None, - product_name: ProductName | None, - pricing_plan_id: PricingPlanId | None, + licensed_resource_data: dict[str, Any] | None = None, + product_name: ProductName | None = None, + pricing_plan_id: PricingPlanId | None = None, ) -> LicensedItemDB: + + query = _create_insert_query( + display_name, + licensed_resource_name, + licensed_resource_type, + licensed_resource_data, + product_name, + pricing_plan_id, + ) async with transaction_context(get_asyncpg_engine(app), connection) as conn: - result = await conn.execute( - licensed_items.insert() - .values( - product_name=product_name, - display_name=display_name, - licensed_resource_name=licensed_resource_name, - licensed_resource_type=licensed_resource_type, - licensed_resource_data=licensed_resource_data, - pricing_plan_id=pricing_plan_id, - created=func.now(), - modified=func.now(), - ) - .returning(*_SELECTION_ARGS) - ) + result = await conn.execute(query) row = result.one() return LicensedItemDB.model_validate(row) +async def create_if_not_exists( + app: web.Application, + connection: AsyncConnection | None = None, + *, + display_name: str, + licensed_resource_name: str, + licensed_resource_type: LicensedResourceType, + licensed_resource_data: dict[str, Any] | None = None, + product_name: ProductName | None = None, + pricing_plan_id: PricingPlanId | None = None, +) -> LicensedItemDB: + + insert_or_none_query = _create_insert_query( + display_name, + licensed_resource_name, + licensed_resource_type, + licensed_resource_data, + product_name, + pricing_plan_id, + ).on_conflict_do_nothing() + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.execute(insert_or_none_query) + row = result.one_or_none() + + if row is None: + select_query = select(*_SELECTION_ARGS).where( + (licensed_resources.c.licensed_resource_name == licensed_resource_name) + & ( + licensed_resources.c.licensed_resource_type + == licensed_resource_type + ) + ) + + result = await conn.execute(select_query) + row = result.one() + + assert row is not None # nosec + return LicensedItemDB.model_validate(row) + + async def list_( app: web.Application, connection: AsyncConnection | None = None, @@ -82,25 +144,25 @@ async def list_( base_query = ( select(*_SELECTION_ARGS) - .select_from(licensed_items) - .where(licensed_items.c.product_name == product_name) + .select_from(licensed_resources) + .where(licensed_resources.c.product_name == product_name) ) # Apply trashed filter if trashed == "exclude": - base_query = base_query.where(licensed_items.c.trashed.is_(None)) + base_query = base_query.where(licensed_resources.c.trashed.is_(None)) elif trashed == "only": - base_query = base_query.where(licensed_items.c.trashed.is_not(None)) + base_query = base_query.where(licensed_resources.c.trashed.is_not(None)) if inactive == "only": base_query = base_query.where( - licensed_items.c.product_name.is_(None) - | licensed_items.c.licensed_item_id.is_(None) + licensed_resources.c.product_name.is_(None) + | licensed_resources.c.licensed_item_id.is_(None) ) elif inactive == "exclude": base_query = base_query.where( - licensed_items.c.product_name.is_not(None) - & licensed_items.c.licensed_item_id.is_not(None) + licensed_resources.c.product_name.is_not(None) + & licensed_resources.c.licensed_item_id.is_not(None) ) # Select total count from base_query @@ -109,10 +171,12 @@ async def list_( # Ordering and pagination if order_by.direction == OrderDirection.ASC: - list_query = base_query.order_by(asc(getattr(licensed_items.c, order_by.field))) + list_query = base_query.order_by( + asc(getattr(licensed_resources.c, order_by.field)) + ) else: list_query = base_query.order_by( - desc(getattr(licensed_items.c, order_by.field)) + desc(getattr(licensed_resources.c, order_by.field)) ) list_query = list_query.offset(offset).limit(limit) @@ -134,23 +198,47 @@ async def get( licensed_item_id: LicensedItemID, product_name: ProductName, ) -> LicensedItemDB: - base_query = ( + select_query = ( select(*_SELECTION_ARGS) - .select_from(licensed_items) + .select_from(licensed_resources) .where( - (licensed_items.c.licensed_item_id == licensed_item_id) - & (licensed_items.c.product_name == product_name) + (licensed_resources.c.licensed_item_id == licensed_item_id) + & (licensed_resources.c.product_name == product_name) ) ) async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: - result = await conn.stream(base_query) - row = await result.first() + result = await conn.execute(select_query) + row = result.one_or_none() if row is None: raise LicensedItemNotFoundError(licensed_item_id=licensed_item_id) return LicensedItemDB.model_validate(row) +async def get_by_resource_identifier( + app: web.Application, + connection: AsyncConnection | None = None, + *, + licensed_resource_name: str, + licensed_resource_type: LicensedResourceType, +) -> LicensedItemDB: + select_query = select(*_SELECTION_ARGS).where( + (licensed_resources.c.licensed_resource_name == licensed_resource_name) + & (licensed_resources.c.licensed_resource_type == licensed_resource_type) + ) + + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + result = await conn.execute(select_query) + row = result.one_or_none() + if row is None: + raise LicensedItemNotFoundError( + licensed_item_id="Unkown", + licensed_resource_name=licensed_resource_name, + licensed_resource_type=licensed_resource_type, + ) + return LicensedItemDB.model_validate(row) + + async def update( app: web.Application, connection: AsyncConnection | None = None, @@ -162,21 +250,21 @@ async def update( # NOTE: at least 'touch' if updated_values is empty _updates = { **updates.model_dump(exclude_unset=True), - licensed_items.c.modified.name: func.now(), + licensed_resources.c.modified.name: func.now(), } # trashing assert "trash" in dict(LicensedItemUpdateDB.model_fields) # nosec if trash := _updates.pop("trash", None): - _updates[licensed_items.c.trashed.name] = func.now() if trash else None + _updates[licensed_resources.c.trashed.name] = func.now() if trash else None async with transaction_context(get_asyncpg_engine(app), connection) as conn: result = await conn.execute( - licensed_items.update() + licensed_resources.update() .values(**_updates) .where( - (licensed_items.c.licensed_item_id == licensed_item_id) - & (licensed_items.c.product_name == product_name) + (licensed_resources.c.licensed_item_id == licensed_item_id) + & (licensed_resources.c.product_name == product_name) ) .returning(*_SELECTION_ARGS) ) @@ -195,8 +283,71 @@ async def delete( ) -> None: async with transaction_context(get_asyncpg_engine(app), connection) as conn: await conn.execute( - licensed_items.delete().where( - (licensed_items.c.licensed_item_id == licensed_item_id) - & (licensed_items.c.product_name == product_name) + licensed_resources.delete().where( + (licensed_resources.c.licensed_item_id == licensed_item_id) + & (licensed_resources.c.product_name == product_name) ) ) + + +#### DOMAIN MODEL + + +# async def list_licensed_resources_v2( +# app: web.Application, +# connection: AsyncConnection | None = None, +# *, +# product_name: ProductName, +# offset: NonNegativeInt, +# limit: NonNegativeInt, +# order_by: OrderBy, +# ) -> tuple[int, list[License]]: + +# base_query = ( +# select(*_SELECTION_ARGS) +# .select_from(licenses) +# .where(licenses.c.product_name == product_name) +# ) + +# # Select total count from base_query +# subquery = base_query.subquery() +# count_query = select(func.count()).select_from(subquery) + +# # Ordering and pagination +# if order_by.direction == OrderDirection.ASC: +# list_query = base_query.order_by(asc(getattr(licenses.c, order_by.field))) +# else: +# list_query = base_query.order_by(desc(getattr(licenses.c, order_by.field))) +# list_query = list_query.offset(offset).limit(limit) + +# async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: +# total_count = await conn.scalar(count_query) + +# result = await conn.stream(list_query) +# items: list[License] = [License.model_validate(row) async for row in result] + +# return cast(int, total_count), items + + +# async def get_licensed_resource_v2( +# app: web.Application, +# connection: AsyncConnection | None = None, +# *, +# license_id: LicenseID, +# product_name: ProductName, +# ) -> License: +# base_query = ( +# select(*_SELECTION_ARGS) +# .select_from(licenses) +# .where( +# (licenses.c.license_id == license_id) +# & (licenses.c.product_name == product_name) +# ) +# ) + +# async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: +# result = await conn.stream(base_query) +# row = await result.first() +# if row is None: +# raise LicenseNotFoundError(license_id=license_id) +# return License.model_validate(row) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_rest.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_rest.py index 1ed9613317f..91a786c6c6e 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_rest.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_rest.py @@ -136,3 +136,21 @@ async def purchase_licensed_item(request: web.Request): body_params=body_params, ) return web.json_response(status=status.HTTP_204_NO_CONTENT) + + +@routes.post( + f"/{VTAG}/catalog/licensed-items/{{licensed_item_id}}:resync", + name="resync_licensed_item", +) +@login_required +@permission_required("catalog/licensed-items.admin") +@handle_plugin_requests_exceptions +async def resync_licensed_item(request: web.Request): + req_ctx = LicensedItemsRequestContext.model_validate(request) + + # TODO: forces resync. can schedule a run with a lock? + # fetches resources + # register resources + # returns issues + + return web.json_response(status=status.HTTP_202_ACCEPTED) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py index f530b4cb910..2a41967633b 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py @@ -2,16 +2,25 @@ import logging from datetime import UTC, datetime, timedelta +from enum import Enum, auto +from pprint import pformat +from typing import NamedTuple from aiohttp import web -from models_library.licensed_items import LicensedItemID +from deepdiff import DeepDiff +from models_library.licensed_items import ( + LicensedItemDB, + LicensedItemID, + LicensedItemUpdateDB, + LicensedResourceType, +) from models_library.products import ProductName from models_library.resource_tracker_licensed_items_purchases import ( LicensedItemsPurchasesCreate, ) from models_library.rest_ordering import OrderBy from models_library.users import UserID -from pydantic import NonNegativeInt +from pydantic import BaseModel, NonNegativeInt from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker import ( licensed_items_purchases, ) @@ -23,11 +32,90 @@ from ..wallets.errors import WalletNotEnoughCreditsError from . import _licensed_items_repository from ._common.models import LicensedItem, LicensedItemPage, LicensedItemsBodyParams -from .errors import LicensedItemPricingPlanMatchError +from .errors import LicensedItemNotFoundError, LicensedItemPricingPlanMatchError _logger = logging.getLogger(__name__) +class RegistrationState(Enum): + ALREADY_REGISTERED = auto() + DIFFERENT_RESOURCE = auto() + NEWLY_REGISTERED = auto() + + +class RegistrationResult(NamedTuple): + registered: LicensedItemDB + state: RegistrationState + message: str | None + + +async def register_resource_as_licensed_item( + app: web.Application, + *, + licensed_resource_name: str, + licensed_resource_type: LicensedResourceType, + licensed_resource_data: BaseModel, + licensed_item_display_name: str, +) -> RegistrationResult: + # NOTE about the implementation choice: + # Using `create_if_not_exists` (INSERT with IGNORE_ON_CONFLICT) would have been an option, + # but it generates excessive error logs due to conflicts. + # + # To avoid this, we first attempt to retrieve the resource using `get_by_resource_identifier` (GET). + # If the resource does not exist, we proceed with `create_if_not_exists` (INSERT with IGNORE_ON_CONFLICT). + # + # This approach not only reduces unnecessary error logs but also helps prevent race conditions + # when multiple concurrent calls attempt to register the same resource. + + resource_key = f"{licensed_resource_type}, {licensed_resource_name}" + new_licensed_resource_data = licensed_resource_data.model_dump( + mode="json", + exclude_unset=True, + ) + + try: + licensed_item = await _licensed_items_repository.get_by_resource_identifier( + app, + licensed_resource_name=licensed_resource_name, + licensed_resource_type=licensed_resource_type, + ) + + if licensed_item.licensed_resource_data != new_licensed_resource_data: + ddiff = DeepDiff( + licensed_item.licensed_resource_data, new_licensed_resource_data + ) + msg = ( + f"DIFFERENT_RESOURCE: {resource_key=} found in licensed_item_id={licensed_item.licensed_item_id} with different data. " + f"Diff:\n\t{pformat(ddiff, indent=2, width=200)}" + ) + return RegistrationResult( + licensed_item, RegistrationState.DIFFERENT_RESOURCE, msg + ) + + return RegistrationResult( + licensed_item, + RegistrationState.ALREADY_REGISTERED, + f"ALREADY_REGISTERED: {resource_key=} found in licensed_item_id={licensed_item.licensed_item_id}", + ) + + except LicensedItemNotFoundError: + licensed_item = await _licensed_items_repository.create_if_not_exists( + app, + display_name=licensed_item_display_name, + licensed_resource_name=licensed_resource_name, + licensed_resource_type=licensed_resource_type, + licensed_resource_data=new_licensed_resource_data, + product_name=None, + pricing_plan_id=None, + ) + + return RegistrationResult( + licensed_item, + RegistrationState.NEWLY_REGISTERED, + f"NEWLY_REGISTERED: {resource_key=} registered with licensed_item_id={licensed_item.licensed_item_id}", + ) + + async def get_licensed_item( app: web.Application, *, @@ -85,6 +173,34 @@ async def list_licensed_items( ) +async def trash_licensed_item( + app: web.Application, + *, + product_name: ProductName, + licensed_item_id: LicensedItemID, +): + await _licensed_items_repository.update( + app, + product_name=product_name, + licensed_item_id=licensed_item_id, + updates=LicensedItemUpdateDB(trash=True), + ) + + +async def untrash_licensed_item( + app: web.Application, + *, + product_name: ProductName, + licensed_item_id: LicensedItemID, +): + await _licensed_items_repository.update( + app, + product_name=product_name, + licensed_item_id=licensed_item_id, + updates=LicensedItemUpdateDB(trash=True), + ) + + async def purchase_licensed_item( app: web.Application, *, diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licenses_repository.py b/services/web/server/src/simcore_service_webserver/licenses/_licenses_repository.py new file mode 100644 index 00000000000..27a97b5ac42 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/_licenses_repository.py @@ -0,0 +1,317 @@ +""" Database API + + - Adds a layer to the postgres API with a focus on the projects comments + +""" + +import logging +from typing import cast + +from aiohttp import web +from models_library.licensed_items import LicensedItemID, LicensedResourceType +from models_library.licenses import License, LicenseDB, LicenseID, LicenseUpdateDB +from models_library.products import ProductName +from models_library.resource_tracker import PricingPlanId +from models_library.rest_ordering import OrderBy, OrderDirection +from pydantic import NonNegativeInt +from simcore_postgres_database.models.license_to_resource import license_to_resource +from simcore_postgres_database.models.licensed_items import licensed_resources +from simcore_postgres_database.models.licenses import licenses +from simcore_postgres_database.utils_repos import ( + get_columns_from_db_model, + pass_or_acquire_connection, + transaction_context, +) +from sqlalchemy import asc, desc, func +from sqlalchemy.ext.asyncio import AsyncConnection +from sqlalchemy.sql import select + +from ..db.plugin import get_asyncpg_engine +from .errors import LicenseNotFoundError + +_logger = logging.getLogger(__name__) + + +_SELECTION_ARGS = get_columns_from_db_model(licenses, LicenseDB) + + +async def create( + app: web.Application, + connection: AsyncConnection | None = None, + *, + display_name: str, + licensed_resource_type: LicensedResourceType, + product_name: ProductName, + pricing_plan_id: PricingPlanId, +) -> LicenseDB: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.execute( + licenses.insert() + .values( + product_name=product_name, + display_name=display_name, + licensed_resource_type=licensed_resource_type, + pricing_plan_id=pricing_plan_id, + created=func.now(), + modified=func.now(), + ) + .returning(*_SELECTION_ARGS) + ) + row = result.one() + return LicenseDB.model_validate(row) + + +async def list_( + app: web.Application, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + offset: NonNegativeInt, + limit: NonNegativeInt, + order_by: OrderBy, +) -> tuple[int, list[LicenseDB]]: + + base_query = ( + select(*_SELECTION_ARGS) + .select_from(licenses) + .where(licenses.c.product_name == product_name) + ) + + # Select total count from base_query + subquery = base_query.subquery() + count_query = select(func.count()).select_from(subquery) + + # Ordering and pagination + if order_by.direction == OrderDirection.ASC: + list_query = base_query.order_by(asc(getattr(licenses.c, order_by.field))) + else: + list_query = base_query.order_by(desc(getattr(licenses.c, order_by.field))) + list_query = list_query.offset(offset).limit(limit) + + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + total_count = await conn.scalar(count_query) + + result = await conn.stream(list_query) + items: list[LicenseDB] = [LicenseDB.model_validate(row) async for row in result] + + return cast(int, total_count), items + + +async def get( + app: web.Application, + connection: AsyncConnection | None = None, + *, + license_id: LicenseID, + product_name: ProductName, +) -> LicenseDB: + base_query = ( + select(*_SELECTION_ARGS) + .select_from(licenses) + .where( + (licenses.c.license_id == license_id) + & (licenses.c.product_name == product_name) + ) + ) + + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream(base_query) + row = await result.first() + if row is None: + raise LicenseNotFoundError(license_id=license_id) + return LicenseDB.model_validate(row) + + +async def update( + app: web.Application, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + license_id: LicenseID, + updates: LicenseUpdateDB, +) -> LicenseDB: + # NOTE: at least 'touch' if updated_values is empty + _updates = { + **updates.model_dump(exclude_unset=True), + licenses.c.modified.name: func.now(), + } + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.execute( + licenses.update() + .values(**_updates) + .where( + (licenses.c.license_id == license_id) + & (licenses.c.product_name == product_name) + ) + .returning(*_SELECTION_ARGS) + ) + row = result.one_or_none() + if row is None: + raise LicenseNotFoundError(license_id=license_id) + return LicenseDB.model_validate(row) + + +async def delete( + app: web.Application, + connection: AsyncConnection | None = None, + *, + license_id: LicenseID, + product_name: ProductName, +) -> None: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.execute( + licenses.delete().where( + (licenses.c.license_id == license_id) + & (licenses.c.product_name == product_name) + ) + ) + + +#### DOMAIN MODEL + + +async def list_licenses( + app: web.Application, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + offset: NonNegativeInt, + limit: NonNegativeInt, + order_by: OrderBy, +) -> tuple[int, list[License]]: + + licensed_resources_subquery = ( + select( + license_to_resource.c.license_id, + func.jsonb_agg(licensed_resources.c.licensed_resource_data).label( + "resources" + ), + ) + .select_from( + license_to_resource.join( + licensed_resources, + license_to_resource.c.licensed_item_id + == licensed_resources.c.licensed_item_id, + ) + ) + .group_by(license_to_resource.c.license_id) + ).subquery("licensed_resources_subquery") + + base_query = ( + select( + *_SELECTION_ARGS, + licensed_resources_subquery.c.resources, + ) + .select_from( + licenses.join( + licensed_resources_subquery, + licenses.c.license_id == licensed_resources_subquery.c.license_id, + ) + ) + .where(licenses.c.product_name == product_name) + ) + + # Select total count from base_query + subquery = base_query.subquery() + count_query = select(func.count()).select_from(subquery) + + # Ordering and pagination + if order_by.direction == OrderDirection.ASC: + list_query = base_query.order_by(asc(getattr(licenses.c, order_by.field))) + else: + list_query = base_query.order_by(desc(getattr(licenses.c, order_by.field))) + list_query = list_query.offset(offset).limit(limit) + + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + total_count = await conn.scalar(count_query) + + result = await conn.stream(list_query) + items: list[License] = [License.model_validate(row) async for row in result] + + return cast(int, total_count), items + + +async def get_license( + app: web.Application, + connection: AsyncConnection | None = None, + *, + license_id: LicenseID, + product_name: ProductName, +) -> License: + licensed_resources_subquery = ( + select( + license_to_resource.c.license_id, + func.jsonb_agg(licensed_resources.c.licensed_resource_data).label( + "resources" + ), + ) + .select_from( + license_to_resource.join( + licensed_resources, + license_to_resource.c.licensed_item_id + == licensed_resources.c.licensed_item_id, + ) + ) + .where(license_to_resource.c.license_id == license_id) + .group_by(license_to_resource.c.license_id) + ).subquery("licensed_resources_subquery") + + base_query = ( + select( + *_SELECTION_ARGS, + licensed_resources_subquery.c.resources, + ) + .select_from( + licenses.join( + licensed_resources_subquery, + licenses.c.license_id == licensed_resources_subquery.c.license_id, + ) + ) + .where( + (licenses.c.product_name == product_name) + & (licenses.c.license_id == license_id) + ) + ) + + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream(base_query) + row = await result.first() + if row is None: + raise LicenseNotFoundError(license_id=license_id) + return License.model_validate(row) + + +### License to Resource + + +async def add_licensed_resource_to_license( + app: web.Application, + connection: AsyncConnection | None = None, + *, + licensed_item_id: LicensedItemID, + license_id: LicenseID, +) -> None: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.execute( + license_to_resource.insert().values( + license_id=license_id, + licensed_item_id=licensed_item_id, + created=func.now(), + modified=func.now(), + ) + ) + + +async def delete_licensed_resource_from_license( + app: web.Application, + connection: AsyncConnection | None = None, + *, + licensed_item_id: LicensedItemID, + license_id: LicenseID, +) -> None: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.execute( + license_to_resource.delete().where( + (license_to_resource.c.license_id == license_id) + & (license_to_resource.c.licensed_item_id == licensed_item_id) + ) + ) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licenses_service.py b/services/web/server/src/simcore_service_webserver/licenses/_licenses_service.py new file mode 100644 index 00000000000..a13f542ffbf --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/_licenses_service.py @@ -0,0 +1,46 @@ +# pylint: disable=unused-argument + +import logging + +from aiohttp import web +from models_library.licenses import License, LicenseID +from models_library.products import ProductName +from models_library.rest_ordering import OrderBy +from pydantic import NonNegativeInt + +from . import _licenses_repository +from ._common.models import LicensePage + +_logger = logging.getLogger(__name__) + + +async def get_license( + app: web.Application, + *, + license_id: LicenseID, + product_name: ProductName, +) -> License: + return await _licenses_repository.get_license( + app, license_id=license_id, product_name=product_name + ) + + +async def list_licenses( + app: web.Application, + *, + product_name: ProductName, + offset: NonNegativeInt, + limit: int, + order_by: OrderBy, +) -> LicensePage: + total_count, items = await _licenses_repository.list_licenses( + app, + product_name=product_name, + offset=offset, + limit=limit, + order_by=order_by, + ) + return LicensePage( + items=items, + total=total_count, + ) diff --git a/services/web/server/src/simcore_service_webserver/licenses/errors.py b/services/web/server/src/simcore_service_webserver/licenses/errors.py index 18c57966123..f4238b8029b 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/errors.py +++ b/services/web/server/src/simcore_service_webserver/licenses/errors.py @@ -5,6 +5,10 @@ class LicensesValueError(WebServerBaseError, ValueError): ... +class LicenseNotFoundError(LicensesValueError): + msg_template = "License {license_id} not found" + + class LicensedItemNotFoundError(LicensesValueError): msg_template = "License good {licensed_item_id} not found" diff --git a/services/web/server/src/simcore_service_webserver/licenses/plugin.py b/services/web/server/src/simcore_service_webserver/licenses/plugin.py index 859a52bf1bd..2402140c1f1 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/plugin.py +++ b/services/web/server/src/simcore_service_webserver/licenses/plugin.py @@ -10,6 +10,7 @@ from ..rabbitmq import setup_rabbitmq from ..rest.plugin import setup_rest from . import ( + _itis_vip_syncer_service, _licensed_items_checkouts_rest, _licensed_items_purchases_rest, _licensed_items_rest, @@ -37,3 +38,6 @@ def setup_licenses(app: web.Application): setup_rabbitmq(app) if app[APP_SETTINGS_KEY].WEBSERVER_RABBITMQ: app.on_startup.append(_rpc.register_rpc_routes_on_startup) + + # TODO: this is temporary + _itis_vip_syncer_service.setup_itis_vip_syncer(app) diff --git a/services/web/server/src/simcore_service_webserver/licenses/settings.py b/services/web/server/src/simcore_service_webserver/licenses/settings.py new file mode 100644 index 00000000000..902a058ee80 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/settings.py @@ -0,0 +1,19 @@ +import datetime +from typing import Annotated + +from pydantic import Field +from settings_library.base import BaseCustomSettings + +from ._itis_vip_settings import ItisVipSettings + + +class LicensesSettings(BaseCustomSettings): + LICENSES_SYNCER_ENABLED: bool + LICENSES_SYNCER_PERIODICITY: datetime.timedelta + + # Registered licensed resources: + LICENSES_ITIS_VIP: Annotated[ + ItisVipSettings | None, Field(description="Settings for VIP license models") + ] + + # other licensed resources come here ... diff --git a/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py b/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py index 0bd7e6a75eb..57a3aedd3f9 100644 --- a/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py +++ b/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py @@ -111,8 +111,9 @@ class PermissionDict(TypedDict, total=False): UserRole.ADMIN: PermissionDict( can=[ "admin.*", - "storage.files.sync", + "catalog/licensed-items.admin", "resource-usage.write", + "storage.files.sync", ], inherits=[UserRole.TESTER], ), diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py b/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py index 5971ed9f168..4e611d31561 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py @@ -5,7 +5,7 @@ # pylint:disable=redefined-outer-name import pytest from aiohttp.test_utils import TestClient -from simcore_postgres_database.models.licensed_items import licensed_items +from simcore_postgres_database.models.licensed_items import licensed_resources from simcore_postgres_database.models.resource_tracker_pricing_plans import ( resource_tracker_pricing_plans, ) @@ -33,12 +33,26 @@ async def pricing_plan_id( ) .returning(resource_tracker_pricing_plans.c.pricing_plan_id) ) - row = result.first() + row = result.one() assert row yield int(row[0]) async with transaction_context(get_asyncpg_engine(client.app)) as conn: - await conn.execute(licensed_items.delete()) + await conn.execute(licensed_resources.delete()) await conn.execute(resource_tracker_pricing_plans.delete()) + + +@pytest.fixture +async def ensure_empty_licensed_items(client: TestClient): + async def _cleanup(): + assert client.app + async with transaction_context(get_asyncpg_engine(client.app)) as conn: + await conn.execute(await conn.execute(licensed_resources.delete()).delete()) + + await _cleanup() + + yield + + await _cleanup() diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_models.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_models.py new file mode 100644 index 00000000000..44b80716fc8 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_models.py @@ -0,0 +1,47 @@ +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments +# pylint: disable=unused-argument +# pylint: disable=unused-variable + + +import pytest +from faker import Faker +from pydantic import ValidationError +from pytest_simcore.helpers.faker_factories import ( + random_itis_vip_available_download_item, +) +from simcore_service_webserver.licenses._itis_vip_models import ( + ItisVipData, + ItisVipResourceData, + _feature_descriptor_to_dict, +) + + +def test_pre_validator_feature_descriptor_to_dict(): + # Makes sure the regex used here, which is vulnerable to polynomial runtime due to backtracking, cannot lead to denial of service. + with pytest.raises(ValidationError) as err_info: + _feature_descriptor_to_dict("a" * 10000 + ": " + "b" * 10000) + assert err_info.value.errors()[0]["type"] == "string_too_long" + + +def test_validation_of_itis_vip_response_model(faker: Faker): + + available_download = random_itis_vip_available_download_item( + identifier=0, + features_functionality="Posable", + fake=faker, + ) + + vip_data = ItisVipData.model_validate(available_download) + + # Dumped as in the source + assert vip_data.model_dump(by_alias=True)["Features"] == vip_data.features + + license_resource_data = ItisVipResourceData.create( + category_id="123", + category_display="This is a resource", + source=vip_data, + ) + + assert license_resource_data.source["Features"] == vip_data.features diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py new file mode 100644 index 00000000000..50f437ecc78 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -0,0 +1,227 @@ +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments +# pylint: disable=unused-argument +# pylint: disable=unused-variable + +from collections.abc import Iterator + +import pytest +import respx +from aiohttp.test_utils import TestClient +from faker import Faker +from httpx import AsyncClient +from models_library.licensed_items import LicensedResourceType +from pydantic import ValidationError +from pytest_mock import MockerFixture +from pytest_simcore.helpers.faker_factories import ( + random_itis_vip_available_download_item, +) +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict +from servicelib.aiohttp import status +from simcore_service_webserver.licenses import ( + _itis_vip_service, + _itis_vip_syncer_service, + _licensed_items_service, +) +from simcore_service_webserver.licenses._itis_vip_models import ItisVipData +from simcore_service_webserver.licenses._itis_vip_service import ItisVipApiResponse +from simcore_service_webserver.licenses._itis_vip_settings import ItisVipSettings +from simcore_service_webserver.licenses._licensed_items_service import RegistrationState + + +@pytest.fixture(scope="session") +def fake_api_base_url() -> str: + return "https://testserver-itis-vip.xyz" + + +@pytest.fixture +def app_environment( + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, + fake_api_base_url: str, + mocker: MockerFixture, +): + # prevents syncer setup + mocker.patch( + "simcore_service_webserver.licenses.plugin._itis_vip_syncer_service.setup_itis_vip_syncer", + autospec=True, + ) + + return app_environment | setenvs_from_dict( + monkeypatch, + { + "ITIS_VIP_API_URL": f"{fake_api_base_url}/PD_DirectDownload/getDownloadableItems/{{category}}", + # NOTE: ItisVipSettings will decode with json.dumps(). Use " and not ' the json keys!! + "ITIS_VIP_CATEGORIES": '{"ComputationalPantom": "Phantoms", "HumanBodyRegion": "Humans (Regions)"}', + }, + ) + + +@pytest.fixture +def mock_itis_vip_downloadables_api( + faker: Faker, fake_api_base_url: str +) -> Iterator[respx.MockRouter]: + response_data = { + "msg": 0, + "availableDownloads": [ + random_itis_vip_available_download_item( + identifier=i, + features_functionality="Posable", + fake=faker, + ) + for i in range(8) + ], + } + + with respx.mock(base_url=fake_api_base_url) as mock: + mock.post(path__regex=r"/getDownloadableItems/(?P\w+)").respond( + status_code=200, json=response_data + ) + yield mock + + +async def test_fetch_and_validate_itis_vip_api( + mock_itis_vip_downloadables_api: respx.MockRouter, fake_api_base_url: str +): + async with AsyncClient(base_url=fake_api_base_url) as client: + response = await client.post("/getDownloadableItems/ComputationalPantom") + assert response.status_code == status.HTTP_200_OK + response_json = response.json() + + try: + validated_data = ItisVipApiResponse(**response_json) + except ValidationError as e: + pytest.fail(f"Response validation failed: {e}") + + assert validated_data.msg == 0 + assert len(validated_data.available_downloads) == 8 + + assert ( + validated_data.available_downloads[0].features.get("functionality") + == "Posable" + ) + + print(validated_data.model_dump_json(indent=1)) + + +async def test_get_category_items( + mock_itis_vip_downloadables_api: respx.MockRouter, + app_environment: EnvVarsDict, +): + settings = ItisVipSettings.create_from_envs() + assert settings.ITIS_VIP_CATEGORIES + + async with AsyncClient() as client: + for url, category in zip( + settings.get_urls(), settings.ITIS_VIP_CATEGORIES, strict=True + ): + assert f"{url}".endswith(category) + + items = await _itis_vip_service.get_category_items(client, url) + + assert items[0].features.get("functionality") == "Posable" + + +async def test_sync_itis_vip_as_licensed_items( + mock_itis_vip_downloadables_api: respx.MockRouter, + app_environment: EnvVarsDict, + client: TestClient, + ensure_empty_licensed_items: None, +): + assert client.app + + settings = ItisVipSettings.create_from_envs() + assert settings.ITIS_VIP_CATEGORIES + + async with AsyncClient() as http_client: + for url, category in zip( + settings.get_urls(), settings.ITIS_VIP_CATEGORIES, strict=True + ): + assert f"{url}".endswith(category) + + vip_resources: list[ + ItisVipData + ] = await _itis_vip_service.get_category_items(http_client, url) + assert vip_resources[0].features.get("functionality") == "Posable" + + for vip in vip_resources: + + # register a NEW resource + ( + licensed_item1, + state1, + _, + ) = await _licensed_items_service.register_resource_as_licensed_item( + client.app, + licensed_resource_name=f"{category}/{vip.id}", + licensed_resource_type=LicensedResourceType.VIP_MODEL, + licensed_resource_data=vip, + licensed_item_display_name="foo", + ) + assert state1 == RegistrationState.NEWLY_REGISTERED + + # register the SAME resource + ( + licensed_item2, + state2, + _, + ) = await _licensed_items_service.register_resource_as_licensed_item( + client.app, + licensed_resource_name=f"{category}/{vip.id}", + licensed_resource_type=LicensedResourceType.VIP_MODEL, + licensed_resource_data=vip, + licensed_item_display_name="foo", + ) + + assert state2 == RegistrationState.ALREADY_REGISTERED + assert licensed_item1 == licensed_item2 + + # register a MODIFIED version of the same resource + ( + licensed_item3, + state3, + msg, + ) = await _licensed_items_service.register_resource_as_licensed_item( + client.app, + licensed_resource_name=f"{category}/{vip.id}", + licensed_resource_type=LicensedResourceType.VIP_MODEL, + licensed_resource_data=vip.model_copy( + update={ + "features": { + **vip.features, + "functionality": "Non-Posable", + } + } + ), + licensed_item_display_name="foo", + ) + assert state3 == RegistrationState.DIFFERENT_RESOURCE + assert licensed_item2 == licensed_item3 + # {'values_changed': {"root['features']['functionality']": {'new_value': 'Non-Posable', 'old_value': 'Posable'}}} + assert "functionality" in msg + + +async def test_itis_vip_syncer_service( + mock_itis_vip_downloadables_api: respx.MockRouter, + app_environment: EnvVarsDict, + client: TestClient, + ensure_empty_licensed_items: None, +): + assert client.app + + settings = ItisVipSettings.create_from_envs() + assert settings.ITIS_VIP_CATEGORIES + + categories = settings.to_categories() + + # one round + await _itis_vip_syncer_service.sync_resources_with_licensed_items( + client.app, categories + ) + + # second round + await _itis_vip_syncer_service.sync_resources_with_licensed_items( + client.app, categories + ) diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_licenses_repository.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_licenses_repository.py new file mode 100644 index 00000000000..f64f5082921 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_licenses_repository.py @@ -0,0 +1,135 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments +# pylint: disable=too-many-statements + +import pytest +from aiohttp.test_utils import TestClient +from models_library.licensed_items import VIP_DETAILS_EXAMPLE, LicensedResourceType +from models_library.licenses import License, LicenseDB, LicenseUpdateDB +from models_library.rest_ordering import OrderBy +from pytest_simcore.helpers.webserver_login import UserInfoDict +from simcore_service_webserver.db.models import UserRole +from simcore_service_webserver.licenses import ( + _licensed_items_repository, + _licenses_repository, +) +from simcore_service_webserver.licenses.errors import LicenseNotFoundError +from simcore_service_webserver.projects.models import ProjectDict + + +@pytest.fixture +def user_role() -> UserRole: + return UserRole.USER + + +async def test_licenses_db_crud( + client: TestClient, + logged_user: UserInfoDict, + user_project: ProjectDict, + osparc_product_name: str, + pricing_plan_id: int, +): + got = await _licensed_items_repository.create( + client.app, + product_name=osparc_product_name, + display_name="Model A Display Name", + licensed_resource_name="Model A", + licensed_resource_type=LicensedResourceType.VIP_MODEL, + licensed_resource_data=VIP_DETAILS_EXAMPLE, + pricing_plan_id=pricing_plan_id, + ) + licensed_item_id = got.licensed_item_id + + ### NEW: + + got = await _licenses_repository.create( + client.app, + product_name=osparc_product_name, + display_name="Model A", + licensed_resource_type=LicensedResourceType.VIP_MODEL, + pricing_plan_id=pricing_plan_id, + ) + license_id = got.license_id + + total_count, items = await _licenses_repository.list_( + client.app, + product_name=osparc_product_name, + offset=0, + limit=10, + order_by=OrderBy(field="display_name"), + ) + assert total_count == 1 + assert items[0].license_id == license_id + + got = await _licenses_repository.get( + client.app, + license_id=license_id, + product_name=osparc_product_name, + ) + assert isinstance(got, LicenseDB) + assert got.display_name == "Model A" + + got = await _licenses_repository.update( + client.app, + license_id=license_id, + product_name=osparc_product_name, + updates=LicenseUpdateDB(display_name="Model B"), + ) + assert isinstance(got, LicenseDB) + + got = await _licenses_repository.get( + client.app, + license_id=license_id, + product_name=osparc_product_name, + ) + assert isinstance(got, LicenseDB) + assert got.display_name == "Model B" + + # CONNECT RESOURCE TO LICENSE + + await _licenses_repository.add_licensed_resource_to_license( + client.app, + license_id=license_id, + licensed_item_id=licensed_item_id, + ) + + got = await _licenses_repository.list_licenses( + client.app, + product_name=osparc_product_name, + offset=0, + limit=10, + order_by=OrderBy(field="display_name"), + ) + assert got[0] == 1 + assert isinstance(got[1], list) + assert isinstance(got[1][0], License) + + got = await _licenses_repository.get_license( + client.app, + product_name=osparc_product_name, + license_id=license_id, + ) + assert isinstance(got, License) + + # DELETE + + await _licenses_repository.delete_licensed_resource_from_license( + client.app, + license_id=license_id, + licensed_item_id=licensed_item_id, + ) + + got = await _licenses_repository.delete( + client.app, + license_id=license_id, + product_name=osparc_product_name, + ) + + with pytest.raises(LicenseNotFoundError): + await _licenses_repository.get( + client.app, + license_id=license_id, + product_name=osparc_product_name, + )