From cf8dd9b6b1b63c0697cd0f167a53bf12a61725c0 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 29 Jan 2025 15:36:47 +0100 Subject: [PATCH 01/46] drafts first tests --- .../test_licensed_itis_vip_downloadables.py | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_itis_vip_downloadables.py diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_itis_vip_downloadables.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_itis_vip_downloadables.py new file mode 100644 index 00000000000..b6068a2f8f7 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_itis_vip_downloadables.py @@ -0,0 +1,85 @@ +import re +from typing import Annotated, Any, Iterator, Literal + +import pytest +import respx +from faker import Faker +from httpx import AsyncClient +from pydantic import BaseModel, Field, HttpUrl, ValidationError +from servicelib.aiohttp import status + + +@pytest.fixture +def mock_itis_vip_downloadables_api(faker: Faker) -> Iterator[respx.MockRouter]: + api_url = "http://testserver" + response_data = { + "msg": 0, + "availableDownloads": [ + { + "ID": faker.random_int(min=70, max=90), + "Description": faker.sentence(), + "Thumbnail": faker.image_url(), + "Features": f"{{name: {faker.name()} Right Hand, version: V{faker.pyint()}.0, sex: {faker.gender()}, age: {faker.age()} years,date: {faker.date()}, ethnicity: Caucasian, functionality: Posable}}", + "DOI": None, + "LicenseKey": faker.bothify(text="MODEL_????_V#"), + "LicenseVersion": "V1.0", + "Protection": "Code", + "AvailableFromURL": None, + } + for _ in range(8) + ], + } + + with respx.mock(base_url=api_url) as mock: + mock.post("getDownloadableItems/ComputationalPantom").respond( + status_code=200, json=response_data + ) + yield mock + + +def descriptor_to_dict(descriptor: str) -> dict[str, Any]: + pattern = r"(\w+): ([^,]+)" + matches = re.findall(pattern, descriptor) + return {key: value for key, value in matches} + + +class AvailableDownload(BaseModel): + id: Annotated[int, Field(alias="ID")] + description: Annotated[str, Field(alias="Description")] + thumbnail: Annotated[str, Field(alias="Thumbnail")] + features: Annotated[dict[str, Any], Field(alias="Features")] + doi: Annotated[str, Field(alias="DOI")] + license_key: Annotated[str | None, Field(alias="LicenseKey")] + license_version: Annotated[str | None, Field(alias="LicenseVersion")] + protection: Annotated[Literal["Code", "PayPal"], Field(alias="Protection")] + available_from_url: Annotated[HttpUrl | None, Field(alias="AvailableFromURL")] + + +class ResponseData(BaseModel): + msg: int = -1 + available_downloads: Annotated[ + list[AvailableDownload], Field(alias="availableDownloads") + ] + + +async def test_computational_pantom_api( + mock_itis_vip_downloadables_api: respx.MockRouter, +): + async with AsyncClient(base_url="http://testserver") as client: + response = await client.post("getDownloadableItems/ComputationalPantom") + assert response.status_code == status.HTTP_200_OK + response_json = response.json() + + try: + validated_data = ResponseData(**response_json) + except ValidationError as e: + pytest.fail(f"Response validation failed: {e}") + + assert validated_data.msg == 0 + assert len(validated_data.availableDownloads) == 8 + + assert ( + validated_data.availableDownloads[0].Features["functionality"] == "Posable" + ) + + print(validated_data.model_dump_json()) From e3d2dc7b64dc694585431b8a9531f53b7ff7f02c Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 29 Jan 2025 15:40:00 +0100 Subject: [PATCH 02/46] respx and httpx --- services/web/server/requirements/_base.in | 1 + services/web/server/requirements/_base.txt | 39 ++++++++++++++++++++++ services/web/server/requirements/_test.in | 1 + services/web/server/requirements/_test.txt | 27 +++++++++++++++ 4 files changed, 68 insertions(+) diff --git a/services/web/server/requirements/_base.in b/services/web/server/requirements/_base.in index caf883fc166..326bef1224f 100644 --- a/services/web/server/requirements/_base.in +++ b/services/web/server/requirements/_base.in @@ -34,6 +34,7 @@ captcha cryptography # security faker # Only used in dev-mode for proof-of-concepts gunicorn[setproctitle] +httpx jinja_app_loader # email json2html jsondiff diff --git a/services/web/server/requirements/_base.txt b/services/web/server/requirements/_base.txt index f35f5ab6212..e019a2d634b 100644 --- a/services/web/server/requirements/_base.txt +++ b/services/web/server/requirements/_base.txt @@ -88,6 +88,7 @@ anyio==4.3.0 # via # fast-depends # faststream + # httpx appdirs==1.4.4 # via pint arrow==1.2.3 @@ -145,6 +146,8 @@ certifi==2023.7.22 # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../../requirements/constraints.txt + # httpcore + # httpx # requests cffi==1.17.1 # via cryptography @@ -225,10 +228,46 @@ grpcio==1.66.0 # via opentelemetry-exporter-otlp-proto-grpc gunicorn==23.0.0 # via -r requirements/_base.in +h11==0.14.0 + # via httpcore +httpcore==1.0.7 + # via httpx +httpx==0.28.1 + # via + # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt + # -c requirements/../../../../requirements/constraints.txt + # -r requirements/_base.in idna==3.3 # via # anyio # email-validator + # httpx # requests # yarl importlib-metadata==8.0.0 diff --git a/services/web/server/requirements/_test.in b/services/web/server/requirements/_test.in index 22ed8e423bc..0644e604010 100644 --- a/services/web/server/requirements/_test.in +++ b/services/web/server/requirements/_test.in @@ -38,6 +38,7 @@ pytest-sugar pytest-xdist python-dotenv redis +respx sqlalchemy[mypy] # adds Mypy / Pep-484 Support for ORM Mappings SEE https://docs.sqlalchemy.org/en/20/orm/extensions/mypy.html tenacity types-aiofiles diff --git a/services/web/server/requirements/_test.txt b/services/web/server/requirements/_test.txt index 6beb543d3f8..08cf7390c84 100644 --- a/services/web/server/requirements/_test.txt +++ b/services/web/server/requirements/_test.txt @@ -14,6 +14,10 @@ alembic==1.8.1 # via # -c requirements/_base.txt # -r requirements/_test.in +anyio==4.3.0 + # via + # -c requirements/_base.txt + # httpx async-timeout==4.0.3 # via # -c requirements/_base.txt @@ -35,6 +39,8 @@ certifi==2023.7.22 # via # -c requirements/../../../../requirements/constraints.txt # -c requirements/_base.txt + # httpcore + # httpx # requests charset-normalizer==2.0.12 # via @@ -68,6 +74,19 @@ greenlet==2.0.2 # via # -c requirements/_base.txt # sqlalchemy +h11==0.14.0 + # via + # -c requirements/_base.txt + # httpcore +httpcore==1.0.7 + # via + # -c requirements/_base.txt + # httpx +httpx==0.28.1 + # via + # -c requirements/../../../../requirements/constraints.txt + # -c requirements/_base.txt + # respx hypothesis==6.91.0 # via -r requirements/_test.in icdiff==2.0.7 @@ -75,6 +94,8 @@ icdiff==2.0.7 idna==3.3 # via # -c requirements/_base.txt + # anyio + # httpx # requests # yarl iniconfig==2.0.0 @@ -185,6 +206,8 @@ requests==2.32.2 # via # -c requirements/_base.txt # docker +respx==0.22.0 + # via -r requirements/_test.in setuptools==69.1.1 # via # -c requirements/_base.txt @@ -195,6 +218,10 @@ six==1.16.0 # -c requirements/_base.txt # jsonschema # python-dateutil +sniffio==1.3.1 + # via + # -c requirements/_base.txt + # anyio sortedcontainers==2.4.0 # via hypothesis sqlalchemy==1.4.47 From 19dff615f805eeb079c58519ebe14fba344c792b Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 29 Jan 2025 15:40:14 +0100 Subject: [PATCH 03/46] drafts test --- ...ables.py => test_licensed_itis_vip_api.py} | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) rename services/web/server/tests/unit/with_dbs/04/licenses/{test_licensed_itis_vip_downloadables.py => test_licensed_itis_vip_api.py} (59%) diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_itis_vip_downloadables.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_itis_vip_api.py similarity index 59% rename from services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_itis_vip_downloadables.py rename to services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_itis_vip_api.py index b6068a2f8f7..9fd45233d64 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_itis_vip_downloadables.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_itis_vip_api.py @@ -1,11 +1,18 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments +# pylint: disable=too-many-statements + import re -from typing import Annotated, Any, Iterator, Literal +from collections.abc import Iterator +from typing import Annotated, Any, Literal import pytest import respx from faker import Faker from httpx import AsyncClient -from pydantic import BaseModel, Field, HttpUrl, ValidationError +from pydantic import BaseModel, BeforeValidator, Field, HttpUrl, ValidationError from servicelib.aiohttp import status @@ -19,12 +26,13 @@ def mock_itis_vip_downloadables_api(faker: Faker) -> Iterator[respx.MockRouter]: "ID": faker.random_int(min=70, max=90), "Description": faker.sentence(), "Thumbnail": faker.image_url(), - "Features": f"{{name: {faker.name()} Right Hand, version: V{faker.pyint()}.0, sex: {faker.gender()}, age: {faker.age()} years,date: {faker.date()}, ethnicity: Caucasian, functionality: Posable}}", - "DOI": None, + # NOTE: this is manually added in the server side so be more robust to errors + "Features": f"{{name: {faker.name()} Right Hand, version: V{faker.pyint()}.0, sex: Male, age: 8 years,date: {faker.date()}, ethnicity: Caucasian, functionality: Posable}}", + "DOI": faker.bothify(text="10.####/ViP#####-##-#"), "LicenseKey": faker.bothify(text="MODEL_????_V#"), - "LicenseVersion": "V1.0", - "Protection": "Code", - "AvailableFromURL": None, + "LicenseVersion": faker.bothify(text="V#.0"), + "Protection": faker.random_element(elements=["Code", "PayPal"]), + "AvailableFromURL": faker.random_element(elements=[None, faker.url()]), } for _ in range(8) ], @@ -37,17 +45,22 @@ def mock_itis_vip_downloadables_api(faker: Faker) -> Iterator[respx.MockRouter]: yield mock -def descriptor_to_dict(descriptor: str) -> dict[str, Any]: +def _feature_descriptor_to_dict(descriptor: str) -> dict[str, Any]: + # NOTE: this is manually added in the server side so be more robust to errors pattern = r"(\w+): ([^,]+)" - matches = re.findall(pattern, descriptor) - return {key: value for key, value in matches} + matches = re.findall(pattern, descriptor.strip("{}")) + return dict(matches) class AvailableDownload(BaseModel): id: Annotated[int, Field(alias="ID")] description: Annotated[str, Field(alias="Description")] thumbnail: Annotated[str, Field(alias="Thumbnail")] - features: Annotated[dict[str, Any], Field(alias="Features")] + features: Annotated[ + dict[str, Any], + BeforeValidator(_feature_descriptor_to_dict), + Field(alias="Features"), + ] doi: Annotated[str, Field(alias="DOI")] license_key: Annotated[str | None, Field(alias="LicenseKey")] license_version: Annotated[str | None, Field(alias="LicenseVersion")] @@ -62,7 +75,7 @@ class ResponseData(BaseModel): ] -async def test_computational_pantom_api( +async def test_fetch_itis_vip_api( mock_itis_vip_downloadables_api: respx.MockRouter, ): async with AsyncClient(base_url="http://testserver") as client: @@ -76,10 +89,10 @@ async def test_computational_pantom_api( pytest.fail(f"Response validation failed: {e}") assert validated_data.msg == 0 - assert len(validated_data.availableDownloads) == 8 + assert len(validated_data.available_downloads) == 8 assert ( - validated_data.availableDownloads[0].Features["functionality"] == "Posable" + validated_data.available_downloads[0].features["functionality"] == "Posable" ) - print(validated_data.model_dump_json()) + print(validated_data.model_dump_json(indent=1)) From 0def310227f6e7d7db9a259845e7b2268a74bb4d Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 29 Jan 2025 16:33:40 +0100 Subject: [PATCH 04/46] gets items --- .../licenses/_itis_vip_service.py | 76 +++++++++++++++++++ ...is_vip_api.py => test_itis_vip_service.py} | 35 +-------- 2 files changed, 78 insertions(+), 33 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py rename services/web/server/tests/unit/with_dbs/04/licenses/{test_licensed_itis_vip_api.py => test_itis_vip_service.py} (65%) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py new file mode 100644 index 00000000000..527c3e5d123 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py @@ -0,0 +1,76 @@ +import re +from typing import Annotated, Any, Literal +from urllib.parse import urlparse + +import httpx +import pytest +from pydantic import BaseModel, BeforeValidator, Field, HttpUrl, ValidationError +from servicelib.aiohttp import status +from settings_library.base import BaseCustomSettings + + +class ItisVipSettings(BaseCustomSettings): + ITIS_VIP_API_URL: str + ITIS_VIP_CATEGORIES: list[str] + + def get_urls(self) -> list[HttpUrl]: + return [ + HttpUrl(self.ITIS_VIP_API_URL.format(category=category)) + for category in self.ITIS_VIP_CATEGORIES + ] + + +def _feature_descriptor_to_dict(descriptor: str) -> dict[str, Any]: + # NOTE: this is manually added in the server side so be more robust to errors + pattern = r"(\w+): ([^,]+)" + matches = re.findall(pattern, descriptor.strip("{}")) + return dict(matches) + + +class AvailableDownload(BaseModel): + id: Annotated[int, Field(alias="ID")] + description: Annotated[str, Field(alias="Description")] + thumbnail: Annotated[str, Field(alias="Thumbnail")] + features: Annotated[ + dict[str, Any], + BeforeValidator(_feature_descriptor_to_dict), + Field(alias="Features"), + ] + doi: Annotated[str, Field(alias="DOI")] + license_key: Annotated[str | None, Field(alias="LicenseKey")] + license_version: Annotated[str | None, Field(alias="LicenseVersion")] + protection: Annotated[Literal["Code", "PayPal"], Field(alias="Protection")] + available_from_url: Annotated[HttpUrl | None, Field(alias="AvailableFromURL")] + + +class ResponseData(BaseModel): + msg: int | None = None # still not used + available_downloads: Annotated[ + list[AvailableDownload], Field(alias="availableDownloads") + ] + + +async def get_downloadable_items( + client: httpx.AsyncClient, url: HttpUrl +) -> list[AvailableDownload]: + response = await client.post(f"{url}") + assert response.status_code == status.HTTP_200_OK + validated_data = ResponseData.model_validate(**response.json()) + return validated_data.available_downloads + + +async def fetch_vip_downloadables(settings: ItisVipSettings): + urls, categories = settings.get_urls(), settings.ITIS_VIP_CATEGORIES + + base_url = f"{urls[0].scheme}://{urlparse(f'{urls[0]}').netloc}" + + async with httpx.AsyncClient() as client: + for url in urls: + response = await client.post(url) + assert response.status_code == status.HTTP_200_OK + response_json = response.json() + + try: + validated_data = ResponseData(**response_json) + except ValidationError as e: + pytest.fail(f"Response validation failed: {e}") diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_itis_vip_api.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py similarity index 65% rename from services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_itis_vip_api.py rename to services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index 9fd45233d64..aa72199d4b4 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_itis_vip_api.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -4,16 +4,15 @@ # pylint: disable=too-many-arguments # pylint: disable=too-many-statements -import re from collections.abc import Iterator -from typing import Annotated, Any, Literal import pytest import respx from faker import Faker from httpx import AsyncClient -from pydantic import BaseModel, BeforeValidator, Field, HttpUrl, ValidationError +from pydantic import ValidationError from servicelib.aiohttp import status +from simcore_service_webserver.licenses._itis_vip_service import ResponseData @pytest.fixture @@ -45,36 +44,6 @@ def mock_itis_vip_downloadables_api(faker: Faker) -> Iterator[respx.MockRouter]: yield mock -def _feature_descriptor_to_dict(descriptor: str) -> dict[str, Any]: - # NOTE: this is manually added in the server side so be more robust to errors - pattern = r"(\w+): ([^,]+)" - matches = re.findall(pattern, descriptor.strip("{}")) - return dict(matches) - - -class AvailableDownload(BaseModel): - id: Annotated[int, Field(alias="ID")] - description: Annotated[str, Field(alias="Description")] - thumbnail: Annotated[str, Field(alias="Thumbnail")] - features: Annotated[ - dict[str, Any], - BeforeValidator(_feature_descriptor_to_dict), - Field(alias="Features"), - ] - doi: Annotated[str, Field(alias="DOI")] - license_key: Annotated[str | None, Field(alias="LicenseKey")] - license_version: Annotated[str | None, Field(alias="LicenseVersion")] - protection: Annotated[Literal["Code", "PayPal"], Field(alias="Protection")] - available_from_url: Annotated[HttpUrl | None, Field(alias="AvailableFromURL")] - - -class ResponseData(BaseModel): - msg: int = -1 - available_downloads: Annotated[ - list[AvailableDownload], Field(alias="availableDownloads") - ] - - async def test_fetch_itis_vip_api( mock_itis_vip_downloadables_api: respx.MockRouter, ): From 874bda2e20bd1a1872f0475706cef15f7835fa41 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 29 Jan 2025 16:54:21 +0100 Subject: [PATCH 05/46] get_category_items and settings --- .../licenses/_itis_vip_service.py | 52 ++++++----------- .../licenses/settings.py | 29 ++++++++++ .../04/licenses/test_itis_vip_service.py | 57 ++++++++++++++++--- 3 files changed, 97 insertions(+), 41 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/licenses/settings.py diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py index 527c3e5d123..158f8dc5a06 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py @@ -1,23 +1,12 @@ import re from typing import Annotated, Any, Literal -from urllib.parse import urlparse import httpx -import pytest -from pydantic import BaseModel, BeforeValidator, Field, HttpUrl, ValidationError -from servicelib.aiohttp import status -from settings_library.base import BaseCustomSettings +from pydantic import BaseModel, BeforeValidator, Field, HttpUrl - -class ItisVipSettings(BaseCustomSettings): - ITIS_VIP_API_URL: str - ITIS_VIP_CATEGORIES: list[str] - - def get_urls(self) -> list[HttpUrl]: - return [ - HttpUrl(self.ITIS_VIP_API_URL.format(category=category)) - for category in self.ITIS_VIP_CATEGORIES - ] +# +# MODELS +# def _feature_descriptor_to_dict(descriptor: str) -> dict[str, Any]: @@ -50,27 +39,22 @@ class ResponseData(BaseModel): ] -async def get_downloadable_items( +# +# API +# + + +async def get_category_items( client: httpx.AsyncClient, url: HttpUrl ) -> list[AvailableDownload]: + """ + + Raises: + Any https://www.python-httpx.org/exceptions/ + httpx.HTTPStatusCode: + + """ response = await client.post(f"{url}") - assert response.status_code == status.HTTP_200_OK + response.raise_for_status() validated_data = ResponseData.model_validate(**response.json()) return validated_data.available_downloads - - -async def fetch_vip_downloadables(settings: ItisVipSettings): - urls, categories = settings.get_urls(), settings.ITIS_VIP_CATEGORIES - - base_url = f"{urls[0].scheme}://{urlparse(f'{urls[0]}').netloc}" - - async with httpx.AsyncClient() as client: - for url in urls: - response = await client.post(url) - assert response.status_code == status.HTTP_200_OK - response_json = response.json() - - try: - validated_data = ResponseData(**response_json) - except ValidationError as e: - pytest.fail(f"Response validation failed: {e}") diff --git a/services/web/server/src/simcore_service_webserver/licenses/settings.py b/services/web/server/src/simcore_service_webserver/licenses/settings.py new file mode 100644 index 00000000000..6575d534474 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/settings.py @@ -0,0 +1,29 @@ +from typing import Annotated + +from pydantic import AfterValidator, Field, HttpUrl +from settings_library.base import BaseCustomSettings + + +def _validate_url_contains_category(url: str) -> str: + if "{category}" not in url: + msg = "URL must contain '{category}'" + raise ValueError(msg) + return url + + +class ItisVipSettings(BaseCustomSettings): + ITIS_VIP_API_URL: Annotated[str, AfterValidator(_validate_url_contains_category)] + ITIS_VIP_CATEGORIES: list[str] + + def get_urls(self) -> list[HttpUrl]: + return [ + HttpUrl(self.ITIS_VIP_API_URL.format(category=category)) + for category in self.ITIS_VIP_CATEGORIES + ] + + +class LicensesSettings(BaseCustomSettings): + LICENSES_ITIS_VIP: Annotated[ + ItisVipSettings | None, Field(description="Settings for VIP license models") + ] + # other licenses resources come here diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index aa72199d4b4..0e8ce960feb 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -11,13 +11,38 @@ from faker import Faker from httpx import AsyncClient from pydantic import ValidationError +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.aiohttp import status +from simcore_service_webserver.licenses import _itis_vip_service from simcore_service_webserver.licenses._itis_vip_service import ResponseData +from simcore_service_webserver.licenses.settings import ItisVipSettings + + +@pytest.fixture(scope="session") +def fake_api_base_url() -> str: + return "https://testserver" @pytest.fixture -def mock_itis_vip_downloadables_api(faker: Faker) -> Iterator[respx.MockRouter]: - api_url = "http://testserver" +def app_environment( + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, + fake_api_base_url: str, +): + return app_environment | setenvs_from_dict( + monkeypatch, + { + "ITIS_VIP_API_URL": f"{fake_api_base_url}/PD_DirectDownload/getDownloadableItems/{{category}}", + "ITIS_VIP_CATEGORIES": "ComputationalPantom,Foo,Bar", + }, + ) + + +@pytest.fixture +def mock_itis_vip_downloadables_api( + faker: Faker, fake_api_base_url: str +) -> Iterator[respx.MockRouter]: response_data = { "msg": 0, "availableDownloads": [ @@ -37,17 +62,17 @@ def mock_itis_vip_downloadables_api(faker: Faker) -> Iterator[respx.MockRouter]: ], } - with respx.mock(base_url=api_url) as mock: - mock.post("getDownloadableItems/ComputationalPantom").respond( + with respx.mock(base_url=fake_api_base_url) as mock: + mock.post(path__regex=r"/getDownloadableItems/(?P\d+)").respond( status_code=200, json=response_data ) yield mock -async def test_fetch_itis_vip_api( - mock_itis_vip_downloadables_api: respx.MockRouter, +async def test_fetch_and_validate_itis_vip_api( + mock_itis_vip_downloadables_api: respx.MockRouter, fake_api_base_url: str ): - async with AsyncClient(base_url="http://testserver") as client: + async with AsyncClient(base_url=fake_api_base_url) as client: response = await client.post("getDownloadableItems/ComputationalPantom") assert response.status_code == status.HTTP_200_OK response_json = response.json() @@ -65,3 +90,21 @@ async def test_fetch_itis_vip_api( ) print(validated_data.model_dump_json(indent=1)) + + +async def test_get_category_items( + mock_itis_vip_downloadables_api: respx.MockRouter, + app_environment: EnvVarsDict, +): + settings = ItisVipSettings.create_from_envs() + assert settings.ITIS_VIP_CATEGORIES + + async with AsyncClient() as client: + for url, category in zip( + settings.get_urls(), settings.ITIS_VIP_CATEGORIES, strict=True + ): + assert f"{url}".endswith(category) + + items = await _itis_vip_service.get_category_items(client, url) + + assert items[0].features["functionality"] == "Posable" From 764b2636a87641d80ca6c305c297292c5787a059 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 29 Jan 2025 17:34:11 +0100 Subject: [PATCH 06/46] test pass --- .../simcore_service_webserver/licenses/_itis_vip_service.py | 2 +- .../unit/with_dbs/04/licenses/test_itis_vip_service.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py index 158f8dc5a06..3594680b816 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py @@ -56,5 +56,5 @@ async def get_category_items( """ response = await client.post(f"{url}") response.raise_for_status() - validated_data = ResponseData.model_validate(**response.json()) + validated_data = ResponseData.model_validate(response.json()) return validated_data.available_downloads diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index 0e8ce960feb..05111113bba 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -34,7 +34,7 @@ def app_environment( monkeypatch, { "ITIS_VIP_API_URL": f"{fake_api_base_url}/PD_DirectDownload/getDownloadableItems/{{category}}", - "ITIS_VIP_CATEGORIES": "ComputationalPantom,Foo,Bar", + "ITIS_VIP_CATEGORIES": '["ComputationalPantom","Foo","Bar"]', # NOTE: ItisVipSettings will decode with json.dumps() }, ) @@ -63,7 +63,7 @@ def mock_itis_vip_downloadables_api( } with respx.mock(base_url=fake_api_base_url) as mock: - mock.post(path__regex=r"/getDownloadableItems/(?P\d+)").respond( + mock.post(path__regex=r"/getDownloadableItems/(?P\w+)").respond( status_code=200, json=response_data ) yield mock @@ -73,7 +73,7 @@ async def test_fetch_and_validate_itis_vip_api( mock_itis_vip_downloadables_api: respx.MockRouter, fake_api_base_url: str ): async with AsyncClient(base_url=fake_api_base_url) as client: - response = await client.post("getDownloadableItems/ComputationalPantom") + response = await client.post("/getDownloadableItems/ComputationalPantom") assert response.status_code == status.HTTP_200_OK response_json = response.json() From 22bcb39344e2736ce924f29ae095115b1c8cbe60 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 29 Jan 2025 17:49:46 +0100 Subject: [PATCH 07/46] adds new column --- .../src/simcore_postgres_database/models/licensed_items.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py b/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py index feddae5fdfd..ef38a72c307 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py @@ -47,6 +47,12 @@ class LicensedResourceType(str, enum.Enum): nullable=True, doc="Resource metadata. Used for read-only purposes", ), + sa.Column( + "licensed_resource_data", + postgresql.JSONB, + nullable=True, + doc="Stores data related to this licensed resource that is used for read-only purposes", + ), sa.Column( "pricing_plan_id", sa.BigInteger, From f5af34a8134930886904680fa2ccfeec32d25698 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 29 Jan 2025 18:16:32 +0100 Subject: [PATCH 08/46] adding trashing --- packages/models-library/src/models_library/licensed_items.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/models-library/src/models_library/licensed_items.py b/packages/models-library/src/models_library/licensed_items.py index 53220d69c8f..57263d4c66f 100644 --- a/packages/models-library/src/models_library/licensed_items.py +++ b/packages/models-library/src/models_library/licensed_items.py @@ -43,5 +43,4 @@ class LicensedItemDB(BaseModel): class LicensedItemUpdateDB(BaseModel): licensed_resource_name: str | None = None pricing_plan_id: PricingPlanId | None = None - trash: bool | None = None From af2fed0808c2641d51e56d05568448d14230f28b Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 30 Jan 2025 00:45:12 +0100 Subject: [PATCH 09/46] tests pass --- .../src/simcore_postgres_database/models/licensed_items.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py b/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py index ef38a72c307..bb613e1a342 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py @@ -51,7 +51,7 @@ class LicensedResourceType(str, enum.Enum): "licensed_resource_data", postgresql.JSONB, nullable=True, - doc="Stores data related to this licensed resource that is used for read-only purposes", + doc="Stores metadata related to this licensed resource. Used for read-only purposes", ), sa.Column( "pricing_plan_id", From f227738c6f14d72508aa0ab103ec0c495d412c5a Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 30 Jan 2025 01:12:25 +0100 Subject: [PATCH 10/46] split settings --- .../licenses/_itis_vip_settings.py | 22 +++++++++++++++++++ .../licenses/settings.py | 20 ++--------------- .../04/licenses/test_itis_vip_service.py | 2 +- 3 files changed, 25 insertions(+), 19 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/licenses/_itis_vip_settings.py diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_settings.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_settings.py new file mode 100644 index 00000000000..2fb2542c396 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_settings.py @@ -0,0 +1,22 @@ +from typing import Annotated + +from pydantic import AfterValidator, HttpUrl +from settings_library.base import BaseCustomSettings + + +def _validate_url_contains_category(url: str) -> str: + if "{category}" not in url: + msg = "URL must contain '{category}'" + raise ValueError(msg) + return url + + +class ItisVipSettings(BaseCustomSettings): + ITIS_VIP_API_URL: Annotated[str, AfterValidator(_validate_url_contains_category)] + ITIS_VIP_CATEGORIES: list[str] + + def get_urls(self) -> list[HttpUrl]: + return [ + HttpUrl(self.ITIS_VIP_API_URL.format(category=category)) + for category in self.ITIS_VIP_CATEGORIES + ] diff --git a/services/web/server/src/simcore_service_webserver/licenses/settings.py b/services/web/server/src/simcore_service_webserver/licenses/settings.py index 6575d534474..a5c8355e58f 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/settings.py +++ b/services/web/server/src/simcore_service_webserver/licenses/settings.py @@ -1,25 +1,9 @@ from typing import Annotated -from pydantic import AfterValidator, Field, HttpUrl +from pydantic import Field from settings_library.base import BaseCustomSettings - -def _validate_url_contains_category(url: str) -> str: - if "{category}" not in url: - msg = "URL must contain '{category}'" - raise ValueError(msg) - return url - - -class ItisVipSettings(BaseCustomSettings): - ITIS_VIP_API_URL: Annotated[str, AfterValidator(_validate_url_contains_category)] - ITIS_VIP_CATEGORIES: list[str] - - def get_urls(self) -> list[HttpUrl]: - return [ - HttpUrl(self.ITIS_VIP_API_URL.format(category=category)) - for category in self.ITIS_VIP_CATEGORIES - ] +from ._itis_vip_settings import ItisVipSettings class LicensesSettings(BaseCustomSettings): diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index 05111113bba..8915b4ec6f0 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -16,7 +16,7 @@ from servicelib.aiohttp import status from simcore_service_webserver.licenses import _itis_vip_service from simcore_service_webserver.licenses._itis_vip_service import ResponseData -from simcore_service_webserver.licenses.settings import ItisVipSettings +from simcore_service_webserver.licenses._itis_vip_settings import ItisVipSettings @pytest.fixture(scope="session") From d245e70637e0407eee2b89b2ce8651a40a6c3436 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 30 Jan 2025 01:14:26 +0100 Subject: [PATCH 11/46] split models --- .../licenses/_itis_vip_models.py | 34 ++++++++++++++ .../licenses/_itis_vip_service.py | 44 +------------------ 2 files changed, 36 insertions(+), 42 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py new file mode 100644 index 00000000000..1b7b64e1e30 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py @@ -0,0 +1,34 @@ +import re +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, BeforeValidator, Field, HttpUrl + + +def _feature_descriptor_to_dict(descriptor: str) -> dict[str, Any]: + # NOTE: this is manually added in the server side so be more robust to errors + pattern = r"(\w+): ([^,]+)" + matches = re.findall(pattern, descriptor.strip("{}")) + return dict(matches) + + +class AvailableDownload(BaseModel): + id: Annotated[int, Field(alias="ID")] + description: Annotated[str, Field(alias="Description")] + thumbnail: Annotated[str, Field(alias="Thumbnail")] + features: Annotated[ + dict[str, Any], + BeforeValidator(_feature_descriptor_to_dict), + Field(alias="Features"), + ] + doi: Annotated[str, Field(alias="DOI")] + license_key: Annotated[str | None, Field(alias="LicenseKey")] + license_version: Annotated[str | None, Field(alias="LicenseVersion")] + protection: Annotated[Literal["Code", "PayPal"], Field(alias="Protection")] + available_from_url: Annotated[HttpUrl | None, Field(alias="AvailableFromURL")] + + +class ResponseData(BaseModel): + msg: int | None = None # still not used + available_downloads: Annotated[ + list[AvailableDownload], Field(alias="availableDownloads") + ] diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py index 3594680b816..306fd08c931 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py @@ -1,47 +1,7 @@ -import re -from typing import Annotated, Any, Literal - import httpx -from pydantic import BaseModel, BeforeValidator, Field, HttpUrl - -# -# MODELS -# - - -def _feature_descriptor_to_dict(descriptor: str) -> dict[str, Any]: - # NOTE: this is manually added in the server side so be more robust to errors - pattern = r"(\w+): ([^,]+)" - matches = re.findall(pattern, descriptor.strip("{}")) - return dict(matches) - - -class AvailableDownload(BaseModel): - id: Annotated[int, Field(alias="ID")] - description: Annotated[str, Field(alias="Description")] - thumbnail: Annotated[str, Field(alias="Thumbnail")] - features: Annotated[ - dict[str, Any], - BeforeValidator(_feature_descriptor_to_dict), - Field(alias="Features"), - ] - doi: Annotated[str, Field(alias="DOI")] - license_key: Annotated[str | None, Field(alias="LicenseKey")] - license_version: Annotated[str | None, Field(alias="LicenseVersion")] - protection: Annotated[Literal["Code", "PayPal"], Field(alias="Protection")] - available_from_url: Annotated[HttpUrl | None, Field(alias="AvailableFromURL")] - - -class ResponseData(BaseModel): - msg: int | None = None # still not used - available_downloads: Annotated[ - list[AvailableDownload], Field(alias="availableDownloads") - ] - +from pydantic import HttpUrl -# -# API -# +from ._itis_vip_models import AvailableDownload, ResponseData async def get_category_items( From 3d40c4b2b69b75d547cdbc51fdaaad168b4d3ea5 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 30 Jan 2025 01:31:03 +0100 Subject: [PATCH 12/46] sonar security --- .../licenses/_itis_vip_models.py | 10 +++++++++- .../unit/with_dbs/04/licenses/test_itis_vip_service.py | 8 ++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py index 1b7b64e1e30..af752ce159c 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py @@ -3,10 +3,18 @@ from pydantic import BaseModel, BeforeValidator, Field, HttpUrl +_MAX_LENGTH = 1_000 + def _feature_descriptor_to_dict(descriptor: str) -> dict[str, Any]: # NOTE: this is manually added in the server side so be more robust to errors - pattern = r"(\w+): ([^,]+)" + # Safe against polynomial runtime vulnerability due to backtracking + if (size := len(descriptor)) and size > _MAX_LENGTH: + msg = f"Features field too long [{size=}]" + raise ValueError(msg) + + pattern = r"(\w{1,100}): ([^,]{1,100})" + matches = re.findall(pattern, descriptor.strip("{}")) return dict(matches) diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index 8915b4ec6f0..fb7e4277bf8 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -15,6 +15,9 @@ from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.aiohttp import status from simcore_service_webserver.licenses import _itis_vip_service +from simcore_service_webserver.licenses._itis_vip_models import ( + _feature_descriptor_to_dict, +) from simcore_service_webserver.licenses._itis_vip_service import ResponseData from simcore_service_webserver.licenses._itis_vip_settings import ItisVipSettings @@ -108,3 +111,8 @@ async def test_get_category_items( items = await _itis_vip_service.get_category_items(client, url) assert items[0].features["functionality"] == "Posable" + + +def test_pre_validator_feature_descriptor_to_dict(): + with pytest.raises(ValueError): + _feature_descriptor_to_dict("a" * 10000 + ": " + "b" * 10000) From cb8bca942e03dd1eff881eee27807fb7577e17dc Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 30 Jan 2025 01:51:29 +0100 Subject: [PATCH 13/46] notes --- .../licenses/_itis_vip_models.py | 27 ++++++++++++------- .../04/licenses/test_itis_vip_service.py | 12 +++++---- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py index af752ce159c..57c542d5a78 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py @@ -1,21 +1,25 @@ import re from typing import Annotated, Any, Literal -from pydantic import BaseModel, BeforeValidator, Field, HttpUrl +from pydantic import ( + BaseModel, + BeforeValidator, + Field, + HttpUrl, + StringConstraints, + TypeAdapter, +) -_MAX_LENGTH = 1_000 +_max_str_adapter = TypeAdapter( + Annotated[str, StringConstraints(strip_whitespace=True, max_length=1_000)] +) def _feature_descriptor_to_dict(descriptor: str) -> dict[str, Any]: # NOTE: this is manually added in the server side so be more robust to errors - # Safe against polynomial runtime vulnerability due to backtracking - if (size := len(descriptor)) and size > _MAX_LENGTH: - msg = f"Features field too long [{size=}]" - raise ValueError(msg) - + descriptor = _max_str_adapter.validate_python(descriptor.strip("{}")) pattern = r"(\w{1,100}): ([^,]{1,100})" - - matches = re.findall(pattern, descriptor.strip("{}")) + matches = re.findall(pattern, descriptor) return dict(matches) @@ -38,5 +42,8 @@ class AvailableDownload(BaseModel): class ResponseData(BaseModel): msg: int | None = None # still not used available_downloads: Annotated[ - list[AvailableDownload], Field(alias="availableDownloads") + list[AvailableDownload], + Field(alias="availableDownloads") + # TODO: consider just parsing those that you are interested in instead of performing so many checks + # and then throw them ] diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index fb7e4277bf8..a53150438e0 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -22,6 +22,13 @@ from simcore_service_webserver.licenses._itis_vip_settings import ItisVipSettings +def test_pre_validator_feature_descriptor_to_dict(): + # Makes sure the regex used here, which is vulnerable to polynomial runtime due to backtracking, cannot lead to denial of service. + with pytest.raises(ValidationError) as err_info: + _feature_descriptor_to_dict("a" * 10000 + ": " + "b" * 10000) + assert err_info.value.errors()[0]["type"] == "string_too_long" + + @pytest.fixture(scope="session") def fake_api_base_url() -> str: return "https://testserver" @@ -111,8 +118,3 @@ async def test_get_category_items( items = await _itis_vip_service.get_category_items(client, url) assert items[0].features["functionality"] == "Posable" - - -def test_pre_validator_feature_descriptor_to_dict(): - with pytest.raises(ValueError): - _feature_descriptor_to_dict("a" * 10000 + ": " + "b" * 10000) From 992906d56d606ea283c281ed65ac4b98db2162aa Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 30 Jan 2025 01:52:41 +0100 Subject: [PATCH 14/46] notes --- .../src/simcore_service_webserver/licenses/settings.py | 8 +++++++- .../unit/with_dbs/04/licenses/test_itis_vip_service.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/settings.py b/services/web/server/src/simcore_service_webserver/licenses/settings.py index a5c8355e58f..902a058ee80 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/settings.py +++ b/services/web/server/src/simcore_service_webserver/licenses/settings.py @@ -1,3 +1,4 @@ +import datetime from typing import Annotated from pydantic import Field @@ -7,7 +8,12 @@ class LicensesSettings(BaseCustomSettings): + LICENSES_SYNCER_ENABLED: bool + LICENSES_SYNCER_PERIODICITY: datetime.timedelta + + # Registered licensed resources: LICENSES_ITIS_VIP: Annotated[ ItisVipSettings | None, Field(description="Settings for VIP license models") ] - # other licenses resources come here + + # other licensed resources come here ... diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index a53150438e0..d794b231f15 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -31,7 +31,7 @@ def test_pre_validator_feature_descriptor_to_dict(): @pytest.fixture(scope="session") def fake_api_base_url() -> str: - return "https://testserver" + return "https://testserver-itis-vip.xyz" @pytest.fixture From 6bdc0560347a43f595ec993ea547dd3834d0c20d Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 30 Jan 2025 16:16:03 +0100 Subject: [PATCH 15/46] updates db --- .../src/simcore_postgres_database/models/licensed_items.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py b/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py index bb613e1a342..feddae5fdfd 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py @@ -47,12 +47,6 @@ class LicensedResourceType(str, enum.Enum): nullable=True, doc="Resource metadata. Used for read-only purposes", ), - sa.Column( - "licensed_resource_data", - postgresql.JSONB, - nullable=True, - doc="Stores metadata related to this licensed resource. Used for read-only purposes", - ), sa.Column( "pricing_plan_id", sa.BigInteger, From d976a9d017bac037643e1b45e33d6395d9f93193 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 30 Jan 2025 16:17:23 +0100 Subject: [PATCH 16/46] fetch&create --- .../licenses/_licensed_items_repository.py | 10 ++-- .../04/licenses/test_itis_vip_service.py | 46 ++++++++++++++++++- .../test_licensed_items_repository.py | 2 +- .../04/licenses/test_licensed_items_rest.py | 4 +- 4 files changed, 54 insertions(+), 8 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py index 45e67629a09..e8c1d8aceec 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py @@ -5,7 +5,7 @@ """ import logging -from typing import Literal, cast +from typing import Any, Literal, cast from aiohttp import web from models_library.licensed_items import ( @@ -43,8 +43,10 @@ async def create( *, licensed_resource_name: str, licensed_resource_type: LicensedResourceType, - product_name: ProductName | None, - pricing_plan_id: PricingPlanId | None, + licensed_resource_data: dict[str, Any] | None = None, + licensed_key: str | None = None, + product_name: ProductName | None = None, + pricing_plan_id: PricingPlanId | None = None, ) -> LicensedItemDB: async with transaction_context(get_asyncpg_engine(app), connection) as conn: result = await conn.execute( @@ -52,6 +54,8 @@ async def create( .values( licensed_resource_name=licensed_resource_name, licensed_resource_type=licensed_resource_type, + licensed_resource_data=licensed_resource_data, + licensed_key=licensed_key, pricing_plan_id=pricing_plan_id, product_name=product_name, created=func.now(), diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index d794b231f15..635c3274664 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -8,14 +8,20 @@ import pytest import respx +from aiohttp.test_utils import TestClient from faker import Faker from httpx import AsyncClient +from models_library.licensed_items import LicensedResourceType from pydantic import ValidationError from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.aiohttp import status -from simcore_service_webserver.licenses import _itis_vip_service +from simcore_service_webserver.licenses import ( + _itis_vip_service, + _licensed_items_repository, +) from simcore_service_webserver.licenses._itis_vip_models import ( + AvailableDownload, _feature_descriptor_to_dict, ) from simcore_service_webserver.licenses._itis_vip_service import ResponseData @@ -44,7 +50,7 @@ def app_environment( monkeypatch, { "ITIS_VIP_API_URL": f"{fake_api_base_url}/PD_DirectDownload/getDownloadableItems/{{category}}", - "ITIS_VIP_CATEGORIES": '["ComputationalPantom","Foo","Bar"]', # NOTE: ItisVipSettings will decode with json.dumps() + "ITIS_VIP_CATEGORIES": '["ComputationalPantom","FooCategory","BarCategory"]', # NOTE: ItisVipSettings will decode with json.dumps() }, ) @@ -118,3 +124,39 @@ async def test_get_category_items( items = await _itis_vip_service.get_category_items(client, url) assert items[0].features["functionality"] == "Posable" + + +async def test_sync_itis_vip_as_licensed_items( + mock_itis_vip_downloadables_api: respx.MockRouter, + app_environment: EnvVarsDict, + client: TestClient, +): + assert client.app + + settings = ItisVipSettings.create_from_envs() + assert settings.ITIS_VIP_CATEGORIES + + async with AsyncClient() as http_client: + for url, category in zip( + settings.get_urls(), settings.ITIS_VIP_CATEGORIES, strict=True + ): + assert f"{url}".endswith(category) + + items: list[AvailableDownload] = await _itis_vip_service.get_category_items( + http_client, url + ) + assert items[0].features["functionality"] == "Posable" + + for item in items: + # TODO: how to update to minimize collisions? one by one? + await _licensed_items_repository.create( + client.app, + licensed_resource_name=f"{category}/{item.id}", + licensed_resource_type=LicensedResourceType.VIP_MODEL, + licensed_resource_data=item.model_dump( + mode="json", exclude_unset=True + ), + licensed_key=item.license_key, + product_name=None, + pricing_plan_id=None, + ) diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_repository.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_repository.py index 7e3a0f4018e..3c2e88a87f6 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_repository.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_repository.py @@ -41,9 +41,9 @@ async def test_licensed_items_db_crud( got = await _licensed_items_repository.create( client.app, - product_name=osparc_product_name, licensed_resource_name="Model A", licensed_resource_type=LicensedResourceType.VIP_MODEL, + product_name=osparc_product_name, pricing_plan_id=pricing_plan_id, ) licensed_item_id = got.licensed_item_id diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_rest.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_rest.py index 491e340bd2f..19bff853b76 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_rest.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_rest.py @@ -41,10 +41,10 @@ async def test_licensed_items_listing( licensed_item_db = await _licensed_items_repository.create( client.app, - product_name=osparc_product_name, licensed_resource_name="Model A", licensed_resource_type=LicensedResourceType.VIP_MODEL, pricing_plan_id=pricing_plan_id, + product_name=osparc_product_name, ) _licensed_item_id = licensed_item_db.licensed_item_id @@ -108,10 +108,10 @@ async def test_licensed_items_purchase( licensed_item_db = await _licensed_items_repository.create( client.app, - product_name=osparc_product_name, licensed_resource_name="Model A", licensed_resource_type=LicensedResourceType.VIP_MODEL, pricing_plan_id=pricing_plan_id, + product_name=osparc_product_name, ) _licensed_item_id = licensed_item_db.licensed_item_id From 77be4fc76a8cf234347112b7f7b7cca16da50611 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 30 Jan 2025 16:49:50 +0100 Subject: [PATCH 17/46] test create --- .../licenses/_licensed_items_repository.py | 31 +++++----- .../licenses/_licensed_items_service.py | 58 ++++++++++++++++++- .../04/licenses/test_itis_vip_service.py | 16 ++--- 3 files changed, 78 insertions(+), 27 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py index e8c1d8aceec..94e7bc0429c 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py @@ -44,25 +44,26 @@ async def create( licensed_resource_name: str, licensed_resource_type: LicensedResourceType, licensed_resource_data: dict[str, Any] | None = None, - licensed_key: str | None = None, + license_key: str | None = None, product_name: ProductName | None = None, pricing_plan_id: PricingPlanId | None = None, ) -> LicensedItemDB: - async with transaction_context(get_asyncpg_engine(app), connection) as conn: - result = await conn.execute( - licensed_items.insert() - .values( - licensed_resource_name=licensed_resource_name, - licensed_resource_type=licensed_resource_type, - licensed_resource_data=licensed_resource_data, - licensed_key=licensed_key, - pricing_plan_id=pricing_plan_id, - product_name=product_name, - created=func.now(), - modified=func.now(), - ) - .returning(*_SELECTION_ARGS) + query = ( + licensed_items.insert() + .values( + licensed_resource_name=licensed_resource_name, + licensed_resource_type=licensed_resource_type, + licensed_resource_data=licensed_resource_data, + license_key=license_key, + pricing_plan_id=pricing_plan_id, + product_name=product_name, + created=func.now(), + modified=func.now(), ) + .returning(*_SELECTION_ARGS) + ) + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.execute(query) row = result.one() return LicensedItemDB.model_validate(row) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py index ec748259385..d248d48ba30 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py @@ -8,14 +8,19 @@ LicensedItemGet, LicensedItemGetPage, ) -from models_library.licensed_items import LicensedItemID +from models_library.licensed_items import ( + LicensedItemDB, + LicensedItemID, + LicensedItemUpdateDB, + LicensedResourceType, +) from models_library.products import ProductName from models_library.resource_tracker_licensed_items_purchases import ( LicensedItemsPurchasesCreate, ) from models_library.rest_ordering import OrderBy from models_library.users import UserID -from pydantic import NonNegativeInt +from pydantic import BaseModel, NonNegativeInt from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker import ( licensed_items_purchases, ) @@ -32,6 +37,27 @@ _logger = logging.getLogger(__name__) +async def create_licensed_item_from_resource( + app: web.Application, + *, + licensed_resource_name: str, + licensed_resource_type: LicensedResourceType, + licensed_resource_data: BaseModel, + license_key: str | None, +) -> LicensedItemDB: + return await _licensed_items_repository.create( + app, + licensed_resource_name=licensed_resource_name, + licensed_resource_type=licensed_resource_type, + licensed_resource_data=licensed_resource_data.model_dump( + mode="json", exclude_unset=True + ), + license_key=license_key, + product_name=None, + pricing_plan_id=None, + ) + + async def get_licensed_item( app: web.Application, *, @@ -71,6 +97,34 @@ async def list_licensed_items( ) +async def trash_licensed_item( + app: web.Application, + *, + product_name: ProductName, + licensed_item_id: LicensedItemID, +): + await _licensed_items_repository.update( + app, + product_name=product_name, + licensed_item_id=licensed_item_id, + updates=LicensedItemUpdateDB(trash=True), + ) + + +async def untrash_licensed_item( + app: web.Application, + *, + product_name: ProductName, + licensed_item_id: LicensedItemID, +): + await _licensed_items_repository.update( + app, + product_name=product_name, + licensed_item_id=licensed_item_id, + updates=LicensedItemUpdateDB(trash=True), + ) + + async def purchase_licensed_item( app: web.Application, *, diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index 635c3274664..54e74e1fccb 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -18,7 +18,7 @@ from servicelib.aiohttp import status from simcore_service_webserver.licenses import ( _itis_vip_service, - _licensed_items_repository, + _licensed_items_service, ) from simcore_service_webserver.licenses._itis_vip_models import ( AvailableDownload, @@ -147,16 +147,12 @@ async def test_sync_itis_vip_as_licensed_items( ) assert items[0].features["functionality"] == "Posable" - for item in items: + for vip_item in items: # TODO: how to update to minimize collisions? one by one? - await _licensed_items_repository.create( + await _licensed_items_service.create_licensed_item_from_resource( client.app, - licensed_resource_name=f"{category}/{item.id}", + licensed_resource_name=f"{category}/{vip_item.id}", licensed_resource_type=LicensedResourceType.VIP_MODEL, - licensed_resource_data=item.model_dump( - mode="json", exclude_unset=True - ), - licensed_key=item.license_key, - product_name=None, - pricing_plan_id=None, + licensed_resource_data=vip_item, + license_key=vip_item.license_key, ) From ae278f007b0b88547127a908401da2e647a351ae Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 30 Jan 2025 18:00:31 +0100 Subject: [PATCH 18/46] if not exists --- .../licenses/_licensed_items_repository.py | 70 ++++++++++++++++--- .../licenses/_licensed_items_service.py | 2 +- 2 files changed, 61 insertions(+), 11 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py index 94e7bc0429c..a4e90297e24 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py @@ -25,6 +25,7 @@ transaction_context, ) from sqlalchemy import asc, desc, func +from sqlalchemy.dialects import postgresql from sqlalchemy.ext.asyncio import AsyncConnection from sqlalchemy.sql import select @@ -37,19 +38,18 @@ _SELECTION_ARGS = get_columns_from_db_model(licensed_items, LicensedItemDB) -async def create( - app: web.Application, - connection: AsyncConnection | None = None, - *, +def _build_insert_query( licensed_resource_name: str, licensed_resource_type: LicensedResourceType, - licensed_resource_data: dict[str, Any] | None = None, - license_key: str | None = None, - product_name: ProductName | None = None, - pricing_plan_id: PricingPlanId | None = None, -) -> LicensedItemDB: + licensed_resource_data: dict[str, Any] | None, + license_key: str | None, + product_name: ProductName | None, + pricing_plan_id: PricingPlanId | None, + *, + on_conflict_do_nothing: bool = False, +) -> postgresql.Insert: query = ( - licensed_items.insert() + postgresql.insert(licensed_items) .values( licensed_resource_name=licensed_resource_name, licensed_resource_type=licensed_resource_type, @@ -62,12 +62,62 @@ async def create( ) .returning(*_SELECTION_ARGS) ) + if on_conflict_do_nothing: + query = query.on_conflict_do_nothing() + return query + + +async def create( + app: web.Application, + connection: AsyncConnection | None = None, + *, + licensed_resource_name: str, + licensed_resource_type: LicensedResourceType, + licensed_resource_data: dict[str, Any] | None = None, + license_key: str | None = None, + product_name: ProductName | None = None, + pricing_plan_id: PricingPlanId | None = None, +) -> LicensedItemDB: + query = _build_insert_query( + licensed_resource_name, + licensed_resource_type, + licensed_resource_data, + license_key, + product_name, + pricing_plan_id, + ) async with transaction_context(get_asyncpg_engine(app), connection) as conn: result = await conn.execute(query) row = result.one() return LicensedItemDB.model_validate(row) +async def create_if_not_exists( + app: web.Application, + connection: AsyncConnection | None = None, + *, + licensed_resource_name: str, + licensed_resource_type: LicensedResourceType, + licensed_resource_data: dict[str, Any] | None = None, + license_key: str | None = None, + product_name: ProductName | None = None, + pricing_plan_id: PricingPlanId | None = None, +) -> LicensedItemDB: + query = _build_insert_query( + licensed_resource_name, + licensed_resource_type, + licensed_resource_data, + license_key, + product_name, + pricing_plan_id, + on_conflict_do_nothing=True, + ) + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.execute(query) + row = result.one_or_none() + return LicensedItemDB.model_validate(row) + + async def list_( app: web.Application, connection: AsyncConnection | None = None, diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py index d248d48ba30..53848d5d0a2 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py @@ -45,7 +45,7 @@ async def create_licensed_item_from_resource( licensed_resource_data: BaseModel, license_key: str | None, ) -> LicensedItemDB: - return await _licensed_items_repository.create( + return await _licensed_items_repository.create_if_not_exists( app, licensed_resource_name=licensed_resource_name, licensed_resource_type=licensed_resource_type, From cfc6caab6c76ab67996e171a3e52a403679afc12 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 30 Jan 2025 19:27:41 +0100 Subject: [PATCH 19/46] cleanup --- .../licenses/_licensed_items_repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py index a4e90297e24..4d691badb94 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py @@ -195,8 +195,8 @@ async def get( ) async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: - result = await conn.stream(base_query) - row = await result.first() + result = await conn.execute(base_query) + row = result.one_or_none() if row is None: raise LicensedItemNotFoundError(licensed_item_id=licensed_item_id) return LicensedItemDB.model_validate(row) From fdae0a9b3ddeb07f5d2978a6fbefa6f12983a95b Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 30 Jan 2025 19:41:34 +0100 Subject: [PATCH 20/46] constrinat --- .../src/simcore_postgres_database/models/licensed_items.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py b/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py index feddae5fdfd..8a672fc5e7d 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py @@ -80,4 +80,9 @@ class LicensedResourceType(str, enum.Enum): column_created_datetime(timezone=True), column_modified_datetime(timezone=True), column_trashed_datetime("licensed_item"), + sa.UniqueConstraint( + "licensed_resource_name", + "licensed_resource_type", + name="uq_licensed_resource_name_type", + ), ) From 326de711e693d771836505ce82b7e383721e9d07 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 30 Jan 2025 19:42:44 +0100 Subject: [PATCH 21/46] migration: --- ..._add_uniqu_constraint_in_licensed_items.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/e71ea59858f4_add_uniqu_constraint_in_licensed_items.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/e71ea59858f4_add_uniqu_constraint_in_licensed_items.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/e71ea59858f4_add_uniqu_constraint_in_licensed_items.py new file mode 100644 index 00000000000..9772cd0b2be --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/e71ea59858f4_add_uniqu_constraint_in_licensed_items.py @@ -0,0 +1,32 @@ +"""add uniqu constraint in licensed_items + +Revision ID: e71ea59858f4 +Revises: 4f31760a63ba +Create Date: 2025-01-30 18:42:15.192968+00:00 + +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = "e71ea59858f4" +down_revision = "4f31760a63ba" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_unique_constraint( + "uq_licensed_resource_name_type", + "licensed_items", + ["licensed_resource_name", "licensed_resource_type"], + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + "uq_licensed_resource_name_type", "licensed_items", type_="unique" + ) + # ### end Alembic commands ### From 684e28f13a67807bac34a37ff9a115ff8658c6cd Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 30 Jan 2025 20:30:04 +0100 Subject: [PATCH 22/46] registration --- .../licenses/_licensed_items_repository.py | 103 +++++++++++------- .../licenses/_licensed_items_service.py | 73 +++++++++++-- .../04/licenses/test_itis_vip_service.py | 27 ++++- 3 files changed, 143 insertions(+), 60 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py index 4d691badb94..2eed5cd9bf2 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py @@ -38,16 +38,18 @@ _SELECTION_ARGS = get_columns_from_db_model(licensed_items, LicensedItemDB) -def _build_insert_query( +async def create( + app: web.Application, + connection: AsyncConnection | None = None, + *, licensed_resource_name: str, licensed_resource_type: LicensedResourceType, - licensed_resource_data: dict[str, Any] | None, - license_key: str | None, - product_name: ProductName | None, - pricing_plan_id: PricingPlanId | None, - *, - on_conflict_do_nothing: bool = False, -) -> postgresql.Insert: + licensed_resource_data: dict[str, Any] | None = None, + license_key: str | None = None, + product_name: ProductName | None = None, + pricing_plan_id: PricingPlanId | None = None, +) -> LicensedItemDB: + query = ( postgresql.insert(licensed_items) .values( @@ -62,30 +64,7 @@ def _build_insert_query( ) .returning(*_SELECTION_ARGS) ) - if on_conflict_do_nothing: - query = query.on_conflict_do_nothing() - return query - -async def create( - app: web.Application, - connection: AsyncConnection | None = None, - *, - licensed_resource_name: str, - licensed_resource_type: LicensedResourceType, - licensed_resource_data: dict[str, Any] | None = None, - license_key: str | None = None, - product_name: ProductName | None = None, - pricing_plan_id: PricingPlanId | None = None, -) -> LicensedItemDB: - query = _build_insert_query( - licensed_resource_name, - licensed_resource_type, - licensed_resource_data, - license_key, - product_name, - pricing_plan_id, - ) async with transaction_context(get_asyncpg_engine(app), connection) as conn: result = await conn.execute(query) row = result.one() @@ -103,18 +82,34 @@ async def create_if_not_exists( product_name: ProductName | None = None, pricing_plan_id: PricingPlanId | None = None, ) -> LicensedItemDB: - query = _build_insert_query( - licensed_resource_name, - licensed_resource_type, - licensed_resource_data, - license_key, - product_name, - pricing_plan_id, - on_conflict_do_nothing=True, + insert_query_or_none_query = ( + postgresql.insert(licensed_items) + .values( + licensed_resource_name=licensed_resource_name, + licensed_resource_type=licensed_resource_type, + licensed_resource_data=licensed_resource_data, + license_key=license_key, + pricing_plan_id=pricing_plan_id, + product_name=product_name, + created=func.now(), + modified=func.now(), + ) + .on_conflict_do_nothing() + .returning(*_SELECTION_ARGS) ) + async with transaction_context(get_asyncpg_engine(app), connection) as conn: - result = await conn.execute(query) + result = await conn.execute(insert_query_or_none_query) row = result.one_or_none() + if row is None: + select_query = select(*_SELECTION_ARGS).where( + (licensed_items.c.licensed_resource_name == licensed_resource_name) + & (licensed_items.c.licensed_resource_type == licensed_resource_type) + ) + result = await conn.execute(select_query) + row = result.one() + + assert row is not None # nosec return LicensedItemDB.model_validate(row) @@ -185,7 +180,7 @@ async def get( licensed_item_id: LicensedItemID, product_name: ProductName, ) -> LicensedItemDB: - base_query = ( + select_query = ( select(*_SELECTION_ARGS) .select_from(licensed_items) .where( @@ -195,13 +190,37 @@ async def get( ) async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: - result = await conn.execute(base_query) + result = await conn.execute(select_query) row = result.one_or_none() if row is None: raise LicensedItemNotFoundError(licensed_item_id=licensed_item_id) return LicensedItemDB.model_validate(row) +async def get_by_resource_identifier( + app: web.Application, + connection: AsyncConnection | None = None, + *, + licensed_resource_name: str, + licensed_resource_type: LicensedResourceType, +) -> LicensedItemDB: + select_query = select(*_SELECTION_ARGS).where( + (licensed_items.c.licensed_resource_name == licensed_resource_name) + & (licensed_items.c.licensed_resource_type == licensed_resource_type) + ) + + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + result = await conn.execute(select_query) + row = result.one_or_none() + if row is None: + raise LicensedItemNotFoundError( + licensed_item_id="Unkown", + licensed_resource_name=licensed_resource_name, + licensed_resource_type=licensed_resource_type, + ) + return LicensedItemDB.model_validate(row) + + async def update( app: web.Application, connection: AsyncConnection | None = None, diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py index 53848d5d0a2..e66fb5a1d58 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py @@ -32,12 +32,24 @@ from ..wallets.errors import WalletNotEnoughCreditsError from . import _licensed_items_repository from ._common.models import LicensedItemsBodyParams -from .errors import LicensedItemPricingPlanMatchError +from .errors import LicensedItemNotFoundError, LicensedItemPricingPlanMatchError _logger = logging.getLogger(__name__) -async def create_licensed_item_from_resource( +def _compute_difference(old_data: dict, new_data: dict): + differences = { + k: {"old": old_data[k], "new": new_data[k]} + for k in old_data + if old_data[k] != new_data.get(k) + } + differences.update( + {k: {"old": None, "new": new_data[k]} for k in new_data if k not in old_data} + ) + return differences + + +async def register_licensed_item_from_resource( app: web.Application, *, licensed_resource_name: str, @@ -45,17 +57,54 @@ async def create_licensed_item_from_resource( licensed_resource_data: BaseModel, license_key: str | None, ) -> LicensedItemDB: - return await _licensed_items_repository.create_if_not_exists( - app, - licensed_resource_name=licensed_resource_name, - licensed_resource_type=licensed_resource_type, - licensed_resource_data=licensed_resource_data.model_dump( + + try: + license_item = await _licensed_items_repository.get_by_resource_identifier( + app, + licensed_resource_name=licensed_resource_name, + licensed_resource_type=licensed_resource_type, + ) + + if license_item.licensed_resource_data != licensed_resource_data.model_dump( mode="json", exclude_unset=True - ), - license_key=license_key, - product_name=None, - pricing_plan_id=None, - ) + ): + differences = _compute_difference( + license_item.licensed_resource_data or {}, + licensed_resource_data.model_dump(mode="json", exclude_unset=True), + ) + _logger.warning( + "CHANGES: NEEDED for %s, %s: Resource differs from the one registered: %s", + licensed_resource_name, + licensed_resource_type, + differences, + ) + else: + _logger.info( + "Resource %s, %s already registered", + licensed_resource_name, + licensed_resource_type, + ) + + except LicensedItemNotFoundError: + license_item = await _licensed_items_repository.create_if_not_exists( + app, + licensed_resource_name=licensed_resource_name, + licensed_resource_type=licensed_resource_type, + licensed_resource_data=licensed_resource_data.model_dump( + mode="json", exclude_unset=True + ), + license_key=license_key, + product_name=None, + pricing_plan_id=None, + ) + + _logger.info( + "NEW license with resource %s, %s already registered", + licensed_resource_name, + licensed_resource_type, + ) + + return license_item async def get_licensed_item( diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index 54e74e1fccb..7fa0a1dc5ab 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -149,10 +149,25 @@ async def test_sync_itis_vip_as_licensed_items( for vip_item in items: # TODO: how to update to minimize collisions? one by one? - await _licensed_items_service.create_licensed_item_from_resource( - client.app, - licensed_resource_name=f"{category}/{vip_item.id}", - licensed_resource_type=LicensedResourceType.VIP_MODEL, - licensed_resource_data=vip_item, - license_key=vip_item.license_key, + + got1 = ( + await _licensed_items_service.register_licensed_item_from_resource( + client.app, + licensed_resource_name=f"{category}/{vip_item.id}", + licensed_resource_type=LicensedResourceType.VIP_MODEL, + licensed_resource_data=vip_item, + license_key=vip_item.license_key, + ) + ) + + got2 = ( + await _licensed_items_service.register_licensed_item_from_resource( + client.app, + licensed_resource_name=f"{category}/{vip_item.id}", + licensed_resource_type=LicensedResourceType.VIP_MODEL, + licensed_resource_data=vip_item, + license_key=vip_item.license_key, + ) ) + + assert got1 == got2 From 894505a8eb04c06b111c508079e028c9b202b5a0 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 30 Jan 2025 20:55:04 +0100 Subject: [PATCH 23/46] states --- .../licenses/_licensed_items_service.py | 44 ++++++----- .../04/licenses/test_itis_vip_service.py | 74 ++++++++++++++----- 2 files changed, 80 insertions(+), 38 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py index e66fb5a1d58..17e87caaef3 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py @@ -2,6 +2,8 @@ import logging from datetime import UTC, datetime, timedelta +from enum import Enum, auto +from pprint import pformat from aiohttp import web from models_library.api_schemas_webserver.licensed_items import ( @@ -37,6 +39,12 @@ _logger = logging.getLogger(__name__) +class RegistrationState(Enum): + ALREADY_REGISTERED = auto() + DIFFERENT_RESOURCE = auto() + NEWLY_REGISTERED = auto() + + def _compute_difference(old_data: dict, new_data: dict): differences = { k: {"old": old_data[k], "new": new_data[k]} @@ -49,44 +57,45 @@ def _compute_difference(old_data: dict, new_data: dict): return differences -async def register_licensed_item_from_resource( +async def register_resource_as_licensed_item( app: web.Application, *, licensed_resource_name: str, licensed_resource_type: LicensedResourceType, licensed_resource_data: BaseModel, license_key: str | None, -) -> LicensedItemDB: - +) -> tuple[LicensedItemDB, RegistrationState]: try: - license_item = await _licensed_items_repository.get_by_resource_identifier( + licensed_item = await _licensed_items_repository.get_by_resource_identifier( app, licensed_resource_name=licensed_resource_name, licensed_resource_type=licensed_resource_type, ) - if license_item.licensed_resource_data != licensed_resource_data.model_dump( + if licensed_item.licensed_resource_data != licensed_resource_data.model_dump( mode="json", exclude_unset=True ): differences = _compute_difference( - license_item.licensed_resource_data or {}, + licensed_item.licensed_resource_data or {}, licensed_resource_data.model_dump(mode="json", exclude_unset=True), ) _logger.warning( - "CHANGES: NEEDED for %s, %s: Resource differs from the one registered: %s", - licensed_resource_name, - licensed_resource_type, - differences, - ) - else: - _logger.info( - "Resource %s, %s already registered", + "DIFFERENT_RESOURCE: %s, %s. Difference: %s", licensed_resource_name, licensed_resource_type, + pformat(differences), ) + return licensed_item, RegistrationState.DIFFERENT_RESOURCE + + _logger.info( + "ALREADY_REGISTERED: %s, %s", + licensed_resource_name, + licensed_resource_type, + ) + return licensed_item, RegistrationState.ALREADY_REGISTERED except LicensedItemNotFoundError: - license_item = await _licensed_items_repository.create_if_not_exists( + licensed_item = await _licensed_items_repository.create_if_not_exists( app, licensed_resource_name=licensed_resource_name, licensed_resource_type=licensed_resource_type, @@ -99,12 +108,11 @@ async def register_licensed_item_from_resource( ) _logger.info( - "NEW license with resource %s, %s already registered", + "NEWLY_REGISTERED: %s, %s", licensed_resource_name, licensed_resource_type, ) - - return license_item + return licensed_item, RegistrationState.NEWLY_REGISTERED async def get_licensed_item( diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index 7fa0a1dc5ab..b76add2fa9d 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -63,7 +63,7 @@ def mock_itis_vip_downloadables_api( "msg": 0, "availableDownloads": [ { - "ID": faker.random_int(min=70, max=90), + "ID": i, "Description": faker.sentence(), "Thumbnail": faker.image_url(), # NOTE: this is manually added in the server side so be more robust to errors @@ -74,7 +74,7 @@ def mock_itis_vip_downloadables_api( "Protection": faker.random_element(elements=["Code", "PayPal"]), "AvailableFromURL": faker.random_element(elements=[None, faker.url()]), } - for _ in range(8) + for i in range(8) ], } @@ -148,26 +148,60 @@ async def test_sync_itis_vip_as_licensed_items( assert items[0].features["functionality"] == "Posable" for vip_item in items: - # TODO: how to update to minimize collisions? one by one? - - got1 = ( - await _licensed_items_service.register_licensed_item_from_resource( - client.app, - licensed_resource_name=f"{category}/{vip_item.id}", - licensed_resource_type=LicensedResourceType.VIP_MODEL, - licensed_resource_data=vip_item, - license_key=vip_item.license_key, - ) + ( + got1, + state1, + ) = await _licensed_items_service.register_resource_as_licensed_item( + client.app, + licensed_resource_name=f"{category}/{vip_item.id}", + licensed_resource_type=LicensedResourceType.VIP_MODEL, + licensed_resource_data=vip_item, + license_key=vip_item.license_key, + ) + assert ( + state1 == _licensed_items_service.RegistrationState.NEWLY_REGISTERED ) - got2 = ( - await _licensed_items_service.register_licensed_item_from_resource( - client.app, - licensed_resource_name=f"{category}/{vip_item.id}", - licensed_resource_type=LicensedResourceType.VIP_MODEL, - licensed_resource_data=vip_item, - license_key=vip_item.license_key, - ) + ( + got2, + state2, + ) = await _licensed_items_service.register_resource_as_licensed_item( + client.app, + licensed_resource_name=f"{category}/{vip_item.id}", + licensed_resource_type=LicensedResourceType.VIP_MODEL, + licensed_resource_data=vip_item, + license_key=vip_item.license_key, ) + assert ( + state2 + == _licensed_items_service.RegistrationState.ALREADY_REGISTERED + ) assert got1 == got2 + + # Modify vip_item and register again + vip_item_modified = vip_item.model_copy( + update={ + "features": { + **vip_item.features, + "functionality": "Non-Posable", + } + } + ) + ( + got3, + state3, + ) = await _licensed_items_service.register_resource_as_licensed_item( + client.app, + licensed_resource_name=f"{category}/{vip_item.id}", + licensed_resource_type=LicensedResourceType.VIP_MODEL, + licensed_resource_data=vip_item_modified, + license_key=vip_item.license_key, + ) + + assert ( + state3 + == _licensed_items_service.RegistrationState.DIFFERENT_RESOURCE + ) + # not stored! + assert got2 == got3 From 54c69f9e42a88a729287052b2f65f9bd8a75a816 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 30 Jan 2025 21:03:30 +0100 Subject: [PATCH 24/46] cleanup --- .../licenses/_licensed_items_repository.py | 8 ++-- .../04/licenses/test_itis_vip_service.py | 38 ++++++++----------- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py index 2eed5cd9bf2..8544baf82bb 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py @@ -82,7 +82,7 @@ async def create_if_not_exists( product_name: ProductName | None = None, pricing_plan_id: PricingPlanId | None = None, ) -> LicensedItemDB: - insert_query_or_none_query = ( + insert_or_none_query = ( postgresql.insert(licensed_items) .values( licensed_resource_name=licensed_resource_name, @@ -94,18 +94,20 @@ async def create_if_not_exists( created=func.now(), modified=func.now(), ) - .on_conflict_do_nothing() .returning(*_SELECTION_ARGS) + .on_conflict_do_nothing() ) async with transaction_context(get_asyncpg_engine(app), connection) as conn: - result = await conn.execute(insert_query_or_none_query) + result = await conn.execute(insert_or_none_query) row = result.one_or_none() + if row is None: select_query = select(*_SELECTION_ARGS).where( (licensed_items.c.licensed_resource_name == licensed_resource_name) & (licensed_items.c.licensed_resource_type == licensed_resource_type) ) + result = await conn.execute(select_query) row = result.one() diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index b76add2fa9d..8a99c71fb92 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -26,6 +26,7 @@ ) from simcore_service_webserver.licenses._itis_vip_service import ResponseData from simcore_service_webserver.licenses._itis_vip_settings import ItisVipSettings +from simcore_service_webserver.licenses._licensed_items_service import RegistrationState def test_pre_validator_feature_descriptor_to_dict(): @@ -148,6 +149,8 @@ async def test_sync_itis_vip_as_licensed_items( assert items[0].features["functionality"] == "Posable" for vip_item in items: + + # register a NEW resource ( got1, state1, @@ -158,10 +161,9 @@ async def test_sync_itis_vip_as_licensed_items( licensed_resource_data=vip_item, license_key=vip_item.license_key, ) - assert ( - state1 == _licensed_items_service.RegistrationState.NEWLY_REGISTERED - ) + assert state1 == RegistrationState.NEWLY_REGISTERED + # register the SAME resource ( got2, state2, @@ -173,21 +175,10 @@ async def test_sync_itis_vip_as_licensed_items( license_key=vip_item.license_key, ) - assert ( - state2 - == _licensed_items_service.RegistrationState.ALREADY_REGISTERED - ) + assert state2 == RegistrationState.ALREADY_REGISTERED assert got1 == got2 - # Modify vip_item and register again - vip_item_modified = vip_item.model_copy( - update={ - "features": { - **vip_item.features, - "functionality": "Non-Posable", - } - } - ) + # register a MODIFIED version of the same resource ( got3, state3, @@ -195,13 +186,16 @@ async def test_sync_itis_vip_as_licensed_items( client.app, licensed_resource_name=f"{category}/{vip_item.id}", licensed_resource_type=LicensedResourceType.VIP_MODEL, - licensed_resource_data=vip_item_modified, + licensed_resource_data=vip_item.model_copy( + update={ + "features": { + **vip_item.features, + "functionality": "Non-Posable", + } + } + ), license_key=vip_item.license_key, ) - assert ( - state3 - == _licensed_items_service.RegistrationState.DIFFERENT_RESOURCE - ) - # not stored! + assert state3 == RegistrationState.DIFFERENT_RESOURCE assert got2 == got3 From c016bb05345011c2ddf432c1661810c12e6e1456 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 30 Jan 2025 21:22:21 +0100 Subject: [PATCH 25/46] drafts ideas for rest entyrpoint --- .../licenses/_licensed_items_rest.py | 18 ++++++++++++++++++ .../security/_authz_access_roles.py | 3 ++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_rest.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_rest.py index ae45f5aba5e..8750413c170 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_rest.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_rest.py @@ -112,3 +112,21 @@ async def purchase_licensed_item(request: web.Request): body_params=body_params, ) return web.json_response(status=status.HTTP_204_NO_CONTENT) + + +@routes.post( + f"/{VTAG}/catalog/licensed-items/{{licensed_item_id}}:resync", + name="resync_licensed_item", +) +@login_required +@permission_required("catalog/licensed-items.admin") +@handle_plugin_requests_exceptions +async def resync_licensed_item(request: web.Request): + req_ctx = LicensedItemsRequestContext.model_validate(request) + + # TODO: forces resync. can schedule a run with a lock? + # fetches resources + # register resources + # returns issues + + return web.json_response(status=status.HTTP_202_ACCEPTED) diff --git a/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py b/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py index 0bd7e6a75eb..57a3aedd3f9 100644 --- a/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py +++ b/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py @@ -111,8 +111,9 @@ class PermissionDict(TypedDict, total=False): UserRole.ADMIN: PermissionDict( can=[ "admin.*", - "storage.files.sync", + "catalog/licensed-items.admin", "resource-usage.write", + "storage.files.sync", ], inherits=[UserRole.TESTER], ), From 60c7d05c0fd2729d51715ddcd71354b6e1b6bb3d Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 31 Jan 2025 10:20:03 +0100 Subject: [PATCH 26/46] cleanup --- .../04/licenses/test_itis_vip_service.py | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index 8a99c71fb92..dfa0aa2fe9b 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -143,59 +143,59 @@ async def test_sync_itis_vip_as_licensed_items( ): assert f"{url}".endswith(category) - items: list[AvailableDownload] = await _itis_vip_service.get_category_items( - http_client, url - ) - assert items[0].features["functionality"] == "Posable" + vip_resources: list[ + AvailableDownload + ] = await _itis_vip_service.get_category_items(http_client, url) + assert vip_resources[0].features["functionality"] == "Posable" - for vip_item in items: + for vip in vip_resources: # register a NEW resource ( - got1, + licensed_item1, state1, ) = await _licensed_items_service.register_resource_as_licensed_item( client.app, - licensed_resource_name=f"{category}/{vip_item.id}", + licensed_resource_name=f"{category}/{vip.id}", licensed_resource_type=LicensedResourceType.VIP_MODEL, - licensed_resource_data=vip_item, - license_key=vip_item.license_key, + licensed_resource_data=vip, + license_key=vip.license_key, ) assert state1 == RegistrationState.NEWLY_REGISTERED # register the SAME resource ( - got2, + licensed_item2, state2, ) = await _licensed_items_service.register_resource_as_licensed_item( client.app, - licensed_resource_name=f"{category}/{vip_item.id}", + licensed_resource_name=f"{category}/{vip.id}", licensed_resource_type=LicensedResourceType.VIP_MODEL, - licensed_resource_data=vip_item, - license_key=vip_item.license_key, + licensed_resource_data=vip, + license_key=vip.license_key, ) assert state2 == RegistrationState.ALREADY_REGISTERED - assert got1 == got2 + assert licensed_item1 == licensed_item2 # register a MODIFIED version of the same resource ( - got3, + licensed_item3, state3, ) = await _licensed_items_service.register_resource_as_licensed_item( client.app, - licensed_resource_name=f"{category}/{vip_item.id}", + licensed_resource_name=f"{category}/{vip.id}", licensed_resource_type=LicensedResourceType.VIP_MODEL, - licensed_resource_data=vip_item.model_copy( + licensed_resource_data=vip.model_copy( update={ "features": { - **vip_item.features, + **vip.features, "functionality": "Non-Posable", } } ), - license_key=vip_item.license_key, + license_key=vip.license_key, ) assert state3 == RegistrationState.DIFFERENT_RESOURCE - assert got2 == got3 + assert licensed_item2 == licensed_item3 From 325bb45192be72d57324a8751974c5491a53f161 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 31 Jan 2025 10:30:07 +0100 Subject: [PATCH 27/46] fixes migration --- .../e71ea59858f4_add_uniqu_constraint_in_licensed_items.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/e71ea59858f4_add_uniqu_constraint_in_licensed_items.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/e71ea59858f4_add_uniqu_constraint_in_licensed_items.py index 9772cd0b2be..3af7ff911f8 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/e71ea59858f4_add_uniqu_constraint_in_licensed_items.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/e71ea59858f4_add_uniqu_constraint_in_licensed_items.py @@ -1,7 +1,7 @@ """add uniqu constraint in licensed_items Revision ID: e71ea59858f4 -Revises: 4f31760a63ba +Revises: 7d1c6425a51d" Create Date: 2025-01-30 18:42:15.192968+00:00 """ @@ -9,7 +9,7 @@ # revision identifiers, used by Alembic. revision = "e71ea59858f4" -down_revision = "4f31760a63ba" +down_revision = "7d1c6425a51d" branch_labels = None depends_on = None From b70feb41b88fd439d03bd31a6eef974ad6a26bcc Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 31 Jan 2025 17:19:19 +0100 Subject: [PATCH 28/46] minor fix on try again --- .../service-library/src/servicelib/background_task.py | 9 +++++++-- .../licenses/_itis_vip_settings.py | 8 ++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/packages/service-library/src/servicelib/background_task.py b/packages/service-library/src/servicelib/background_task.py index 793d05b1f9b..feeb06ff475 100644 --- a/packages/service-library/src/servicelib/background_task.py +++ b/packages/service-library/src/servicelib/background_task.py @@ -60,6 +60,11 @@ def periodic( def _decorator( func: Callable[P, Coroutine[Any, Any, None]], ) -> Callable[P, Coroutine[Any, Any, None]]: + class _InternalTryAgain(TryAgain): + # Local exception to prevent reacting to similarTryAgain exceptions raised by the wrapped func + # e.g. when this decorators is used twice on the same function + ... + nap = ( asyncio.sleep if early_wake_up_event is None @@ -71,7 +76,7 @@ def _decorator( wait=wait_fixed(interval.total_seconds()), reraise=True, retry=( - retry_if_exception_type(TryAgain) + retry_if_exception_type(_InternalTryAgain) if raise_on_error else retry_if_exception_type() ), @@ -80,7 +85,7 @@ def _decorator( @functools.wraps(func) async def _wrapper(*args: P.args, **kwargs: P.kwargs) -> None: await func(*args, **kwargs) - raise TryAgain + raise _InternalTryAgain return _wrapper diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_settings.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_settings.py index 2fb2542c396..d1218835cdc 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_settings.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_settings.py @@ -1,5 +1,7 @@ from typing import Annotated +from common_library.pydantic_basic_types import ShortTruncatedStr +from models_library.basic_types import IDStr from pydantic import AfterValidator, HttpUrl from settings_library.base import BaseCustomSettings @@ -20,3 +22,9 @@ def get_urls(self) -> list[HttpUrl]: HttpUrl(self.ITIS_VIP_API_URL.format(category=category)) for category in self.ITIS_VIP_CATEGORIES ] + + +class ItisVipCategorySettings(BaseCustomSettings): + ITIS_VIP_CATEGORY_URL: HttpUrl + ITIS_VIP_CATEGORY_ID: IDStr # use same as in vip-api + ITIS_VIP_CATEGORY_DISPLAY_NAME: ShortTruncatedStr From 3665e10fa82aa97b481943156eca57b215debab2 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 31 Jan 2025 18:27:05 +0100 Subject: [PATCH 29/46] update model features --- .../licenses/_itis_vip_models.py | 42 +++++++++++++++---- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py index 57c542d5a78..5c5bcf9db7c 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py @@ -1,6 +1,7 @@ import re -from typing import Annotated, Any, Literal +from typing import Annotated, Any, Literal, NotRequired, TypedDict +from models_library.basic_types import IDStr from pydantic import ( BaseModel, BeforeValidator, @@ -23,12 +24,29 @@ def _feature_descriptor_to_dict(descriptor: str) -> dict[str, Any]: return dict(matches) -class AvailableDownload(BaseModel): +# +# ITIS-VIP API Schema +# + + +class FeaturesDict(TypedDict): + name: str + version: str + sex: NotRequired[str] + age: NotRequired[str] + weight: NotRequired[str] + height: NotRequired[str] + date: NotRequired[str] + ethnicity: NotRequired[str] + functionality: NotRequired[str] + + +class ItisVipData(BaseModel): id: Annotated[int, Field(alias="ID")] description: Annotated[str, Field(alias="Description")] thumbnail: Annotated[str, Field(alias="Thumbnail")] features: Annotated[ - dict[str, Any], + FeaturesDict, BeforeValidator(_feature_descriptor_to_dict), Field(alias="Features"), ] @@ -39,11 +57,17 @@ class AvailableDownload(BaseModel): available_from_url: Annotated[HttpUrl | None, Field(alias="AvailableFromURL")] -class ResponseData(BaseModel): +class ItisVipApiResponse(BaseModel): msg: int | None = None # still not used - available_downloads: Annotated[ - list[AvailableDownload], - Field(alias="availableDownloads") - # TODO: consider just parsing those that you are interested in instead of performing so many checks - # and then throw them + available_downloads: Annotated[list[ItisVipData], Field(alias="availableDownloads")] + + +# +# RESOURCE +# +class ItisVipResourceData(BaseModel): + category_id: IDStr + category_display: str + data: Annotated[ + ItisVipData, Field(description="Original published data in the api") ] From a5b3d0715d03f46764c342e9afa36aae6e4d6b38 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 31 Jan 2025 18:47:27 +0100 Subject: [PATCH 30/46] retry policy --- .../licenses/_itis_vip_service.py | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py index 306fd08c931..87b249f5b9d 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py @@ -1,20 +1,31 @@ import httpx from pydantic import HttpUrl +from tenacity import ( + retry, + retry_if_exception_cause_type, + stop_after_attempt, + wait_exponential, +) -from ._itis_vip_models import AvailableDownload, ResponseData +from ._itis_vip_models import ItisVipApiResponse, ItisVipData +@retry( + wait=wait_exponential(multiplier=1, min=4, max=10), + stop=stop_after_attempt(5), + retry=retry_if_exception_cause_type(httpx.RequestError), +) async def get_category_items( client: httpx.AsyncClient, url: HttpUrl -) -> list[AvailableDownload]: +) -> list[ItisVipData]: """ Raises: - Any https://www.python-httpx.org/exceptions/ - httpx.HTTPStatusCode: - + httpx.HTTPStatusError + pydantic.ValidationError """ response = await client.post(f"{url}") response.raise_for_status() - validated_data = ResponseData.model_validate(response.json()) + + validated_data = ItisVipApiResponse.model_validate(response.json()) return validated_data.available_downloads From f8c58789c74cb247beed0b5331fc778a1c860f65 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 31 Jan 2025 18:48:05 +0100 Subject: [PATCH 31/46] updates --- .../with_dbs/04/licenses/test_itis_vip_service.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index dfa0aa2fe9b..18c2b404fda 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -21,10 +21,10 @@ _licensed_items_service, ) from simcore_service_webserver.licenses._itis_vip_models import ( - AvailableDownload, + ItisVipData, _feature_descriptor_to_dict, ) -from simcore_service_webserver.licenses._itis_vip_service import ResponseData +from simcore_service_webserver.licenses._itis_vip_service import ItisVipApiResponse from simcore_service_webserver.licenses._itis_vip_settings import ItisVipSettings from simcore_service_webserver.licenses._licensed_items_service import RegistrationState @@ -95,7 +95,7 @@ async def test_fetch_and_validate_itis_vip_api( response_json = response.json() try: - validated_data = ResponseData(**response_json) + validated_data = ItisVipApiResponse(**response_json) except ValidationError as e: pytest.fail(f"Response validation failed: {e}") @@ -144,7 +144,7 @@ async def test_sync_itis_vip_as_licensed_items( assert f"{url}".endswith(category) vip_resources: list[ - AvailableDownload + ItisVipData ] = await _itis_vip_service.get_category_items(http_client, url) assert vip_resources[0].features["functionality"] == "Posable" @@ -159,7 +159,7 @@ async def test_sync_itis_vip_as_licensed_items( licensed_resource_name=f"{category}/{vip.id}", licensed_resource_type=LicensedResourceType.VIP_MODEL, licensed_resource_data=vip, - license_key=vip.license_key, + licensed_item_display_name=f"{vip.features.get('name','')}", ) assert state1 == RegistrationState.NEWLY_REGISTERED @@ -172,7 +172,7 @@ async def test_sync_itis_vip_as_licensed_items( licensed_resource_name=f"{category}/{vip.id}", licensed_resource_type=LicensedResourceType.VIP_MODEL, licensed_resource_data=vip, - license_key=vip.license_key, + licensed_item_display_name=vip.license_key, ) assert state2 == RegistrationState.ALREADY_REGISTERED @@ -194,7 +194,7 @@ async def test_sync_itis_vip_as_licensed_items( } } ), - license_key=vip.license_key, + licensed_item_display_name=vip.license_key, ) assert state3 == RegistrationState.DIFFERENT_RESOURCE From 610131ab224069a3a6b802efdb97e23b0f3b88d5 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 31 Jan 2025 19:35:39 +0100 Subject: [PATCH 32/46] tests pass --- .../licenses/_itis_vip_models.py | 3 +- .../licenses/_itis_vip_settings.py | 15 ++--- .../licenses/_licensed_items_repository.py | 63 +++++++++++-------- .../licenses/_licensed_items_service.py | 4 +- .../04/licenses/test_itis_vip_service.py | 16 ++--- 5 files changed, 56 insertions(+), 45 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py index 5c5bcf9db7c..7a9ee2252fe 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py @@ -1,5 +1,5 @@ import re -from typing import Annotated, Any, Literal, NotRequired, TypedDict +from typing import Annotated, Any, Literal, NotRequired from models_library.basic_types import IDStr from pydantic import ( @@ -10,6 +10,7 @@ StringConstraints, TypeAdapter, ) +from typing_extensions import TypedDict _max_str_adapter = TypeAdapter( Annotated[str, StringConstraints(strip_whitespace=True, max_length=1_000)] diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_settings.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_settings.py index d1218835cdc..67e7d860d0c 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_settings.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_settings.py @@ -1,6 +1,5 @@ -from typing import Annotated +from typing import Annotated, TypeAlias -from common_library.pydantic_basic_types import ShortTruncatedStr from models_library.basic_types import IDStr from pydantic import AfterValidator, HttpUrl from settings_library.base import BaseCustomSettings @@ -13,18 +12,16 @@ def _validate_url_contains_category(url: str) -> str: return url +CategoryID: TypeAlias = IDStr +CategoryDisplay: TypeAlias = str + + class ItisVipSettings(BaseCustomSettings): ITIS_VIP_API_URL: Annotated[str, AfterValidator(_validate_url_contains_category)] - ITIS_VIP_CATEGORIES: list[str] + ITIS_VIP_CATEGORIES: dict[CategoryID, CategoryDisplay] def get_urls(self) -> list[HttpUrl]: return [ HttpUrl(self.ITIS_VIP_API_URL.format(category=category)) for category in self.ITIS_VIP_CATEGORIES ] - - -class ItisVipCategorySettings(BaseCustomSettings): - ITIS_VIP_CATEGORY_URL: HttpUrl - ITIS_VIP_CATEGORY_ID: IDStr # use same as in vip-api - ITIS_VIP_CATEGORY_DISPLAY_NAME: ShortTruncatedStr diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py index 460cb6ecea1..4ecc41a7fc8 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py @@ -38,19 +38,15 @@ _SELECTION_ARGS = get_columns_from_db_model(licensed_items, LicensedItemDB) -async def create( - app: web.Application, - connection: AsyncConnection | None = None, - *, +def _create_insert_query( display_name: str, licensed_resource_name: str, licensed_resource_type: LicensedResourceType, - licensed_resource_data: dict[str, Any] | None = None, - product_name: ProductName | None = None, - pricing_plan_id: PricingPlanId | None = None, -) -> LicensedItemDB: - - query = ( + licensed_resource_data: dict[str, Any] | None, + product_name: ProductName | None, + pricing_plan_id: PricingPlanId | None, +) -> postgresql.Insert: + return ( postgresql.insert(licensed_items) .values( licensed_resource_name=licensed_resource_name, @@ -65,6 +61,27 @@ async def create( .returning(*_SELECTION_ARGS) ) + +async def create( + app: web.Application, + connection: AsyncConnection | None = None, + *, + display_name: str, + licensed_resource_name: str, + licensed_resource_type: LicensedResourceType, + licensed_resource_data: dict[str, Any] | None = None, + product_name: ProductName | None = None, + pricing_plan_id: PricingPlanId | None = None, +) -> LicensedItemDB: + + query = _create_insert_query( + display_name, + licensed_resource_name, + licensed_resource_type, + licensed_resource_data, + product_name, + pricing_plan_id, + ) async with transaction_context(get_asyncpg_engine(app), connection) as conn: result = await conn.execute(query) row = result.one() @@ -75,28 +92,22 @@ async def create_if_not_exists( app: web.Application, connection: AsyncConnection | None = None, *, + display_name: str, licensed_resource_name: str, licensed_resource_type: LicensedResourceType, licensed_resource_data: dict[str, Any] | None = None, - license_key: str | None = None, product_name: ProductName | None = None, pricing_plan_id: PricingPlanId | None = None, ) -> LicensedItemDB: - insert_or_none_query = ( - postgresql.insert(licensed_items) - .values( - licensed_resource_name=licensed_resource_name, - licensed_resource_type=licensed_resource_type, - licensed_resource_data=licensed_resource_data, - license_key=license_key, - pricing_plan_id=pricing_plan_id, - product_name=product_name, - created=func.now(), - modified=func.now(), - ) - .returning(*_SELECTION_ARGS) - .on_conflict_do_nothing() - ) + + insert_or_none_query = _create_insert_query( + display_name, + licensed_resource_name, + licensed_resource_type, + licensed_resource_data, + product_name, + pricing_plan_id, + ).on_conflict_do_nothing() async with transaction_context(get_asyncpg_engine(app), connection) as conn: result = await conn.execute(insert_or_none_query) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py index ac24eac1c12..139df16ae5a 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py @@ -59,7 +59,7 @@ async def register_resource_as_licensed_item( licensed_resource_name: str, licensed_resource_type: LicensedResourceType, licensed_resource_data: BaseModel, - license_key: str | None, + licensed_item_display_name: str, ) -> tuple[LicensedItemDB, RegistrationState]: try: licensed_item = await _licensed_items_repository.get_by_resource_identifier( @@ -93,12 +93,12 @@ async def register_resource_as_licensed_item( except LicensedItemNotFoundError: licensed_item = await _licensed_items_repository.create_if_not_exists( app, + display_name=licensed_item_display_name, licensed_resource_name=licensed_resource_name, licensed_resource_type=licensed_resource_type, licensed_resource_data=licensed_resource_data.model_dump( mode="json", exclude_unset=True ), - license_key=license_key, product_name=None, pricing_plan_id=None, ) diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index 18c2b404fda..25e33d51e73 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -51,7 +51,8 @@ def app_environment( monkeypatch, { "ITIS_VIP_API_URL": f"{fake_api_base_url}/PD_DirectDownload/getDownloadableItems/{{category}}", - "ITIS_VIP_CATEGORIES": '["ComputationalPantom","FooCategory","BarCategory"]', # NOTE: ItisVipSettings will decode with json.dumps() + # NOTE: ItisVipSettings will decode with json.dumps(). Use " and not ' the json keys!! + "ITIS_VIP_CATEGORIES": '{"ComputationalPantom": "Phantoms", "HumanBodyRegion": "Humans (Regions)"}', }, ) @@ -103,7 +104,8 @@ async def test_fetch_and_validate_itis_vip_api( assert len(validated_data.available_downloads) == 8 assert ( - validated_data.available_downloads[0].features["functionality"] == "Posable" + validated_data.available_downloads[0].features.get("functionality") + == "Posable" ) print(validated_data.model_dump_json(indent=1)) @@ -124,7 +126,7 @@ async def test_get_category_items( items = await _itis_vip_service.get_category_items(client, url) - assert items[0].features["functionality"] == "Posable" + assert items[0].features.get("functionality") == "Posable" async def test_sync_itis_vip_as_licensed_items( @@ -146,7 +148,7 @@ async def test_sync_itis_vip_as_licensed_items( vip_resources: list[ ItisVipData ] = await _itis_vip_service.get_category_items(http_client, url) - assert vip_resources[0].features["functionality"] == "Posable" + assert vip_resources[0].features.get("functionality") == "Posable" for vip in vip_resources: @@ -159,7 +161,7 @@ async def test_sync_itis_vip_as_licensed_items( licensed_resource_name=f"{category}/{vip.id}", licensed_resource_type=LicensedResourceType.VIP_MODEL, licensed_resource_data=vip, - licensed_item_display_name=f"{vip.features.get('name','')}", + licensed_item_display_name="foo", ) assert state1 == RegistrationState.NEWLY_REGISTERED @@ -172,7 +174,7 @@ async def test_sync_itis_vip_as_licensed_items( licensed_resource_name=f"{category}/{vip.id}", licensed_resource_type=LicensedResourceType.VIP_MODEL, licensed_resource_data=vip, - licensed_item_display_name=vip.license_key, + licensed_item_display_name="foo", ) assert state2 == RegistrationState.ALREADY_REGISTERED @@ -194,7 +196,7 @@ async def test_sync_itis_vip_as_licensed_items( } } ), - licensed_item_display_name=vip.license_key, + licensed_item_display_name="foo", ) assert state3 == RegistrationState.DIFFERENT_RESOURCE From 6bba3e14fd8e4b95d5a0e22e3d11638a506ae741 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 31 Jan 2025 20:10:38 +0100 Subject: [PATCH 33/46] result --- .../licenses/_itis_vip_models.py | 13 ++++- .../licenses/_licensed_items_service.py | 51 +++++++++++-------- .../04/licenses/test_itis_vip_service.py | 3 ++ 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py index 7a9ee2252fe..ba75c62d103 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py @@ -1,5 +1,5 @@ import re -from typing import Annotated, Any, Literal, NotRequired +from typing import Annotated, Any, Literal, NamedTuple, NotRequired from models_library.basic_types import IDStr from pydantic import ( @@ -72,3 +72,14 @@ class ItisVipResourceData(BaseModel): data: Annotated[ ItisVipData, Field(description="Original published data in the api") ] + + +# +# INTERNAL +# + + +class CategoryTuple(NamedTuple): + url: HttpUrl + id: IDStr + display: str diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py index 139df16ae5a..865b192874c 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py @@ -4,6 +4,7 @@ from datetime import UTC, datetime, timedelta from enum import Enum, auto from pprint import pformat +from typing import NamedTuple from aiohttp import web from models_library.licensed_items import ( @@ -41,6 +42,12 @@ class RegistrationState(Enum): NEWLY_REGISTERED = auto() +class RegistrationResult(NamedTuple): + registered: LicensedItemDB + state: RegistrationState + message: str | None + + def _compute_difference(old_data: dict, new_data: dict): differences = { k: {"old": old_data[k], "new": new_data[k]} @@ -60,7 +67,16 @@ async def register_resource_as_licensed_item( licensed_resource_type: LicensedResourceType, licensed_resource_data: BaseModel, licensed_item_display_name: str, -) -> tuple[LicensedItemDB, RegistrationState]: +) -> RegistrationResult: + # NOTE about the implementation choice: + # Using `create_if_not_exists` (INSERT with IGNORE_ON_CONFLICT) would have been an option, + # but it generates excessive error logs due to conflicts. + # + # To avoid this, we first attempt to retrieve the resource using `get_by_resource_identifier` (GET). + # If the resource does not exist, we proceed with `create_if_not_exists` (INSERT with IGNORE_ON_CONFLICT). + # + # This approach not only reduces unnecessary error logs but also helps prevent race conditions + # when multiple concurrent calls attempt to register the same resource. try: licensed_item = await _licensed_items_repository.get_by_resource_identifier( app, @@ -68,27 +84,23 @@ async def register_resource_as_licensed_item( licensed_resource_type=licensed_resource_type, ) - if licensed_item.licensed_resource_data != licensed_resource_data.model_dump( - mode="json", exclude_unset=True - ): + new_data = licensed_resource_data.model_dump(mode="json", exclude_unset=True) + + if licensed_item.licensed_resource_data != new_data: differences = _compute_difference( licensed_item.licensed_resource_data or {}, licensed_resource_data.model_dump(mode="json", exclude_unset=True), ) - _logger.warning( - "DIFFERENT_RESOURCE: %s, %s. Difference: %s", - licensed_resource_name, - licensed_resource_type, - pformat(differences), + msg = f"DIFFERENT RESOURCE: {licensed_resource_name}, {licensed_resource_type}. Difference:\n{pformat(differences)}" + return RegistrationResult( + licensed_item, RegistrationState.DIFFERENT_RESOURCE, msg ) - return licensed_item, RegistrationState.DIFFERENT_RESOURCE - _logger.info( - "ALREADY_REGISTERED: %s, %s", - licensed_resource_name, - licensed_resource_type, + return RegistrationResult( + licensed_item, + RegistrationState.ALREADY_REGISTERED, + f"ALREADY REGISTERED: {licensed_resource_name}, {licensed_resource_type}", ) - return licensed_item, RegistrationState.ALREADY_REGISTERED except LicensedItemNotFoundError: licensed_item = await _licensed_items_repository.create_if_not_exists( @@ -103,12 +115,11 @@ async def register_resource_as_licensed_item( pricing_plan_id=None, ) - _logger.info( - "NEWLY_REGISTERED: %s, %s", - licensed_resource_name, - licensed_resource_type, + return RegistrationResult( + licensed_item, + RegistrationState.NEWLY_REGISTERED, + f"NEWLY REGISTERED: {licensed_resource_name}, {licensed_resource_type}", ) - return licensed_item, RegistrationState.NEWLY_REGISTERED async def get_licensed_item( diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index 25e33d51e73..2f53a4b0c9b 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -156,6 +156,7 @@ async def test_sync_itis_vip_as_licensed_items( ( licensed_item1, state1, + _, ) = await _licensed_items_service.register_resource_as_licensed_item( client.app, licensed_resource_name=f"{category}/{vip.id}", @@ -169,6 +170,7 @@ async def test_sync_itis_vip_as_licensed_items( ( licensed_item2, state2, + _, ) = await _licensed_items_service.register_resource_as_licensed_item( client.app, licensed_resource_name=f"{category}/{vip.id}", @@ -184,6 +186,7 @@ async def test_sync_itis_vip_as_licensed_items( ( licensed_item3, state3, + _, ) = await _licensed_items_service.register_resource_as_licensed_item( client.app, licensed_resource_name=f"{category}/{vip.id}", From dde6c43567219a765c9862861a4aaabe71cf18a9 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 31 Jan 2025 20:46:55 +0100 Subject: [PATCH 34/46] cleanup --- .../licenses/_itis_vip_models.py | 9 ++++++--- .../licenses/_itis_vip_settings.py | 19 +++++++++++++------ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py index ba75c62d103..ffc7f9e49a3 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py @@ -1,5 +1,5 @@ import re -from typing import Annotated, Any, Literal, NamedTuple, NotRequired +from typing import Annotated, Any, Literal, NamedTuple, NotRequired, TypeAlias from models_library.basic_types import IDStr from pydantic import ( @@ -78,8 +78,11 @@ class ItisVipResourceData(BaseModel): # INTERNAL # +CategoryID: TypeAlias = IDStr +CategoryDisplay: TypeAlias = str + class CategoryTuple(NamedTuple): url: HttpUrl - id: IDStr - display: str + id: CategoryID + display: CategoryDisplay diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_settings.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_settings.py index 67e7d860d0c..a961472afbc 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_settings.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_settings.py @@ -1,9 +1,10 @@ -from typing import Annotated, TypeAlias +from typing import Annotated -from models_library.basic_types import IDStr from pydantic import AfterValidator, HttpUrl from settings_library.base import BaseCustomSettings +from ._itis_vip_models import CategoryDisplay, CategoryID, CategoryTuple + def _validate_url_contains_category(url: str) -> str: if "{category}" not in url: @@ -12,10 +13,6 @@ def _validate_url_contains_category(url: str) -> str: return url -CategoryID: TypeAlias = IDStr -CategoryDisplay: TypeAlias = str - - class ItisVipSettings(BaseCustomSettings): ITIS_VIP_API_URL: Annotated[str, AfterValidator(_validate_url_contains_category)] ITIS_VIP_CATEGORIES: dict[CategoryID, CategoryDisplay] @@ -25,3 +22,13 @@ def get_urls(self) -> list[HttpUrl]: HttpUrl(self.ITIS_VIP_API_URL.format(category=category)) for category in self.ITIS_VIP_CATEGORIES ] + + def to_categories(self) -> list[CategoryTuple]: + return [ + CategoryTuple( + url=HttpUrl(self.ITIS_VIP_API_URL.format(category=category_id)), + id=category_id, + display=category_display, + ) + for category_id, category_display in self.ITIS_VIP_CATEGORIES.items() + ] From 06f0c1162cb99b298c6640d457cfdd3d6a4ac959 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 31 Jan 2025 20:47:03 +0100 Subject: [PATCH 35/46] drafts syncer --- .../licenses/_itis_vip_syncer_service.py | 67 +++++++++++++++++++ .../04/licenses/test_itis_vip_service.py | 43 +++++++++++- 2 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py new file mode 100644 index 00000000000..a53bb35bab4 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py @@ -0,0 +1,67 @@ +import logging + +from aiohttp import web +from httpx import AsyncClient +from models_library.licensed_items import LicensedResourceType +from simcore_service_webserver.licenses import ( + _itis_vip_service, + _licensed_items_service, +) + +from ._itis_vip_models import CategoryTuple, ItisVipData, ItisVipResourceData +from ._licensed_items_service import RegistrationState + +_logger = logging.getLogger(__name__) + + +async def sync_resources_with_licensed_items( + app: web.Application, categories: list[CategoryTuple] +): + async with AsyncClient() as http_client: + for category_url, category_id, category_display in categories: + assert f"{category_url}".endswith(category_id) # nosec + + # FETCH & VALIDATION + vip_data_items: list[ + ItisVipData + # TODO: handle errors to avoid disrupting other categories? + ] = await _itis_vip_service.get_category_items(http_client, category_url) + + # REGISTRATION + for vip_data in vip_data_items: + + # TODO: handle error to avoid disrupting other vip_data_items? + result = await _licensed_items_service.register_resource_as_licensed_item( + app, + licensed_item_display_name=f"{vip_data.features['name']} {vip_data.features['version']}", + # RESOURCE unique identifiers + licensed_resource_name=f"{category_id}/{vip_data.id}", + licensed_resource_type=LicensedResourceType.VIP_MODEL, + # RESOURCE extended data + licensed_resource_data=ItisVipResourceData( + category_id=category_id, + category_display=category_display, + data=vip_data, + ), + ) + + if result.state == RegistrationState.ALREADY_REGISTERED: + # NOTE: not really interesting + _logger.debug(result.message) + + elif result.state == RegistrationState.DIFFERENT_RESOURCE: + # NOTE: notify since need human decision + _logger.warning(result.message) + + else: + assert result.state == RegistrationState.NEWLY_REGISTERED # nosec + # NOTE: inform since needs curation + _logger.info( + "%s . New licensed_item_id=%s pending for activation.", + result.message, + result.registered.licensed_item_id, + ) + + +async def background_periodic_sync_lifecycle(app: web.Application): + ... diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index 2f53a4b0c9b..c820562e555 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -1,9 +1,10 @@ +# pylint: disable=protected-access # pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments # pylint: disable=unused-argument # pylint: disable=unused-variable -# pylint: disable=too-many-arguments -# pylint: disable=too-many-statements +import logging from collections.abc import Iterator import pytest @@ -18,6 +19,7 @@ from servicelib.aiohttp import status from simcore_service_webserver.licenses import ( _itis_vip_service, + _itis_vip_syncer_service, _licensed_items_service, ) from simcore_service_webserver.licenses._itis_vip_models import ( @@ -204,3 +206,40 @@ async def test_sync_itis_vip_as_licensed_items( assert state3 == RegistrationState.DIFFERENT_RESOURCE assert licensed_item2 == licensed_item3 + + +async def test_itis_vip_syncer_service( + mock_itis_vip_downloadables_api: respx.MockRouter, + app_environment: EnvVarsDict, + client: TestClient, + caplog: pytest.LogCaptureFixture, +): + assert client.app + + settings = ItisVipSettings.create_from_envs() + assert settings.ITIS_VIP_CATEGORIES + + categories = settings.to_categories() + + with caplog.at_level(logging.DEBUG, _itis_vip_syncer_service._logger.name): + caplog.clear() + + # one round + await _itis_vip_syncer_service.sync_resources_with_licensed_items( + client.app, categories + ) + + levels_logged = [o[1] for o in caplog.record_tuples] + assert logging.DEBUG not in levels_logged + assert logging.INFO in levels_logged + assert logging.WARNING not in levels_logged + + caplog.clear() + # second round + await _itis_vip_syncer_service.sync_resources_with_licensed_items( + client.app, categories + ) + + assert logging.DEBUG in levels_logged + assert logging.INFO not in levels_logged + assert logging.WARNING not in levels_logged From 94f0fedcea79a58731fefc31e71e8524a53facd3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 31 Jan 2025 21:30:13 +0100 Subject: [PATCH 36/46] tests pass --- .../licenses/_licensed_items_service.py | 27 ++++++++++--------- .../unit/with_dbs/04/licenses/conftest.py | 16 ++++++++++- .../04/licenses/test_itis_vip_service.py | 15 ++++++++--- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py index 865b192874c..168f21882a9 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py @@ -77,6 +77,10 @@ async def register_resource_as_licensed_item( # # This approach not only reduces unnecessary error logs but also helps prevent race conditions # when multiple concurrent calls attempt to register the same resource. + new_licensed_resource_data = licensed_resource_data.model_dump( + mode="json", exclude_unset=True + ) + try: licensed_item = await _licensed_items_repository.get_by_resource_identifier( app, @@ -84,14 +88,13 @@ async def register_resource_as_licensed_item( licensed_resource_type=licensed_resource_type, ) - new_data = licensed_resource_data.model_dump(mode="json", exclude_unset=True) - - if licensed_item.licensed_resource_data != new_data: - differences = _compute_difference( - licensed_item.licensed_resource_data or {}, - licensed_resource_data.model_dump(mode="json", exclude_unset=True), - ) - msg = f"DIFFERENT RESOURCE: {licensed_resource_name}, {licensed_resource_type}. Difference:\n{pformat(differences)}" + if licensed_item.licensed_resource_data != new_licensed_resource_data: + # differences = _compute_difference( + # licensed_item.licensed_resource_data or {}, + # new_licensed_resource_data, + # ) + differences = "there are differences TMP" + msg = f"DIFFERENT_RESOURCE: {licensed_resource_name}, {licensed_resource_type}. Difference:\n{pformat(differences)}" return RegistrationResult( licensed_item, RegistrationState.DIFFERENT_RESOURCE, msg ) @@ -99,7 +102,7 @@ async def register_resource_as_licensed_item( return RegistrationResult( licensed_item, RegistrationState.ALREADY_REGISTERED, - f"ALREADY REGISTERED: {licensed_resource_name}, {licensed_resource_type}", + f"ALREADY_REGISTERED: {licensed_resource_name}, {licensed_resource_type}", ) except LicensedItemNotFoundError: @@ -108,9 +111,7 @@ async def register_resource_as_licensed_item( display_name=licensed_item_display_name, licensed_resource_name=licensed_resource_name, licensed_resource_type=licensed_resource_type, - licensed_resource_data=licensed_resource_data.model_dump( - mode="json", exclude_unset=True - ), + licensed_resource_data=new_licensed_resource_data, product_name=None, pricing_plan_id=None, ) @@ -118,7 +119,7 @@ async def register_resource_as_licensed_item( return RegistrationResult( licensed_item, RegistrationState.NEWLY_REGISTERED, - f"NEWLY REGISTERED: {licensed_resource_name}, {licensed_resource_type}", + f"NEWLY_REGISTERED: {licensed_resource_name}, {licensed_resource_type}", ) diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py b/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py index 5971ed9f168..ffe1a6f338c 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py @@ -33,7 +33,7 @@ async def pricing_plan_id( ) .returning(resource_tracker_pricing_plans.c.pricing_plan_id) ) - row = result.first() + row = result.one() assert row @@ -42,3 +42,17 @@ async def pricing_plan_id( async with transaction_context(get_asyncpg_engine(client.app)) as conn: await conn.execute(licensed_items.delete()) await conn.execute(resource_tracker_pricing_plans.delete()) + + +@pytest.fixture +async def ensure_empty_licensed_items(client: TestClient): + async def _cleanup(): + assert client.app + async with transaction_context(get_asyncpg_engine(client.app)) as conn: + await conn.execute(licensed_items.delete()) + + await _cleanup() + + yield + + await _cleanup() diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index c820562e555..a935c49db57 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -135,6 +135,7 @@ async def test_sync_itis_vip_as_licensed_items( mock_itis_vip_downloadables_api: respx.MockRouter, app_environment: EnvVarsDict, client: TestClient, + ensure_empty_licensed_items: None, ): assert client.app @@ -213,6 +214,7 @@ async def test_itis_vip_syncer_service( app_environment: EnvVarsDict, client: TestClient, caplog: pytest.LogCaptureFixture, + ensure_empty_licensed_items: None, ): assert client.app @@ -222,14 +224,20 @@ async def test_itis_vip_syncer_service( categories = settings.to_categories() with caplog.at_level(logging.DEBUG, _itis_vip_syncer_service._logger.name): - caplog.clear() + + def _get_captured_levels(): + return [ + rc[1] + for rc in caplog.record_tuples + if rc[0] == _itis_vip_syncer_service._logger.name + ] # one round + caplog.clear() await _itis_vip_syncer_service.sync_resources_with_licensed_items( client.app, categories ) - - levels_logged = [o[1] for o in caplog.record_tuples] + levels_logged = _get_captured_levels() assert logging.DEBUG not in levels_logged assert logging.INFO in levels_logged assert logging.WARNING not in levels_logged @@ -240,6 +248,7 @@ async def test_itis_vip_syncer_service( client.app, categories ) + levels_logged = _get_captured_levels() assert logging.DEBUG in levels_logged assert logging.INFO not in levels_logged assert logging.WARNING not in levels_logged From 75fbcb39110eb021293db7d4da9563412b26393b Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 31 Jan 2025 21:53:40 +0100 Subject: [PATCH 37/46] adds deepdiff --- services/web/server/requirements/_base.in | 1 + services/web/server/requirements/_base.txt | 5 +++ .../licenses/_licensed_items_service.py | 32 +++++++------------ .../04/licenses/test_itis_vip_service.py | 5 +-- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/services/web/server/requirements/_base.in b/services/web/server/requirements/_base.in index 326bef1224f..f8e72f62ca6 100644 --- a/services/web/server/requirements/_base.in +++ b/services/web/server/requirements/_base.in @@ -32,6 +32,7 @@ aiosmtplib # email asyncpg # db captcha cryptography # security +deepdiff[optimize] # diffs data-structures faker # Only used in dev-mode for proof-of-concepts gunicorn[setproctitle] httpx diff --git a/services/web/server/requirements/_base.txt b/services/web/server/requirements/_base.txt index e019a2d634b..562b0eab736 100644 --- a/services/web/server/requirements/_base.txt +++ b/services/web/server/requirements/_base.txt @@ -189,6 +189,8 @@ cryptography==41.0.7 # -c requirements/../../../../requirements/constraints.txt # -r requirements/_base.in # aiohttp-session +deepdiff==8.1.1 + # via -r requirements/_base.in deprecated==1.2.14 # via # opentelemetry-api @@ -455,6 +457,8 @@ opentelemetry-util-http==0.48b0 # opentelemetry-instrumentation-aiohttp-client # opentelemetry-instrumentation-aiohttp-server # opentelemetry-instrumentation-requests +orderly-set==5.2.3 + # via deepdiff orjson==3.10.0 # via # -c requirements/../../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -504,6 +508,7 @@ orjson==3.10.0 # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/_base.in + # deepdiff packaging==24.1 # via # -r requirements/../../../../packages/simcore-sdk/requirements/_base.in diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py index 168f21882a9..b559150ee49 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py @@ -7,6 +7,7 @@ from typing import NamedTuple from aiohttp import web +from deepdiff import DeepDiff from models_library.licensed_items import ( LicensedItemDB, LicensedItemID, @@ -48,18 +49,6 @@ class RegistrationResult(NamedTuple): message: str | None -def _compute_difference(old_data: dict, new_data: dict): - differences = { - k: {"old": old_data[k], "new": new_data[k]} - for k in old_data - if old_data[k] != new_data.get(k) - } - differences.update( - {k: {"old": None, "new": new_data[k]} for k in new_data if k not in old_data} - ) - return differences - - async def register_resource_as_licensed_item( app: web.Application, *, @@ -77,6 +66,8 @@ async def register_resource_as_licensed_item( # # This approach not only reduces unnecessary error logs but also helps prevent race conditions # when multiple concurrent calls attempt to register the same resource. + + resource_key = f"{licensed_resource_type}, {licensed_resource_name}" new_licensed_resource_data = licensed_resource_data.model_dump( mode="json", exclude_unset=True ) @@ -89,12 +80,13 @@ async def register_resource_as_licensed_item( ) if licensed_item.licensed_resource_data != new_licensed_resource_data: - # differences = _compute_difference( - # licensed_item.licensed_resource_data or {}, - # new_licensed_resource_data, - # ) - differences = "there are differences TMP" - msg = f"DIFFERENT_RESOURCE: {licensed_resource_name}, {licensed_resource_type}. Difference:\n{pformat(differences)}" + ddiff = DeepDiff( + licensed_item.licensed_resource_data, new_licensed_resource_data + ) + msg = ( + f"DIFFERENT_RESOURCE: {resource_key=} found in licensed_item_id={licensed_item.licensed_item_id} with different data. " + f"Diff:\n\t{pformat(ddiff, indent=2, width=200)}" + ) return RegistrationResult( licensed_item, RegistrationState.DIFFERENT_RESOURCE, msg ) @@ -102,7 +94,7 @@ async def register_resource_as_licensed_item( return RegistrationResult( licensed_item, RegistrationState.ALREADY_REGISTERED, - f"ALREADY_REGISTERED: {licensed_resource_name}, {licensed_resource_type}", + f"ALREADY_REGISTERED: {resource_key=} found in licensed_item_id={licensed_item.licensed_item_id}", ) except LicensedItemNotFoundError: @@ -119,7 +111,7 @@ async def register_resource_as_licensed_item( return RegistrationResult( licensed_item, RegistrationState.NEWLY_REGISTERED, - f"NEWLY_REGISTERED: {licensed_resource_name}, {licensed_resource_type}", + f"NEWLY_REGISTERED: {resource_key=} registered with licensed_item_id={licensed_item.licensed_item_id}", ) diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index a935c49db57..580d14a9ae7 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -189,7 +189,7 @@ async def test_sync_itis_vip_as_licensed_items( ( licensed_item3, state3, - _, + msg, ) = await _licensed_items_service.register_resource_as_licensed_item( client.app, licensed_resource_name=f"{category}/{vip.id}", @@ -204,9 +204,10 @@ async def test_sync_itis_vip_as_licensed_items( ), licensed_item_display_name="foo", ) - assert state3 == RegistrationState.DIFFERENT_RESOURCE assert licensed_item2 == licensed_item3 + # {'values_changed': {"root['features']['functionality']": {'new_value': 'Non-Posable', 'old_value': 'Posable'}}} + assert "functionality" in msg async def test_itis_vip_syncer_service( From dfa43f9c49e3be31ee2e73b30a6c711d82c99960 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 31 Jan 2025 22:13:03 +0100 Subject: [PATCH 38/46] drafts task --- .env-devel | 4 ++ .../licenses/_itis_vip_syncer_service.py | 51 ++++++++++++++++++- .../licenses/plugin.py | 4 ++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/.env-devel b/.env-devel index e2f8baef3ca..b503d8f49d9 100644 --- a/.env-devel +++ b/.env-devel @@ -133,6 +133,10 @@ DYNAMIC_SCHEDULER_UI_STORAGE_SECRET=adminadmin FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "unknown@osparc.io", "affiliation": "unknown"}}' + +ITIS_VIP_API_URL=https://itis.swiss/PD_DirectDownload/getDownloadableItems/{category} +ITIS_VIP_CATEGORIES='{"HumanWholeBody": "Humans", "HumanBodyRegion": "Humans (Region)", "AnimalWholeBody": "Animal", "ComputationalPhantom": "Phantoms"}' + # Can use 'docker run -it itisfoundation/invitations:latest simcore-service-invitations generate-dotenv --auto-password' INVITATIONS_DEFAULT_PRODUCT=osparc INVITATIONS_HOST=invitations diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py index a53bb35bab4..e63b5bf3951 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py @@ -1,13 +1,21 @@ +import asyncio import logging +from datetime import timedelta from aiohttp import web from httpx import AsyncClient from models_library.licensed_items import LicensedResourceType +from pydantic import ValidationError +from servicelib.async_utils import cancel_wait_task +from servicelib.background_task_utils import exclusive_periodic +from servicelib.logging_utils import log_catch, log_context from simcore_service_webserver.licenses import ( _itis_vip_service, _licensed_items_service, ) +from simcore_service_webserver.licenses._itis_vip_settings import ItisVipSettings +from ..redis import get_redis_lock_manager_client_sdk, setup_redis from ._itis_vip_models import CategoryTuple, ItisVipData, ItisVipResourceData from ._licensed_items_service import RegistrationState @@ -63,5 +71,44 @@ async def sync_resources_with_licensed_items( ) -async def background_periodic_sync_lifecycle(app: web.Application): - ... +_BACKGROUND_TASK_NAME = f"{__name__}.itis_vip_syncer_cleanup_ctx._periodic_sync" + + +def setup_itis_vip_syncer(app: web.Application): + categories = [] + + try: + settings = ItisVipSettings.create_from_envs() + categories = settings.to_categories() + + except ValidationError as err: + _logger.warning("IT'IS VIP syncer disabled. Skipping. %s", err) + return + + async def _cleanup_ctx(app: web.Application): + with ( + log_context( + _logger, logging.INFO, f"IT'IS VIP syncing {len(categories)} categories" + ), + log_catch(_logger, reraise=False), + ): + + @exclusive_periodic( + get_redis_lock_manager_client_sdk(app), + task_interval=timedelta(days=1), + retry_after=timedelta(minutes=5), + ) + async def _periodic_sync() -> None: + await sync_resources_with_licensed_items(app, categories=categories) + + background_task = asyncio.create_task( + _periodic_sync(), name=_BACKGROUND_TASK_NAME + ) + + yield + + await cancel_wait_task(background_task) + + if categories: + setup_redis(app) + app.cleanup_ctx.append(_cleanup_ctx) diff --git a/services/web/server/src/simcore_service_webserver/licenses/plugin.py b/services/web/server/src/simcore_service_webserver/licenses/plugin.py index 859a52bf1bd..2402140c1f1 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/plugin.py +++ b/services/web/server/src/simcore_service_webserver/licenses/plugin.py @@ -10,6 +10,7 @@ from ..rabbitmq import setup_rabbitmq from ..rest.plugin import setup_rest from . import ( + _itis_vip_syncer_service, _licensed_items_checkouts_rest, _licensed_items_purchases_rest, _licensed_items_rest, @@ -37,3 +38,6 @@ def setup_licenses(app: web.Application): setup_rabbitmq(app) if app[APP_SETTINGS_KEY].WEBSERVER_RABBITMQ: app.on_startup.append(_rpc.register_rpc_routes_on_startup) + + # TODO: this is temporary + _itis_vip_syncer_service.setup_itis_vip_syncer(app) From a152413f3a4690830db4c09e434c670c5c0472df Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 3 Feb 2025 11:52:07 +0100 Subject: [PATCH 39/46] draft demo --- services/docker-compose.yml | 2 + .../licenses/_itis_vip_models.py | 4 +- .../licenses/_itis_vip_service.py | 1 + .../licenses/_itis_vip_syncer_service.py | 85 +++++++++++-------- 4 files changed, 54 insertions(+), 38 deletions(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 555f47c4c9d..6a7e8bc966f 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -705,6 +705,8 @@ services: INVITATIONS_USERNAME: ${INVITATIONS_USERNAME} WEBSERVER_LICENSES: ${WEBSERVER_LICENSES} + ITIS_VIP_API_URL: ${ITIS_VIP_API_URL} + ITIS_VIP_CATEGORIES: ${ITIS_VIP_CATEGORIES} WEBSERVER_LOGIN: ${WEBSERVER_LOGIN} LOGIN_ACCOUNT_DELETION_RETENTION_DAYS: ${LOGIN_ACCOUNT_DELETION_RETENTION_DAYS} diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py index ffc7f9e49a3..c840e543a4b 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py @@ -47,11 +47,11 @@ class ItisVipData(BaseModel): description: Annotated[str, Field(alias="Description")] thumbnail: Annotated[str, Field(alias="Thumbnail")] features: Annotated[ - FeaturesDict, + dict, BeforeValidator(_feature_descriptor_to_dict), Field(alias="Features"), ] - doi: Annotated[str, Field(alias="DOI")] + doi: Annotated[str | None, Field(alias="DOI")] license_key: Annotated[str | None, Field(alias="LicenseKey")] license_version: Annotated[str | None, Field(alias="LicenseVersion")] protection: Annotated[Literal["Code", "PayPal"], Field(alias="Protection")] diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py index 87b249f5b9d..2f3b9deab1e 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_service.py @@ -28,4 +28,5 @@ async def get_category_items( response.raise_for_status() validated_data = ItisVipApiResponse.model_validate(response.json()) + return validated_data.available_downloads diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py index e63b5bf3951..05a4a477c29 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py @@ -30,46 +30,59 @@ async def sync_resources_with_licensed_items( assert f"{category_url}".endswith(category_id) # nosec # FETCH & VALIDATION - vip_data_items: list[ - ItisVipData - # TODO: handle errors to avoid disrupting other categories? - ] = await _itis_vip_service.get_category_items(http_client, category_url) + with log_context( + _logger, logging.INFO, "Fetching %s and validating", category_url + ), log_catch(_logger, reraise=True): + vip_data_items: list[ + ItisVipData + # TODO: handle errors to avoid disrupting other categories? + ] = await _itis_vip_service.get_category_items( + http_client, category_url + ) # REGISTRATION for vip_data in vip_data_items: - # TODO: handle error to avoid disrupting other vip_data_items? - result = await _licensed_items_service.register_resource_as_licensed_item( - app, - licensed_item_display_name=f"{vip_data.features['name']} {vip_data.features['version']}", - # RESOURCE unique identifiers - licensed_resource_name=f"{category_id}/{vip_data.id}", - licensed_resource_type=LicensedResourceType.VIP_MODEL, - # RESOURCE extended data - licensed_resource_data=ItisVipResourceData( - category_id=category_id, - category_display=category_display, - data=vip_data, - ), - ) - - if result.state == RegistrationState.ALREADY_REGISTERED: - # NOTE: not really interesting - _logger.debug(result.message) - - elif result.state == RegistrationState.DIFFERENT_RESOURCE: - # NOTE: notify since need human decision - _logger.warning(result.message) - - else: - assert result.state == RegistrationState.NEWLY_REGISTERED # nosec - # NOTE: inform since needs curation - _logger.info( - "%s . New licensed_item_id=%s pending for activation.", - result.message, - result.registered.licensed_item_id, + licensed_resource_name = f"{category_id}/{vip_data.id}" + + with log_context( + _logger, logging.INFO, "Registering %s", licensed_resource_name + ), log_catch(_logger, reraise=False): + # TODO: handle error to avoid disrupting other vip_data_items? + result = await _licensed_items_service.register_resource_as_licensed_item( + app, + licensed_item_display_name=f"{vip_data.features.get('name', 'UNNAMED!!')} " + f"{vip_data.features.get('version', 'UNVERSIONED!!')}", + # RESOURCE unique identifiers + licensed_resource_name=licensed_resource_name, + licensed_resource_type=LicensedResourceType.VIP_MODEL, + # RESOURCE extended data + licensed_resource_data=ItisVipResourceData( + category_id=category_id, + category_display=category_display, + data=vip_data, + ), ) + if result.state == RegistrationState.ALREADY_REGISTERED: + # NOTE: not really interesting + _logger.debug(result.message) + + elif result.state == RegistrationState.DIFFERENT_RESOURCE: + # NOTE: notify since need human decision + _logger.warning(result.message) + + else: + assert ( + result.state == RegistrationState.NEWLY_REGISTERED + ) # nosec + # NOTE: inform since needs curation + _logger.info( + "%s . New licensed_item_id=%s pending for activation.", + result.message, + result.registered.licensed_item_id, + ) + _BACKGROUND_TASK_NAME = f"{__name__}.itis_vip_syncer_cleanup_ctx._periodic_sync" @@ -95,8 +108,8 @@ async def _cleanup_ctx(app: web.Application): @exclusive_periodic( get_redis_lock_manager_client_sdk(app), - task_interval=timedelta(days=1), - retry_after=timedelta(minutes=5), + task_interval=timedelta(minutes=1), + retry_after=timedelta(minutes=2), ) async def _periodic_sync() -> None: await sync_resources_with_licensed_items(app, categories=categories) From 2238c9324044627449e2d2fd81d19499cb011df5 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 3 Feb 2025 15:53:05 +0100 Subject: [PATCH 40/46] fixes dev mode --- services/api-server/src/simcore_service_api_server/main.py | 3 +-- services/resource-usage-tracker/docker/boot.sh | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/main.py b/services/api-server/src/simcore_service_api_server/main.py index 493874ee6eb..8b636ac4315 100644 --- a/services/api-server/src/simcore_service_api_server/main.py +++ b/services/api-server/src/simcore_service_api_server/main.py @@ -1,8 +1,7 @@ """Main application to be deployed in for example uvicorn. """ from fastapi import FastAPI - -from .core.application import init_app +from simcore_service_api_server.core.application import init_app # SINGLETON FastAPI app the_app: FastAPI = init_app() diff --git a/services/resource-usage-tracker/docker/boot.sh b/services/resource-usage-tracker/docker/boot.sh index 28854b7b2b5..2b3c1bebc5f 100755 --- a/services/resource-usage-tracker/docker/boot.sh +++ b/services/resource-usage-tracker/docker/boot.sh @@ -48,7 +48,7 @@ if [ "${SC_BOOT_MODE}" = "debug" ]; then exec sh -c " cd services/resource-usage-tracker/src/simcore_service_resource_usage_tracker && \ - python -m debugpy --listen 0.0.0.0:${RESOURCE_USAGE_TRACKER_REMOTE_DEBUGGING_PORT} -m uvicorn web_main:the_app \ + python -m debugpy --listen 0.0.0.0:${RESOURCE_USAGE_TRACKER_REMOTE_DEBUGGING_PORT} -m uvicorn main:the_app \ --host 0.0.0.0 \ --reload \ $reload_dir_packages From ac35a493c89bc7bf456c0de3b9a163914127305f Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 3 Feb 2025 15:58:51 +0100 Subject: [PATCH 41/46] drafts demo --- .env-devel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env-devel b/.env-devel index b503d8f49d9..2505f3cd3f0 100644 --- a/.env-devel +++ b/.env-devel @@ -135,7 +135,7 @@ FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "unknown@osparc.i ITIS_VIP_API_URL=https://itis.swiss/PD_DirectDownload/getDownloadableItems/{category} -ITIS_VIP_CATEGORIES='{"HumanWholeBody": "Humans", "HumanBodyRegion": "Humans (Region)", "AnimalWholeBody": "Animal", "ComputationalPhantom": "Phantoms"}' +ITIS_VIP_CATEGORIES='{"HumanWholeBody": "Humans", "HumanBodyRegion": "Humans (Region)", "AnimalWholeBody": "Animal"}' # Can use 'docker run -it itisfoundation/invitations:latest simcore-service-invitations generate-dotenv --auto-password' INVITATIONS_DEFAULT_PRODUCT=osparc From 3d2eec69b80ffe9e3fc7bfe3fda5abea2ec81ee8 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 3 Feb 2025 16:39:11 +0100 Subject: [PATCH 42/46] new fake --- .../pytest_simcore/helpers/faker_factories.py | 32 +++++++++++++++++++ .../04/licenses/test_itis_vip_service.py | 32 ++++++------------- 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py index ce7c5685e94..9e7ac7da9ec 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py @@ -437,3 +437,35 @@ def random_service_access_rights( data.update(**overrides) return data + + +def random_itis_vip_available_download_item( + identifier: int, + fake: Faker = DEFAULT_FAKER, + features_functionality: str = "Posable", + **overrides, +): + features_str = ( + "{" + f"name: {fake.name()} Right Hand," # w/o spaces + f" version: V{fake.pyint()}.0, " # w/ x2 spaces + f"sex: Male, age: 8 years," # w/o spaces + f"date: {fake.date()}, " # w/ x1 spaces + f"ethnicity: Caucasian, functionality: {features_functionality} " + "}" + ) + + data = { + "ID": identifier, + "Description": fake.sentence(), + "Thumbnail": fake.image_url(), + "Features": features_str, + "DOI": fake.bothify(text="10.####/ViP#####-##-#"), + "LicenseKey": fake.bothify(text="MODEL_????_V#"), + "LicenseVersion": fake.bothify(text="V#.0"), + "Protection": fake.random_element(elements=["Code", "PayPal"]), + "AvailableFromURL": fake.random_element(elements=[None, fake.url()]), + } + + data.update(**overrides) + return data diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index 580d14a9ae7..3c35bec4055 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -14,6 +14,9 @@ from httpx import AsyncClient from models_library.licensed_items import LicensedResourceType from pydantic import ValidationError +from pytest_simcore.helpers.faker_factories import ( + random_itis_vip_available_download_item, +) from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.aiohttp import status @@ -22,22 +25,12 @@ _itis_vip_syncer_service, _licensed_items_service, ) -from simcore_service_webserver.licenses._itis_vip_models import ( - ItisVipData, - _feature_descriptor_to_dict, -) +from simcore_service_webserver.licenses._itis_vip_models import ItisVipData from simcore_service_webserver.licenses._itis_vip_service import ItisVipApiResponse from simcore_service_webserver.licenses._itis_vip_settings import ItisVipSettings from simcore_service_webserver.licenses._licensed_items_service import RegistrationState -def test_pre_validator_feature_descriptor_to_dict(): - # Makes sure the regex used here, which is vulnerable to polynomial runtime due to backtracking, cannot lead to denial of service. - with pytest.raises(ValidationError) as err_info: - _feature_descriptor_to_dict("a" * 10000 + ": " + "b" * 10000) - assert err_info.value.errors()[0]["type"] == "string_too_long" - - @pytest.fixture(scope="session") def fake_api_base_url() -> str: return "https://testserver-itis-vip.xyz" @@ -66,18 +59,11 @@ def mock_itis_vip_downloadables_api( response_data = { "msg": 0, "availableDownloads": [ - { - "ID": i, - "Description": faker.sentence(), - "Thumbnail": faker.image_url(), - # NOTE: this is manually added in the server side so be more robust to errors - "Features": f"{{name: {faker.name()} Right Hand, version: V{faker.pyint()}.0, sex: Male, age: 8 years,date: {faker.date()}, ethnicity: Caucasian, functionality: Posable}}", - "DOI": faker.bothify(text="10.####/ViP#####-##-#"), - "LicenseKey": faker.bothify(text="MODEL_????_V#"), - "LicenseVersion": faker.bothify(text="V#.0"), - "Protection": faker.random_element(elements=["Code", "PayPal"]), - "AvailableFromURL": faker.random_element(elements=[None, faker.url()]), - } + random_itis_vip_available_download_item( + identifier=i, + features_functionality="Posable", + faker=faker, + ) for i in range(8) ], } From 6a7822ece6d08c51bdb5df1b0daef4f8228f6b1c Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 3 Feb 2025 16:40:08 +0100 Subject: [PATCH 43/46] tests model guarantees --- .../licenses/_itis_vip_models.py | 2 +- .../licenses/_itis_vip_syncer_service.py | 2 +- .../licenses/_licensed_items_service.py | 4 +- .../04/licenses/test_itis_vip_models.py | 37 +++++++++++++++++++ 4 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_models.py diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py index c840e543a4b..5c4cdbcec7e 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py @@ -69,7 +69,7 @@ class ItisVipApiResponse(BaseModel): class ItisVipResourceData(BaseModel): category_id: IDStr category_display: str - data: Annotated[ + source: Annotated[ ItisVipData, Field(description="Original published data in the api") ] diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py index 05a4a477c29..efcd5345d5a 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py @@ -60,7 +60,7 @@ async def sync_resources_with_licensed_items( licensed_resource_data=ItisVipResourceData( category_id=category_id, category_display=category_display, - data=vip_data, + source=vip_data, ), ) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py index b559150ee49..8ea8f812a8f 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py @@ -69,7 +69,9 @@ async def register_resource_as_licensed_item( resource_key = f"{licensed_resource_type}, {licensed_resource_name}" new_licensed_resource_data = licensed_resource_data.model_dump( - mode="json", exclude_unset=True + mode="json", + exclude_unset=True, + by_alias=True, # FIXME: ensure only feature are exposed by_alias ) try: diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_models.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_models.py new file mode 100644 index 00000000000..b338dccf87b --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_models.py @@ -0,0 +1,37 @@ +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments +# pylint: disable=unused-argument +# pylint: disable=unused-variable + + +import pytest +from faker import Faker +from pydantic import ValidationError +from pytest_simcore.helpers.faker_factories import ( + random_itis_vip_available_download_item, +) +from simcore_service_webserver.licenses._itis_vip_models import ( + ItisVipData, + _feature_descriptor_to_dict, +) + + +def test_pre_validator_feature_descriptor_to_dict(): + # Makes sure the regex used here, which is vulnerable to polynomial runtime due to backtracking, cannot lead to denial of service. + with pytest.raises(ValidationError) as err_info: + _feature_descriptor_to_dict("a" * 10000 + ": " + "b" * 10000) + assert err_info.value.errors()[0]["type"] == "string_too_long" + + +def test_model(faker: Faker): + + available_download = random_itis_vip_available_download_item( + identifier=0, + features_functionality="Posable", + faker=faker, + ) + + vip_data = ItisVipData.model_validate(available_download) + + print(vip_data.model_dump(by_alias=True)) From de1ec81dde785e4209d8ca01831338b2ddf98d06 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 3 Feb 2025 17:00:18 +0100 Subject: [PATCH 44/46] source is respected --- .../licenses/_itis_vip_models.py | 19 ++++++++++++++++--- .../licenses/_itis_vip_syncer_service.py | 2 +- .../licenses/_licensed_items_service.py | 1 - .../04/licenses/test_itis_vip_models.py | 14 ++++++++++++-- 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py index 5c4cdbcec7e..8e0d83378b6 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_models.py @@ -1,5 +1,5 @@ import re -from typing import Annotated, Any, Literal, NamedTuple, NotRequired, TypeAlias +from typing import Annotated, Any, Literal, NamedTuple, NotRequired, Self, TypeAlias from models_library.basic_types import IDStr from pydantic import ( @@ -47,7 +47,7 @@ class ItisVipData(BaseModel): description: Annotated[str, Field(alias="Description")] thumbnail: Annotated[str, Field(alias="Thumbnail")] features: Annotated[ - dict, + dict[str, Any], # NOTE: for the moment FeaturesDict is NOT used BeforeValidator(_feature_descriptor_to_dict), Field(alias="Features"), ] @@ -66,13 +66,26 @@ class ItisVipApiResponse(BaseModel): # # RESOURCE # + + class ItisVipResourceData(BaseModel): category_id: IDStr category_display: str source: Annotated[ - ItisVipData, Field(description="Original published data in the api") + dict[str, Any], Field(description="Original published data in the api") ] + @classmethod + def create( + cls, category_id: IDStr, category_display: str, source: ItisVipData + ) -> Self: + return cls( + category_id=category_id, + category_display=category_display, + # NOTE: ensures source data is the same as the one in the original VIP API model + source=source.model_dump(mode="json", by_alias=True), + ) + # # INTERNAL diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py index efcd5345d5a..ec03719f137 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py @@ -57,7 +57,7 @@ async def sync_resources_with_licensed_items( licensed_resource_name=licensed_resource_name, licensed_resource_type=LicensedResourceType.VIP_MODEL, # RESOURCE extended data - licensed_resource_data=ItisVipResourceData( + licensed_resource_data=ItisVipResourceData.create( category_id=category_id, category_display=category_display, source=vip_data, diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py index 8ea8f812a8f..2a41967633b 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_service.py @@ -71,7 +71,6 @@ async def register_resource_as_licensed_item( new_licensed_resource_data = licensed_resource_data.model_dump( mode="json", exclude_unset=True, - by_alias=True, # FIXME: ensure only feature are exposed by_alias ) try: diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_models.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_models.py index b338dccf87b..ec2ad7f19af 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_models.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_models.py @@ -13,6 +13,7 @@ ) from simcore_service_webserver.licenses._itis_vip_models import ( ItisVipData, + ItisVipResourceData, _feature_descriptor_to_dict, ) @@ -24,7 +25,7 @@ def test_pre_validator_feature_descriptor_to_dict(): assert err_info.value.errors()[0]["type"] == "string_too_long" -def test_model(faker: Faker): +def test_validation_of_itis_vip_response_model(faker: Faker): available_download = random_itis_vip_available_download_item( identifier=0, @@ -34,4 +35,13 @@ def test_model(faker: Faker): vip_data = ItisVipData.model_validate(available_download) - print(vip_data.model_dump(by_alias=True)) + # Dumped as in the source + assert vip_data.model_dump(by_alias=True)["Features"] == vip_data.features + + license_resource_data = ItisVipResourceData.create( + category_id="123", + category_display="This is a resource", + source=vip_data, + ) + + assert license_resource_data.source["Features"] == vip_data.features From f89181c261cc664c3082cbcad564f505587d8974 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 3 Feb 2025 17:13:57 +0100 Subject: [PATCH 45/46] adapting tests --- .../licenses/_itis_vip_syncer_service.py | 47 ++++++++++-------- .../04/licenses/test_itis_vip_models.py | 2 +- .../04/licenses/test_itis_vip_service.py | 48 +++++++------------ 3 files changed, 45 insertions(+), 52 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py index ec03719f137..f5214b53d8a 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_itis_vip_syncer_service.py @@ -83,6 +83,8 @@ async def sync_resources_with_licensed_items( result.registered.licensed_item_id, ) + # TODO: check delete!? + _BACKGROUND_TASK_NAME = f"{__name__}.itis_vip_syncer_cleanup_ctx._periodic_sync" @@ -98,30 +100,35 @@ def setup_itis_vip_syncer(app: web.Application): _logger.warning("IT'IS VIP syncer disabled. Skipping. %s", err) return - async def _cleanup_ctx(app: web.Application): - with ( - log_context( - _logger, logging.INFO, f"IT'IS VIP syncing {len(categories)} categories" - ), - log_catch(_logger, reraise=False), - ): + if categories: - @exclusive_periodic( - get_redis_lock_manager_client_sdk(app), - task_interval=timedelta(minutes=1), - retry_after=timedelta(minutes=2), - ) - async def _periodic_sync() -> None: - await sync_resources_with_licensed_items(app, categories=categories) + async def _cleanup_ctx(app_: web.Application): + with ( + log_context( + _logger, + logging.INFO, + f"IT'IS VIP syncing {len(categories)} categories", + ), + log_catch(_logger, reraise=False), + ): + + @exclusive_periodic( + get_redis_lock_manager_client_sdk(app_), + task_interval=timedelta(minutes=1), + retry_after=timedelta(minutes=2), + ) + async def _periodic_sync() -> None: + await sync_resources_with_licensed_items( + app_, categories=categories + ) - background_task = asyncio.create_task( - _periodic_sync(), name=_BACKGROUND_TASK_NAME - ) + background_task = asyncio.create_task( + _periodic_sync(), name=_BACKGROUND_TASK_NAME + ) - yield + yield - await cancel_wait_task(background_task) + await cancel_wait_task(background_task) - if categories: setup_redis(app) app.cleanup_ctx.append(_cleanup_ctx) diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_models.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_models.py index ec2ad7f19af..44b80716fc8 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_models.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_models.py @@ -30,7 +30,7 @@ def test_validation_of_itis_vip_response_model(faker: Faker): available_download = random_itis_vip_available_download_item( identifier=0, features_functionality="Posable", - faker=faker, + fake=faker, ) vip_data = ItisVipData.model_validate(available_download) diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py index 3c35bec4055..50f437ecc78 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_itis_vip_service.py @@ -4,7 +4,6 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable -import logging from collections.abc import Iterator import pytest @@ -14,6 +13,7 @@ from httpx import AsyncClient from models_library.licensed_items import LicensedResourceType from pydantic import ValidationError +from pytest_mock import MockerFixture from pytest_simcore.helpers.faker_factories import ( random_itis_vip_available_download_item, ) @@ -41,7 +41,14 @@ def app_environment( monkeypatch: pytest.MonkeyPatch, app_environment: EnvVarsDict, fake_api_base_url: str, + mocker: MockerFixture, ): + # prevents syncer setup + mocker.patch( + "simcore_service_webserver.licenses.plugin._itis_vip_syncer_service.setup_itis_vip_syncer", + autospec=True, + ) + return app_environment | setenvs_from_dict( monkeypatch, { @@ -62,7 +69,7 @@ def mock_itis_vip_downloadables_api( random_itis_vip_available_download_item( identifier=i, features_functionality="Posable", - faker=faker, + fake=faker, ) for i in range(8) ], @@ -200,7 +207,6 @@ async def test_itis_vip_syncer_service( mock_itis_vip_downloadables_api: respx.MockRouter, app_environment: EnvVarsDict, client: TestClient, - caplog: pytest.LogCaptureFixture, ensure_empty_licensed_items: None, ): assert client.app @@ -210,32 +216,12 @@ async def test_itis_vip_syncer_service( categories = settings.to_categories() - with caplog.at_level(logging.DEBUG, _itis_vip_syncer_service._logger.name): - - def _get_captured_levels(): - return [ - rc[1] - for rc in caplog.record_tuples - if rc[0] == _itis_vip_syncer_service._logger.name - ] - - # one round - caplog.clear() - await _itis_vip_syncer_service.sync_resources_with_licensed_items( - client.app, categories - ) - levels_logged = _get_captured_levels() - assert logging.DEBUG not in levels_logged - assert logging.INFO in levels_logged - assert logging.WARNING not in levels_logged - - caplog.clear() - # second round - await _itis_vip_syncer_service.sync_resources_with_licensed_items( - client.app, categories - ) + # one round + await _itis_vip_syncer_service.sync_resources_with_licensed_items( + client.app, categories + ) - levels_logged = _get_captured_levels() - assert logging.DEBUG in levels_logged - assert logging.INFO not in levels_logged - assert logging.WARNING not in levels_logged + # second round + await _itis_vip_syncer_service.sync_resources_with_licensed_items( + client.app, categories + ) From 322e5cfb755e7fde3058941ac69b05bbe581b75a Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 5 Feb 2025 10:06:39 +0100 Subject: [PATCH 46/46] 1.part of spliting licensed_items to licenses + licensed_resources --- .../src/models_library/licenses.py | 51 +++ .../2215301c2496_add_licenses_table.py | 235 +++++++++++++ .../models/license_to_resource.py | 34 ++ .../models/licensed_items.py | 17 +- .../models/licenses.py | 60 ++++ .../licenses/_common/models.py | 6 + .../licenses/_licensed_items_repository.py | 124 +++++-- .../licenses/_licenses_repository.py | 317 ++++++++++++++++++ .../licenses/_licenses_service.py | 46 +++ .../licenses/errors.py | 4 + .../unit/with_dbs/04/licenses/conftest.py | 6 +- .../04/licenses/test_licenses_repository.py | 135 ++++++++ 12 files changed, 991 insertions(+), 44 deletions(-) create mode 100644 packages/models-library/src/models_library/licenses.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/2215301c2496_add_licenses_table.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/license_to_resource.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/licenses.py create mode 100644 services/web/server/src/simcore_service_webserver/licenses/_licenses_repository.py create mode 100644 services/web/server/src/simcore_service_webserver/licenses/_licenses_service.py create mode 100644 services/web/server/tests/unit/with_dbs/04/licenses/test_licenses_repository.py diff --git a/packages/models-library/src/models_library/licenses.py b/packages/models-library/src/models_library/licenses.py new file mode 100644 index 00000000000..a59daea4733 --- /dev/null +++ b/packages/models-library/src/models_library/licenses.py @@ -0,0 +1,51 @@ +from datetime import datetime +from typing import TypeAlias +from uuid import UUID + +from pydantic import BaseModel, ConfigDict + +from .licensed_items import LicensedResourceType +from .products import ProductName +from .resource_tracker import PricingPlanId + +LicenseID: TypeAlias = UUID + + +# +# DB +# + + +class LicenseDB(BaseModel): + license_id: LicenseID + display_name: str + licensed_resource_type: LicensedResourceType + pricing_plan_id: PricingPlanId + product_name: ProductName + + # states + created: datetime + modified: datetime + + model_config = ConfigDict(from_attributes=True) + + +class LicenseUpdateDB(BaseModel): + display_name: str | None = None + pricing_plan_id: PricingPlanId | None = None + + +# +# License Domain +# + + +class License(BaseModel): + license_id: LicenseID + display_name: str + licensed_resource_type: LicensedResourceType + resources: list[dict] + pricing_plan_id: PricingPlanId + product_name: ProductName + + model_config = ConfigDict(from_attributes=True) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/2215301c2496_add_licenses_table.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/2215301c2496_add_licenses_table.py new file mode 100644 index 00000000000..5c0d8443614 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/2215301c2496_add_licenses_table.py @@ -0,0 +1,235 @@ +"""add licenses table + +Revision ID: 2215301c2496 +Revises: e71ea59858f4 +Create Date: 2025-02-04 15:26:27.325429+00:00 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "2215301c2496" +down_revision = "e71ea59858f4" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "licensed_resources", + sa.Column( + "licensed_item_id", + postgresql.UUID(as_uuid=True), + server_default=sa.text("gen_random_uuid()"), + nullable=False, + ), + sa.Column("display_name", sa.String(), nullable=False), + sa.Column("licensed_resource_name", sa.String(), nullable=False), + sa.Column( + "licensed_resource_type", + sa.Enum("VIP_MODEL", name="licensedresourcetype"), + nullable=False, + ), + sa.Column( + "licensed_resource_data", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + ), + sa.Column("pricing_plan_id", sa.BigInteger(), nullable=True), + sa.Column("product_name", sa.String(), nullable=True), + sa.Column( + "created", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "trashed", + sa.DateTime(timezone=True), + nullable=True, + comment="The date and time when the licensed_item was marked as trashed. Null if the licensed_item has not been trashed [default].", + ), + sa.ForeignKeyConstraint( + ["pricing_plan_id"], + ["resource_tracker_pricing_plans.pricing_plan_id"], + name="fk_resource_tracker_license_packages_pricing_plan_id", + onupdate="CASCADE", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["product_name"], + ["products.name"], + name="fk_resource_tracker_license_packages_product_name", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("licensed_item_id"), + # sa.UniqueConstraint( + # "licensed_resource_name", + # "licensed_resource_type", + # name="uq_licensed_resource_name_type", + # ), + ) + op.create_table( + "licenses", + sa.Column( + "license_id", + postgresql.UUID(as_uuid=True), + server_default=sa.text("gen_random_uuid()"), + nullable=False, + ), + sa.Column("display_name", sa.String(), nullable=False), + sa.Column( + "licensed_resource_type", + sa.Enum("VIP_MODEL", name="licensedresourcetype"), + nullable=False, + ), + sa.Column("pricing_plan_id", sa.BigInteger(), nullable=False), + sa.Column("product_name", sa.String(), nullable=False), + sa.Column( + "created", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["pricing_plan_id"], + ["resource_tracker_pricing_plans.pricing_plan_id"], + name="fk_resource_tracker_license_packages_pricing_plan_id", + onupdate="CASCADE", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["product_name"], + ["products.name"], + name="fk_resource_tracker_license_packages_product_name", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("license_id"), + ) + op.create_table( + "license_to_resource", + sa.Column("license_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("licensed_item_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column( + "created", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["license_id"], + ["licenses.license_id"], + name="fk_license_to_resource_license_id", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["licensed_item_id"], + ["licensed_resources.licensed_item_id"], + name="fk_license_to_resource_licensed_item_id", + onupdate="CASCADE", + ondelete="CASCADE", + ), + ) + op.drop_table("licensed_items") + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "licensed_items", + sa.Column( + "licensed_item_id", + postgresql.UUID(), + server_default=sa.text("gen_random_uuid()"), + autoincrement=False, + nullable=False, + ), + sa.Column( + "licensed_resource_name", sa.VARCHAR(), autoincrement=False, nullable=False + ), + sa.Column( + "licensed_resource_type", + postgresql.ENUM("VIP_MODEL", name="licensedresourcetype"), + autoincrement=False, + nullable=False, + ), + sa.Column("pricing_plan_id", sa.BIGINT(), autoincrement=False, nullable=True), + sa.Column("product_name", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column( + "created", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column( + "modified", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column( + "licensed_resource_data", + postgresql.JSONB(astext_type=sa.Text()), + autoincrement=False, + nullable=True, + ), + sa.Column( + "trashed", + postgresql.TIMESTAMP(timezone=True), + autoincrement=False, + nullable=True, + comment="The date and time when the licensed_item was marked as trashed. Null if the licensed_item has not been trashed [default].", + ), + sa.Column("display_name", sa.VARCHAR(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint( + ["pricing_plan_id"], + ["resource_tracker_pricing_plans.pricing_plan_id"], + name="fk_resource_tracker_license_packages_pricing_plan_id", + onupdate="CASCADE", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["product_name"], + ["products.name"], + name="fk_resource_tracker_license_packages_product_name", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("licensed_item_id", name="licensed_items_pkey"), + sa.UniqueConstraint( + "licensed_resource_name", + "licensed_resource_type", + name="uq_licensed_resource_name_type", + ), + ) + op.drop_table("license_to_resource") + op.drop_table("licenses") + op.drop_table("licensed_resources") + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/license_to_resource.py b/packages/postgres-database/src/simcore_postgres_database/models/license_to_resource.py new file mode 100644 index 00000000000..09813c5816c --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/license_to_resource.py @@ -0,0 +1,34 @@ +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from ._common import RefActions, column_created_datetime, column_modified_datetime +from .base import metadata + +license_to_resource = sa.Table( + "license_to_resource", + metadata, + sa.Column( + "license_id", + postgresql.UUID(as_uuid=True), + sa.ForeignKey( + "licenses.license_id", + name="fk_license_to_resource_license_id", + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + ), + nullable=False, + ), + sa.Column( + "licensed_item_id", # <-- This will be renamed to "licensed_resource_id" + postgresql.UUID(as_uuid=True), + sa.ForeignKey( + "licensed_resources.licensed_item_id", # <-- This will be renamed to "licensed_resource_id" + name="fk_license_to_resource_licensed_item_id", + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + ), + nullable=False, + ), + column_created_datetime(timezone=True), + column_modified_datetime(timezone=True), +) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py b/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py index f625a2d1be6..149abd2e681 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py @@ -1,8 +1,3 @@ -""" resource_tracker_service_runs table -""" - -import enum - import sqlalchemy as sa from sqlalchemy.dialects import postgresql @@ -13,17 +8,13 @@ column_trashed_datetime, ) from .base import metadata +from .licenses import LicensedResourceType - -class LicensedResourceType(str, enum.Enum): - VIP_MODEL = "VIP_MODEL" - - -licensed_items = sa.Table( - "licensed_items", +licensed_resources = sa.Table( + "licensed_resources", # <-- This will be renamed to "licensed_resources" metadata, sa.Column( - "licensed_item_id", + "licensed_item_id", # <-- This will be renamed to "licensed_resource_id" postgresql.UUID(as_uuid=True), nullable=False, primary_key=True, diff --git a/packages/postgres-database/src/simcore_postgres_database/models/licenses.py b/packages/postgres-database/src/simcore_postgres_database/models/licenses.py new file mode 100644 index 00000000000..14af962dc09 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/licenses.py @@ -0,0 +1,60 @@ +import enum + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from ._common import RefActions, column_created_datetime, column_modified_datetime +from .base import metadata + + +class LicensedResourceType(str, enum.Enum): + VIP_MODEL = "VIP_MODEL" + + +licenses = sa.Table( + "licenses", + metadata, + sa.Column( + "license_id", + postgresql.UUID(as_uuid=True), + nullable=False, + primary_key=True, + server_default=sa.text("gen_random_uuid()"), + ), + sa.Column( + "display_name", + sa.String, + nullable=False, + doc="Display name for front-end", + ), + sa.Column( + "licensed_resource_type", + sa.Enum(LicensedResourceType), + nullable=False, + doc="Resource type, ex. VIP_MODEL", + ), + sa.Column( + "pricing_plan_id", + sa.BigInteger, + sa.ForeignKey( + "resource_tracker_pricing_plans.pricing_plan_id", + name="fk_resource_tracker_license_packages_pricing_plan_id", + onupdate=RefActions.CASCADE, + ondelete=RefActions.RESTRICT, + ), + nullable=False, + ), + sa.Column( + "product_name", + sa.String, + sa.ForeignKey( + "products.name", + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_resource_tracker_license_packages_product_name", + ), + nullable=False, + ), + column_created_datetime(timezone=True), + column_modified_datetime(timezone=True), +) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_common/models.py b/services/web/server/src/simcore_service_webserver/licenses/_common/models.py index 887a6db6f59..0207cd6b99e 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_common/models.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_common/models.py @@ -8,6 +8,7 @@ LicensedItemID, LicensedResourceType, ) +from models_library.licenses import License from models_library.resource_tracker import PricingPlanId, PricingUnitId from models_library.resource_tracker_licensed_items_purchases import ( LicensedItemPurchaseID, @@ -67,6 +68,11 @@ class LicensedItemPage(NamedTuple): total: PositiveInt +class LicensePage(NamedTuple): + items: list[License] + total: PositiveInt + + class LicensedItemsRequestContext(RequestParameters): user_id: UserID = Field(..., alias=RQT_USERID_KEY) # type: ignore[literal-required] product_name: str = Field(..., alias=RQ_PRODUCT_KEY) # type: ignore[literal-required] diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py index 4ecc41a7fc8..a6c107778b7 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_licensed_items_repository.py @@ -18,7 +18,7 @@ from models_library.resource_tracker import PricingPlanId from models_library.rest_ordering import OrderBy, OrderDirection from pydantic import NonNegativeInt -from simcore_postgres_database.models.licensed_items import licensed_items +from simcore_postgres_database.models.licensed_items import licensed_resources from simcore_postgres_database.utils_repos import ( get_columns_from_db_model, pass_or_acquire_connection, @@ -35,7 +35,7 @@ _logger = logging.getLogger(__name__) -_SELECTION_ARGS = get_columns_from_db_model(licensed_items, LicensedItemDB) +_SELECTION_ARGS = get_columns_from_db_model(licensed_resources, LicensedItemDB) def _create_insert_query( @@ -47,7 +47,7 @@ def _create_insert_query( pricing_plan_id: PricingPlanId | None, ) -> postgresql.Insert: return ( - postgresql.insert(licensed_items) + postgresql.insert(licensed_resources) .values( licensed_resource_name=licensed_resource_name, licensed_resource_type=licensed_resource_type, @@ -115,8 +115,11 @@ async def create_if_not_exists( if row is None: select_query = select(*_SELECTION_ARGS).where( - (licensed_items.c.licensed_resource_name == licensed_resource_name) - & (licensed_items.c.licensed_resource_type == licensed_resource_type) + (licensed_resources.c.licensed_resource_name == licensed_resource_name) + & ( + licensed_resources.c.licensed_resource_type + == licensed_resource_type + ) ) result = await conn.execute(select_query) @@ -141,25 +144,25 @@ async def list_( base_query = ( select(*_SELECTION_ARGS) - .select_from(licensed_items) - .where(licensed_items.c.product_name == product_name) + .select_from(licensed_resources) + .where(licensed_resources.c.product_name == product_name) ) # Apply trashed filter if trashed == "exclude": - base_query = base_query.where(licensed_items.c.trashed.is_(None)) + base_query = base_query.where(licensed_resources.c.trashed.is_(None)) elif trashed == "only": - base_query = base_query.where(licensed_items.c.trashed.is_not(None)) + base_query = base_query.where(licensed_resources.c.trashed.is_not(None)) if inactive == "only": base_query = base_query.where( - licensed_items.c.product_name.is_(None) - | licensed_items.c.licensed_item_id.is_(None) + licensed_resources.c.product_name.is_(None) + | licensed_resources.c.licensed_item_id.is_(None) ) elif inactive == "exclude": base_query = base_query.where( - licensed_items.c.product_name.is_not(None) - & licensed_items.c.licensed_item_id.is_not(None) + licensed_resources.c.product_name.is_not(None) + & licensed_resources.c.licensed_item_id.is_not(None) ) # Select total count from base_query @@ -168,10 +171,12 @@ async def list_( # Ordering and pagination if order_by.direction == OrderDirection.ASC: - list_query = base_query.order_by(asc(getattr(licensed_items.c, order_by.field))) + list_query = base_query.order_by( + asc(getattr(licensed_resources.c, order_by.field)) + ) else: list_query = base_query.order_by( - desc(getattr(licensed_items.c, order_by.field)) + desc(getattr(licensed_resources.c, order_by.field)) ) list_query = list_query.offset(offset).limit(limit) @@ -195,10 +200,10 @@ async def get( ) -> LicensedItemDB: select_query = ( select(*_SELECTION_ARGS) - .select_from(licensed_items) + .select_from(licensed_resources) .where( - (licensed_items.c.licensed_item_id == licensed_item_id) - & (licensed_items.c.product_name == product_name) + (licensed_resources.c.licensed_item_id == licensed_item_id) + & (licensed_resources.c.product_name == product_name) ) ) @@ -218,8 +223,8 @@ async def get_by_resource_identifier( licensed_resource_type: LicensedResourceType, ) -> LicensedItemDB: select_query = select(*_SELECTION_ARGS).where( - (licensed_items.c.licensed_resource_name == licensed_resource_name) - & (licensed_items.c.licensed_resource_type == licensed_resource_type) + (licensed_resources.c.licensed_resource_name == licensed_resource_name) + & (licensed_resources.c.licensed_resource_type == licensed_resource_type) ) async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: @@ -245,21 +250,21 @@ async def update( # NOTE: at least 'touch' if updated_values is empty _updates = { **updates.model_dump(exclude_unset=True), - licensed_items.c.modified.name: func.now(), + licensed_resources.c.modified.name: func.now(), } # trashing assert "trash" in dict(LicensedItemUpdateDB.model_fields) # nosec if trash := _updates.pop("trash", None): - _updates[licensed_items.c.trashed.name] = func.now() if trash else None + _updates[licensed_resources.c.trashed.name] = func.now() if trash else None async with transaction_context(get_asyncpg_engine(app), connection) as conn: result = await conn.execute( - licensed_items.update() + licensed_resources.update() .values(**_updates) .where( - (licensed_items.c.licensed_item_id == licensed_item_id) - & (licensed_items.c.product_name == product_name) + (licensed_resources.c.licensed_item_id == licensed_item_id) + & (licensed_resources.c.product_name == product_name) ) .returning(*_SELECTION_ARGS) ) @@ -278,8 +283,71 @@ async def delete( ) -> None: async with transaction_context(get_asyncpg_engine(app), connection) as conn: await conn.execute( - licensed_items.delete().where( - (licensed_items.c.licensed_item_id == licensed_item_id) - & (licensed_items.c.product_name == product_name) + licensed_resources.delete().where( + (licensed_resources.c.licensed_item_id == licensed_item_id) + & (licensed_resources.c.product_name == product_name) ) ) + + +#### DOMAIN MODEL + + +# async def list_licensed_resources_v2( +# app: web.Application, +# connection: AsyncConnection | None = None, +# *, +# product_name: ProductName, +# offset: NonNegativeInt, +# limit: NonNegativeInt, +# order_by: OrderBy, +# ) -> tuple[int, list[License]]: + +# base_query = ( +# select(*_SELECTION_ARGS) +# .select_from(licenses) +# .where(licenses.c.product_name == product_name) +# ) + +# # Select total count from base_query +# subquery = base_query.subquery() +# count_query = select(func.count()).select_from(subquery) + +# # Ordering and pagination +# if order_by.direction == OrderDirection.ASC: +# list_query = base_query.order_by(asc(getattr(licenses.c, order_by.field))) +# else: +# list_query = base_query.order_by(desc(getattr(licenses.c, order_by.field))) +# list_query = list_query.offset(offset).limit(limit) + +# async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: +# total_count = await conn.scalar(count_query) + +# result = await conn.stream(list_query) +# items: list[License] = [License.model_validate(row) async for row in result] + +# return cast(int, total_count), items + + +# async def get_licensed_resource_v2( +# app: web.Application, +# connection: AsyncConnection | None = None, +# *, +# license_id: LicenseID, +# product_name: ProductName, +# ) -> License: +# base_query = ( +# select(*_SELECTION_ARGS) +# .select_from(licenses) +# .where( +# (licenses.c.license_id == license_id) +# & (licenses.c.product_name == product_name) +# ) +# ) + +# async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: +# result = await conn.stream(base_query) +# row = await result.first() +# if row is None: +# raise LicenseNotFoundError(license_id=license_id) +# return License.model_validate(row) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licenses_repository.py b/services/web/server/src/simcore_service_webserver/licenses/_licenses_repository.py new file mode 100644 index 00000000000..27a97b5ac42 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/_licenses_repository.py @@ -0,0 +1,317 @@ +""" Database API + + - Adds a layer to the postgres API with a focus on the projects comments + +""" + +import logging +from typing import cast + +from aiohttp import web +from models_library.licensed_items import LicensedItemID, LicensedResourceType +from models_library.licenses import License, LicenseDB, LicenseID, LicenseUpdateDB +from models_library.products import ProductName +from models_library.resource_tracker import PricingPlanId +from models_library.rest_ordering import OrderBy, OrderDirection +from pydantic import NonNegativeInt +from simcore_postgres_database.models.license_to_resource import license_to_resource +from simcore_postgres_database.models.licensed_items import licensed_resources +from simcore_postgres_database.models.licenses import licenses +from simcore_postgres_database.utils_repos import ( + get_columns_from_db_model, + pass_or_acquire_connection, + transaction_context, +) +from sqlalchemy import asc, desc, func +from sqlalchemy.ext.asyncio import AsyncConnection +from sqlalchemy.sql import select + +from ..db.plugin import get_asyncpg_engine +from .errors import LicenseNotFoundError + +_logger = logging.getLogger(__name__) + + +_SELECTION_ARGS = get_columns_from_db_model(licenses, LicenseDB) + + +async def create( + app: web.Application, + connection: AsyncConnection | None = None, + *, + display_name: str, + licensed_resource_type: LicensedResourceType, + product_name: ProductName, + pricing_plan_id: PricingPlanId, +) -> LicenseDB: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.execute( + licenses.insert() + .values( + product_name=product_name, + display_name=display_name, + licensed_resource_type=licensed_resource_type, + pricing_plan_id=pricing_plan_id, + created=func.now(), + modified=func.now(), + ) + .returning(*_SELECTION_ARGS) + ) + row = result.one() + return LicenseDB.model_validate(row) + + +async def list_( + app: web.Application, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + offset: NonNegativeInt, + limit: NonNegativeInt, + order_by: OrderBy, +) -> tuple[int, list[LicenseDB]]: + + base_query = ( + select(*_SELECTION_ARGS) + .select_from(licenses) + .where(licenses.c.product_name == product_name) + ) + + # Select total count from base_query + subquery = base_query.subquery() + count_query = select(func.count()).select_from(subquery) + + # Ordering and pagination + if order_by.direction == OrderDirection.ASC: + list_query = base_query.order_by(asc(getattr(licenses.c, order_by.field))) + else: + list_query = base_query.order_by(desc(getattr(licenses.c, order_by.field))) + list_query = list_query.offset(offset).limit(limit) + + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + total_count = await conn.scalar(count_query) + + result = await conn.stream(list_query) + items: list[LicenseDB] = [LicenseDB.model_validate(row) async for row in result] + + return cast(int, total_count), items + + +async def get( + app: web.Application, + connection: AsyncConnection | None = None, + *, + license_id: LicenseID, + product_name: ProductName, +) -> LicenseDB: + base_query = ( + select(*_SELECTION_ARGS) + .select_from(licenses) + .where( + (licenses.c.license_id == license_id) + & (licenses.c.product_name == product_name) + ) + ) + + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream(base_query) + row = await result.first() + if row is None: + raise LicenseNotFoundError(license_id=license_id) + return LicenseDB.model_validate(row) + + +async def update( + app: web.Application, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + license_id: LicenseID, + updates: LicenseUpdateDB, +) -> LicenseDB: + # NOTE: at least 'touch' if updated_values is empty + _updates = { + **updates.model_dump(exclude_unset=True), + licenses.c.modified.name: func.now(), + } + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.execute( + licenses.update() + .values(**_updates) + .where( + (licenses.c.license_id == license_id) + & (licenses.c.product_name == product_name) + ) + .returning(*_SELECTION_ARGS) + ) + row = result.one_or_none() + if row is None: + raise LicenseNotFoundError(license_id=license_id) + return LicenseDB.model_validate(row) + + +async def delete( + app: web.Application, + connection: AsyncConnection | None = None, + *, + license_id: LicenseID, + product_name: ProductName, +) -> None: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.execute( + licenses.delete().where( + (licenses.c.license_id == license_id) + & (licenses.c.product_name == product_name) + ) + ) + + +#### DOMAIN MODEL + + +async def list_licenses( + app: web.Application, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + offset: NonNegativeInt, + limit: NonNegativeInt, + order_by: OrderBy, +) -> tuple[int, list[License]]: + + licensed_resources_subquery = ( + select( + license_to_resource.c.license_id, + func.jsonb_agg(licensed_resources.c.licensed_resource_data).label( + "resources" + ), + ) + .select_from( + license_to_resource.join( + licensed_resources, + license_to_resource.c.licensed_item_id + == licensed_resources.c.licensed_item_id, + ) + ) + .group_by(license_to_resource.c.license_id) + ).subquery("licensed_resources_subquery") + + base_query = ( + select( + *_SELECTION_ARGS, + licensed_resources_subquery.c.resources, + ) + .select_from( + licenses.join( + licensed_resources_subquery, + licenses.c.license_id == licensed_resources_subquery.c.license_id, + ) + ) + .where(licenses.c.product_name == product_name) + ) + + # Select total count from base_query + subquery = base_query.subquery() + count_query = select(func.count()).select_from(subquery) + + # Ordering and pagination + if order_by.direction == OrderDirection.ASC: + list_query = base_query.order_by(asc(getattr(licenses.c, order_by.field))) + else: + list_query = base_query.order_by(desc(getattr(licenses.c, order_by.field))) + list_query = list_query.offset(offset).limit(limit) + + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + total_count = await conn.scalar(count_query) + + result = await conn.stream(list_query) + items: list[License] = [License.model_validate(row) async for row in result] + + return cast(int, total_count), items + + +async def get_license( + app: web.Application, + connection: AsyncConnection | None = None, + *, + license_id: LicenseID, + product_name: ProductName, +) -> License: + licensed_resources_subquery = ( + select( + license_to_resource.c.license_id, + func.jsonb_agg(licensed_resources.c.licensed_resource_data).label( + "resources" + ), + ) + .select_from( + license_to_resource.join( + licensed_resources, + license_to_resource.c.licensed_item_id + == licensed_resources.c.licensed_item_id, + ) + ) + .where(license_to_resource.c.license_id == license_id) + .group_by(license_to_resource.c.license_id) + ).subquery("licensed_resources_subquery") + + base_query = ( + select( + *_SELECTION_ARGS, + licensed_resources_subquery.c.resources, + ) + .select_from( + licenses.join( + licensed_resources_subquery, + licenses.c.license_id == licensed_resources_subquery.c.license_id, + ) + ) + .where( + (licenses.c.product_name == product_name) + & (licenses.c.license_id == license_id) + ) + ) + + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream(base_query) + row = await result.first() + if row is None: + raise LicenseNotFoundError(license_id=license_id) + return License.model_validate(row) + + +### License to Resource + + +async def add_licensed_resource_to_license( + app: web.Application, + connection: AsyncConnection | None = None, + *, + licensed_item_id: LicensedItemID, + license_id: LicenseID, +) -> None: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.execute( + license_to_resource.insert().values( + license_id=license_id, + licensed_item_id=licensed_item_id, + created=func.now(), + modified=func.now(), + ) + ) + + +async def delete_licensed_resource_from_license( + app: web.Application, + connection: AsyncConnection | None = None, + *, + licensed_item_id: LicensedItemID, + license_id: LicenseID, +) -> None: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.execute( + license_to_resource.delete().where( + (license_to_resource.c.license_id == license_id) + & (license_to_resource.c.licensed_item_id == licensed_item_id) + ) + ) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_licenses_service.py b/services/web/server/src/simcore_service_webserver/licenses/_licenses_service.py new file mode 100644 index 00000000000..a13f542ffbf --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/_licenses_service.py @@ -0,0 +1,46 @@ +# pylint: disable=unused-argument + +import logging + +from aiohttp import web +from models_library.licenses import License, LicenseID +from models_library.products import ProductName +from models_library.rest_ordering import OrderBy +from pydantic import NonNegativeInt + +from . import _licenses_repository +from ._common.models import LicensePage + +_logger = logging.getLogger(__name__) + + +async def get_license( + app: web.Application, + *, + license_id: LicenseID, + product_name: ProductName, +) -> License: + return await _licenses_repository.get_license( + app, license_id=license_id, product_name=product_name + ) + + +async def list_licenses( + app: web.Application, + *, + product_name: ProductName, + offset: NonNegativeInt, + limit: int, + order_by: OrderBy, +) -> LicensePage: + total_count, items = await _licenses_repository.list_licenses( + app, + product_name=product_name, + offset=offset, + limit=limit, + order_by=order_by, + ) + return LicensePage( + items=items, + total=total_count, + ) diff --git a/services/web/server/src/simcore_service_webserver/licenses/errors.py b/services/web/server/src/simcore_service_webserver/licenses/errors.py index 18c57966123..f4238b8029b 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/errors.py +++ b/services/web/server/src/simcore_service_webserver/licenses/errors.py @@ -5,6 +5,10 @@ class LicensesValueError(WebServerBaseError, ValueError): ... +class LicenseNotFoundError(LicensesValueError): + msg_template = "License {license_id} not found" + + class LicensedItemNotFoundError(LicensesValueError): msg_template = "License good {licensed_item_id} not found" diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py b/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py index ffe1a6f338c..4e611d31561 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py @@ -5,7 +5,7 @@ # pylint:disable=redefined-outer-name import pytest from aiohttp.test_utils import TestClient -from simcore_postgres_database.models.licensed_items import licensed_items +from simcore_postgres_database.models.licensed_items import licensed_resources from simcore_postgres_database.models.resource_tracker_pricing_plans import ( resource_tracker_pricing_plans, ) @@ -40,7 +40,7 @@ async def pricing_plan_id( yield int(row[0]) async with transaction_context(get_asyncpg_engine(client.app)) as conn: - await conn.execute(licensed_items.delete()) + await conn.execute(licensed_resources.delete()) await conn.execute(resource_tracker_pricing_plans.delete()) @@ -49,7 +49,7 @@ async def ensure_empty_licensed_items(client: TestClient): async def _cleanup(): assert client.app async with transaction_context(get_asyncpg_engine(client.app)) as conn: - await conn.execute(licensed_items.delete()) + await conn.execute(await conn.execute(licensed_resources.delete()).delete()) await _cleanup() diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_licenses_repository.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_licenses_repository.py new file mode 100644 index 00000000000..f64f5082921 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_licenses_repository.py @@ -0,0 +1,135 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments +# pylint: disable=too-many-statements + +import pytest +from aiohttp.test_utils import TestClient +from models_library.licensed_items import VIP_DETAILS_EXAMPLE, LicensedResourceType +from models_library.licenses import License, LicenseDB, LicenseUpdateDB +from models_library.rest_ordering import OrderBy +from pytest_simcore.helpers.webserver_login import UserInfoDict +from simcore_service_webserver.db.models import UserRole +from simcore_service_webserver.licenses import ( + _licensed_items_repository, + _licenses_repository, +) +from simcore_service_webserver.licenses.errors import LicenseNotFoundError +from simcore_service_webserver.projects.models import ProjectDict + + +@pytest.fixture +def user_role() -> UserRole: + return UserRole.USER + + +async def test_licenses_db_crud( + client: TestClient, + logged_user: UserInfoDict, + user_project: ProjectDict, + osparc_product_name: str, + pricing_plan_id: int, +): + got = await _licensed_items_repository.create( + client.app, + product_name=osparc_product_name, + display_name="Model A Display Name", + licensed_resource_name="Model A", + licensed_resource_type=LicensedResourceType.VIP_MODEL, + licensed_resource_data=VIP_DETAILS_EXAMPLE, + pricing_plan_id=pricing_plan_id, + ) + licensed_item_id = got.licensed_item_id + + ### NEW: + + got = await _licenses_repository.create( + client.app, + product_name=osparc_product_name, + display_name="Model A", + licensed_resource_type=LicensedResourceType.VIP_MODEL, + pricing_plan_id=pricing_plan_id, + ) + license_id = got.license_id + + total_count, items = await _licenses_repository.list_( + client.app, + product_name=osparc_product_name, + offset=0, + limit=10, + order_by=OrderBy(field="display_name"), + ) + assert total_count == 1 + assert items[0].license_id == license_id + + got = await _licenses_repository.get( + client.app, + license_id=license_id, + product_name=osparc_product_name, + ) + assert isinstance(got, LicenseDB) + assert got.display_name == "Model A" + + got = await _licenses_repository.update( + client.app, + license_id=license_id, + product_name=osparc_product_name, + updates=LicenseUpdateDB(display_name="Model B"), + ) + assert isinstance(got, LicenseDB) + + got = await _licenses_repository.get( + client.app, + license_id=license_id, + product_name=osparc_product_name, + ) + assert isinstance(got, LicenseDB) + assert got.display_name == "Model B" + + # CONNECT RESOURCE TO LICENSE + + await _licenses_repository.add_licensed_resource_to_license( + client.app, + license_id=license_id, + licensed_item_id=licensed_item_id, + ) + + got = await _licenses_repository.list_licenses( + client.app, + product_name=osparc_product_name, + offset=0, + limit=10, + order_by=OrderBy(field="display_name"), + ) + assert got[0] == 1 + assert isinstance(got[1], list) + assert isinstance(got[1][0], License) + + got = await _licenses_repository.get_license( + client.app, + product_name=osparc_product_name, + license_id=license_id, + ) + assert isinstance(got, License) + + # DELETE + + await _licenses_repository.delete_licensed_resource_from_license( + client.app, + license_id=license_id, + licensed_item_id=licensed_item_id, + ) + + got = await _licenses_repository.delete( + client.app, + license_id=license_id, + product_name=osparc_product_name, + ) + + with pytest.raises(LicenseNotFoundError): + await _licenses_repository.get( + client.app, + license_id=license_id, + product_name=osparc_product_name, + )