migrated aiopg -> aymcpg in pg products

Author: pcrespov

packages/postgres-database/tests/products/conftest.py

@@ -7,15 +7,14 @@
 from collections.abc import Callable
 
 import pytest
-from aiopg.sa.exc import ResourceClosedError
 from faker import Faker
 from pytest_simcore.helpers.faker_factories import random_product
 from simcore_postgres_database.webserver_models import products
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 
 
 @pytest.fixture
-def products_regex() -> dict:
+def products_regex() -> dict[str, str]:
     return {
         "s4l": r"(^s4l[\.-])|(^sim4life\.)",
         "osparc": r"^osparc.",
@@ -24,19 +23,20 @@ def products_regex() -> dict:
 
 
 @pytest.fixture
-def products_names(products_regex: dict) -> list[str]:
+def products_names(products_regex: dict[str, str]) -> list[str]:
     return list(products_regex)
 
 
 @pytest.fixture
-def make_products_table(products_regex: dict, faker: Faker) -> Callable:
+def make_products_table(products_regex: dict[str, str], faker: Faker) -> Callable:
     async def _make(conn) -> None:
         for n, (name, regex) in enumerate(products_regex.items()):
 
             result = await conn.execute(
                 pg_insert(products)
                 .values(
                     **random_product(
+                        fake=faker,
                         name=name,
                         display_name=f"Product {name.capitalize()}",
                         short_name=name[:3].lower(),
@@ -45,6 +45,7 @@ async def _make(conn) -> None:
                     )
                 )
                 .on_conflict_do_update(
+                    # osparc might be already injected as default!
                     index_elements=[products.c.name],
                     set_={
                         "display_name": f"Product {name.capitalize()}",
@@ -55,9 +56,7 @@ async def _make(conn) -> None:
                 )
             )
 
-            assert result.closed
+            assert not result.closed
             assert not result.returns_rows
-            with pytest.raises(ResourceClosedError):
-                await result.scalar()
 
     return _make

packages/postgres-database/tests/products/test_models_products.py

@@ -3,15 +3,10 @@
 # pylint: disable=redefined-outer-name
 # pylint: disable=unused-argument
 
-
-import json
 from collections.abc import Callable
 from pathlib import Path
-from pprint import pprint
 
 import sqlalchemy as sa
-from aiopg.sa.engine import Engine
-from aiopg.sa.result import ResultProxy, RowProxy
 from simcore_postgres_database.models.jinja2_templates import jinja2_templates
 from simcore_postgres_database.models.products import (
     EmailFeedback,
@@ -23,40 +18,37 @@
 )
 from simcore_postgres_database.webserver_models import products
 from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.ext.asyncio import AsyncEngine
 
 
 async def test_load_products(
-    aiopg_engine: Engine, make_products_table: Callable, products_regex: dict
+    asyncpg_engine: AsyncEngine, make_products_table: Callable, products_regex: dict
 ):
     exclude = {
         products.c.created,
         products.c.modified,
     }
 
-    async with aiopg_engine.acquire() as conn:
+    async with asyncpg_engine.connect() as conn:
         await make_products_table(conn)
 
         stmt = sa.select(*[c for c in products.columns if c not in exclude])
-        result: ResultProxy = await conn.execute(stmt)
-        assert result.returns_rows
-
-        rows: list[RowProxy] = await result.fetchall()
+        result = await conn.execute(stmt)
+        rows = result.fetchall()
         assert rows
 
-        assert {
-            row[products.c.name]: row[products.c.host_regex] for row in rows
-        } == products_regex
+        assert {row.name: row.host_regex for row in rows} == products_regex
 
 
 async def test_jinja2_templates_table(
-    aiopg_engine: Engine, osparc_simcore_services_dir: Path
+    asyncpg_engine: AsyncEngine, osparc_simcore_services_dir: Path
 ):
     templates_common_dir = (
         osparc_simcore_services_dir
         / "web/server/src/simcore_service_webserver/templates/common"
     )
 
-    async with aiopg_engine.acquire() as conn:
+    async with asyncpg_engine.connect() as conn:
         templates = []
         # templates table
         for p in templates_common_dir.glob("*.jinja2"):
@@ -105,10 +97,9 @@ async def test_jinja2_templates_table(
             products.c.name, jinja2_templates.c.name, products.c.short_name
         ).select_from(j)
 
-        result: ResultProxy = await conn.execute(stmt)
-        assert result.rowcount == 2
-        rows = await result.fetchall()
-        assert sorted(r.as_tuple() for r in rows) == sorted(
+        result = await conn.execute(stmt)
+        rows = result.fetchall()
+        assert sorted(tuple(r) for r in rows) == sorted(
             [
                 ("osparc", "registration_email.jinja2", "osparc"),
                 ("s4l", "registration_email.jinja2", "s4l web"),
@@ -135,7 +126,7 @@ async def test_jinja2_templates_table(
 
 
 async def test_insert_select_product(
-    aiopg_engine: Engine,
+    asyncpg_engine: AsyncEngine,
 ):
     osparc_product = {
         "name": "osparc",
@@ -172,9 +163,7 @@ async def test_insert_select_product(
         ],
     }
 
-    print(json.dumps(osparc_product))
-
-    async with aiopg_engine.acquire() as conn:
+    async with asyncpg_engine.begin() as conn:
         # writes
         stmt = (
             pg_insert(products)
@@ -188,12 +177,9 @@ async def test_insert_select_product(
 
         # reads
         stmt = sa.select(products).where(products.c.name == name)
-        row = await (await conn.execute(stmt)).fetchone()
-        print(row)
+        row = (await conn.execute(stmt)).one_or_none()
         assert row
 
-        pprint(dict(**row))
-
         assert row.manuals
         assert row.manuals == osparc_product["manuals"]
 

packages/postgres-database/tests/products/test_products_to_templates.py

@@ -10,12 +10,12 @@
 
 import pytest
 import sqlalchemy as sa
-from aiopg.sa.connection import SAConnection
 from faker import Faker
 from simcore_postgres_database.models.jinja2_templates import jinja2_templates
 from simcore_postgres_database.models.products import products
 from simcore_postgres_database.models.products_to_templates import products_to_templates
 from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.ext.asyncio import AsyncEngine
 
 
 @pytest.fixture
@@ -48,54 +48,58 @@ def templates_dir(
 
 @pytest.fixture
 async def product_templates_in_db(
-    connection: SAConnection,
+    asyncpg_engine: AsyncEngine,
     make_products_table: Callable,
     products_names: list[str],
     templates_names: list[str],
 ):
-    await make_products_table(connection)
-
-    # one version of all tempaltes
-    for template_name in templates_names:
-        await connection.execute(
-            jinja2_templates.insert().values(
-                name=template_name, content="fake template in database"
+    async with asyncpg_engine.begin() as conn:
+        await make_products_table(conn)
+
+        # one version of all tempaltes
+        for template_name in templates_names:
+            await conn.execute(
+                jinja2_templates.insert().values(
+                    name=template_name, content="fake template in database"
+                )
             )
-        )
 
-        # only even products have templates
-        for product_name in products_names[0::2]:
-            await connection.execute(
-                products_to_templates.insert().values(
-                    template_name=template_name, product_name=product_name
+            # only even products have templates
+            for product_name in products_names[0::2]:
+                await conn.execute(
+                    products_to_templates.insert().values(
+                        template_name=template_name, product_name=product_name
+                    )
                 )
-            )
 
 
 async def test_export_and_import_table(
-    connection: SAConnection,
+    asyncpg_engine: AsyncEngine,
     product_templates_in_db: None,
 ):
-    exported_values = []
-    excluded_names = {"created", "modified", "group_id"}
-    async for row in connection.execute(
-        sa.select(*(c for c in products.c if c.name not in excluded_names))
-    ):
-        assert row
-        exported_values.append(dict(row))
-
-    # now just upsert them
-    for values in exported_values:
-        values["display_name"] += "-changed"
-        await connection.execute(
-            pg_insert(products)
-            .values(**values)
-            .on_conflict_do_update(index_elements=[products.c.name], set_=values)
+
+    async with asyncpg_engine.connect() as connection:
+        exported_values = []
+        excluded_names = {"created", "modified", "group_id"}
+        result = await connection.stream(
+            sa.select(*(c for c in products.c if c.name not in excluded_names))
         )
+        async for row in result:
+            assert row
+            exported_values.append(row._asdict())
+
+        # now just upsert them
+        for values in exported_values:
+            values["display_name"] += "-changed"
+            await connection.execute(
+                pg_insert(products)
+                .values(**values)
+                .on_conflict_do_update(index_elements=[products.c.name], set_=values)
+            )
 
 
 async def test_create_templates_products_folder(
-    connection: SAConnection,
+    asyncpg_engine: AsyncEngine,
     templates_dir: Path,
     products_names: list[str],
     tmp_path: Path,
@@ -121,20 +125,22 @@ async def test_create_templates_products_folder(
                     shutil.copy(p, product_folder / p.name, follow_symlinks=False)
 
         # overrides if with files in database
-        async for row in connection.execute(
-            sa.select(
-                products_to_templates.c.product_name,
-                jinja2_templates.c.name,
-                jinja2_templates.c.content,
+        async with asyncpg_engine.connect() as conn:
+            result = await conn.stream(
+                sa.select(
+                    products_to_templates.c.product_name,
+                    jinja2_templates.c.name,
+                    jinja2_templates.c.content,
+                )
+                .select_from(products_to_templates.join(jinja2_templates))
+                .where(products_to_templates.c.product_name == product_name)
             )
-            .select_from(products_to_templates.join(jinja2_templates))
-            .where(products_to_templates.c.product_name == product_name)
-        ):
-            assert row
 
-            template_path = product_folder / row.name
-            template_path.write_text(row.content)
+            async for row in result:
+                assert row
+                template_path = product_folder / row.name
+                template_path.write_text(row.content)
 
-        assert sorted(
-            product_folder / template_name for template_name in templates_names
-        ) == sorted(product_folder.rglob("*.*"))
+            assert sorted(
+                product_folder / template_name for template_name in templates_names
+            ) == sorted(product_folder.rglob("*.*"))

packages/postgres-database/tests/products/test_utils_products.py

@@ -3,7 +3,6 @@
 # pylint: disable=redefined-outer-name
 # pylint: disable=unused-argument
 
-import asyncio
 from collections.abc import Callable
 
 import pytest
@@ -13,7 +12,6 @@
 from simcore_postgres_database.utils_products import (
     execute_get_or_create_product_group,
     get_default_product_name,
-    get_or_create_product_group,
     get_product_group_id,
 )
 from sqlalchemy.ext.asyncio import AsyncEngine
@@ -22,7 +20,7 @@
 async def test_default_product(
     asyncpg_engine: AsyncEngine, make_products_table: Callable
 ):
-    async with asyncpg_engine.connect() as conn:
+    async with asyncpg_engine.begin() as conn:
         await make_products_table(conn)
         default_product = await get_default_product_name(conn)
         assert default_product == "s4l"
@@ -31,7 +29,7 @@ async def test_default_product(
 @pytest.mark.parametrize("pg_sa_engine", ["sqlModels"], indirect=True)
 async def test_default_product_undefined(asyncpg_engine: AsyncEngine):
     async with asyncpg_engine.connect() as conn:
-        with pytest.raises(ValueError):
+        with pytest.raises(ValueError, match="No product"):
             await get_default_product_name(conn)
 
 
@@ -100,32 +98,3 @@ async def test_get_or_create_group_product(
                 conn, product_name=product_row.name
             )
             assert product_group_id is None
-
-
-@pytest.mark.skip(
-    reason="Not relevant. Will review in https://github.com/ITISFoundation/osparc-simcore/issues/3754"
-)
-async def test_get_or_create_group_product_concurrent(
-    asyncpg_engine: AsyncEngine, make_products_table: Callable
-):
-    async with asyncpg_engine.connect() as conn:
-        await make_products_table(conn)
-
-    async def _auto_create_products_groups():
-        async with asyncpg_engine.connect() as conn:
-            async for product_row in await conn.stream(
-                sa.select(products.c.name, products.c.group_id).order_by(
-                    products.c.priority
-                )
-            ):
-                # get or create
-                return await get_or_create_product_group(
-                    conn, product_name=product_row.name
-                )
-            return None
-
-    tasks = [asyncio.create_task(_auto_create_products_groups()) for _ in range(5)]
-
-    results = await asyncio.gather(*tasks)
-
-    assert all(res == results[0] for res in results[1:])

services/catalog/src/simcore_service_catalog/db/repositories/products.py

@@ -5,6 +5,6 @@
 
 class ProductsRepository(BaseRepository):
     async def get_default_product_name(self) -> str:
-        async with self.db_engine.begin() as conn:
+        async with self.db_engine.connect() as conn:
             product_name: str = await get_default_product_name(conn)
             return product_name

services/web/server/src/simcore_service_webserver/products/_repository.py

@@ -68,8 +68,8 @@ async def _get_product_payment_fields(
             min_payment_amount_usd=None,
         )
 
-    assert price_info.usd_per_credit > 0
-    assert price_info.min_payment_amount_usd > 0
+    assert price_info.usd_per_credit > 0  # nosec
+    assert price_info.min_payment_amount_usd > 0  # nosec
 
     return PaymentFieldsTuple(
         enabled=True,