🗃️ New templates to products map table (#5358)

Author: pcrespov

packages/postgres-database/src/simcore_postgres_database/migration/versions/f3a5484fe05d_products_to_templates_map_table.py

@@ -0,0 +1,93 @@
+"""products_to_templates map table
+
+Revision ID: f3a5484fe05d
+Revises: f20f4c9fca71
+Create Date: 2024-02-21 19:33:48.169810+00:00
+
+"""
+from typing import Final
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "f3a5484fe05d"
+down_revision = "f20f4c9fca71"
+branch_labels = None
+depends_on = None
+
+
+# auto-update modified
+# TRIGGERS ------------------------
+_TABLE_NAME: Final[str] = "products_to_templates"
+_TRIGGER_NAME: Final[str] = "trigger_auto_update"  # NOTE: scoped on table
+_PROCEDURE_NAME: Final[
+    str
+] = f"{_TABLE_NAME}_auto_update_modified()"  # NOTE: scoped on database
+
+modified_timestamp_trigger = sa.DDL(
+    f"""
+DROP TRIGGER IF EXISTS {_TRIGGER_NAME} on {_TABLE_NAME};
+CREATE TRIGGER {_TRIGGER_NAME}
+BEFORE INSERT OR UPDATE ON {_TABLE_NAME}
+FOR EACH ROW EXECUTE PROCEDURE {_PROCEDURE_NAME};
+    """
+)
+
+# PROCEDURES ------------------------
+update_modified_timestamp_procedure = sa.DDL(
+    f"""
+CREATE OR REPLACE FUNCTION {_PROCEDURE_NAME}
+RETURNS TRIGGER AS $$
+BEGIN
+  NEW.modified := current_timestamp;
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+    """
+)
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "products_to_templates",
+        sa.Column("product_name", sa.String(), nullable=False),
+        sa.Column("template_name", sa.String(), nullable=True),
+        sa.Column(
+            "created", sa.DateTime(), server_default=sa.text("now()"), nullable=False
+        ),
+        sa.Column(
+            "modified", sa.DateTime(), server_default=sa.text("now()"), nullable=False
+        ),
+        sa.ForeignKeyConstraint(
+            ["product_name"],
+            ["products.name"],
+            name="fk_products_to_templates_product_name",
+            onupdate="CASCADE",
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["template_name"],
+            ["jinja2_templates.name"],
+            name="fk_products_to_templates_template_name",
+            onupdate="CASCADE",
+            ondelete="CASCADE",
+        ),
+        sa.UniqueConstraint("product_name", "template_name"),
+    )
+    # ### end Alembic commands ###
+
+    # custom
+    op.execute(update_modified_timestamp_procedure)
+    op.execute(modified_timestamp_trigger)
+
+
+def downgrade():
+    # custom
+    op.execute(f"DROP TRIGGER IF EXISTS {_TRIGGER_NAME} on {_TABLE_NAME};")
+    op.execute(f"DROP FUNCTION {_PROCEDURE_NAME};")
+
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("products_to_templates")
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/models/_common.py

@@ -34,7 +34,7 @@ def register_modified_datetime_auto_update_trigger(table: sa.Table) -> None:
     update of the 'modified' timestamp column when a row is modified.
 
     NOTE: Add a *hard-coded* version in the alembic migration code!!!
-    see [this example](https://github.com/ITISFoundation/osparc-simcore/blob/78bc54e5815e8be5a8ed6a08a7bbe5591bbd2bd9/packages/postgres-database/src/simcore_postgres_database/migration/versions/e0a2557dec27_add_services_limitations.py)
+    SEE https://github.com/ITISFoundation/osparc-simcore/blob/78bc54e5815e8be5a8ed6a08a7bbe5591bbd2bd9/packages/postgres-database/src/simcore_postgres_database/migration/versions/e0a2557dec27_add_services_limitations.py
 
 
     Arguments:

packages/postgres-database/src/simcore_postgres_database/models/products_to_templates.py

@@ -0,0 +1,43 @@
+import sqlalchemy as sa
+
+from ._common import (
+    column_created_datetime,
+    column_modified_datetime,
+    register_modified_datetime_auto_update_trigger,
+)
+from .base import metadata
+from .jinja2_templates import jinja2_templates
+
+products_to_templates = sa.Table(
+    "products_to_templates",
+    metadata,
+    sa.Column(
+        "product_name",
+        sa.String,
+        sa.ForeignKey(
+            "products.name",
+            onupdate="CASCADE",
+            ondelete="CASCADE",
+            name="fk_products_to_templates_product_name",
+        ),
+        nullable=False,
+    ),
+    sa.Column(
+        "template_name",
+        sa.String,
+        sa.ForeignKey(
+            jinja2_templates.c.name,
+            name="fk_products_to_templates_template_name",
+            ondelete="CASCADE",
+            onupdate="CASCADE",
+        ),
+        nullable=True,
+        doc="Custom jinja2 template",
+    ),
+    # TIME STAMPS ----
+    column_created_datetime(timezone=False),
+    column_modified_datetime(timezone=False),
+    sa.UniqueConstraint("product_name", "template_name"),
+)
+
+register_modified_datetime_auto_update_trigger(products_to_templates)

packages/postgres-database/src/simcore_postgres_database/utils.py

@@ -14,7 +14,7 @@ def build_url(
     user: str = "",
     password: str = "",
     host: str = "127.0.0.1",
-    port: int = 5432,
+    port: int | str = 5432,
     **_kwargs,
 ) -> URL:
     """
@@ -25,7 +25,7 @@ def build_url(
         user=user,
         password=password,
         host=host,
-        port=port,
+        port=int(port),
         path=f"/{database}",
     )
     # _kwargs allows expand on larger dicts without raising exceptions

packages/postgres-database/tests/products/conftest.py

@@ -8,6 +8,8 @@
 
 import pytest
 from aiopg.sa.exc import ResourceClosedError
+from faker import Faker
+from pytest_simcore.helpers.rawdata_fakers import random_product
 from simcore_postgres_database.webserver_models import products
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 
@@ -22,19 +24,25 @@ def products_regex() -> dict:
 
 
 @pytest.fixture
-def make_products_table(
-    products_regex: dict,
-) -> Callable:
+def products_names(products_regex: dict) -> list[str]:
+    return list(products_regex)
+
+
+@pytest.fixture
+def make_products_table(products_regex: dict, faker: Faker) -> Callable:
     async def _make(conn) -> None:
         for n, (name, regex) in enumerate(products_regex.items()):
+
             result = await conn.execute(
                 pg_insert(products)
                 .values(
-                    name=name,
-                    display_name=f"Product {name.capitalize()}",
-                    short_name=name[:3].lower(),
-                    host_regex=regex,
-                    priority=n,
+                    **random_product(
+                        name=name,
+                        display_name=f"Product {name.capitalize()}",
+                        short_name=name[:3].lower(),
+                        host_regex=regex,
+                        priority=n,
+                    )
                 )
                 .on_conflict_do_update(
                     index_elements=[products.c.name],

packages/postgres-database/tests/products/test_products_to_templates.py

@@ -0,0 +1,140 @@
+# pylint: disable=no-name-in-module
+# pylint: disable=no-value-for-parameter
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+
+
+import shutil
+from collections.abc import Callable
+from pathlib import Path
+
+import pytest
+import sqlalchemy as sa
+from aiopg.sa.connection import SAConnection
+from faker import Faker
+from simcore_postgres_database.models.jinja2_templates import jinja2_templates
+from simcore_postgres_database.models.products import products
+from simcore_postgres_database.models.products_to_templates import products_to_templates
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+
+
+@pytest.fixture
+def templates_names(faker: Faker) -> list[str]:
+    return [faker.file_name(extension="html") for _ in range(3)]
+
+
+@pytest.fixture
+def templates_dir(
+    tmp_path: Path, products_names: list[str], templates_names: list[str]
+) -> Path:
+    templates_path = tmp_path / "templates"
+
+    # common keeps default templates
+    (templates_path / "common").mkdir(parents=True)
+    for template_name in templates_names:
+        (templates_path / "common" / template_name).write_text(
+            "Fake template for 'common'"
+        )
+
+    # only odd products have the first template
+    for product_name in products_names[1::2]:
+        (templates_path / product_name).mkdir(parents=True)
+        (templates_path / product_name / templates_names[0]).write_text(
+            f"Fake template for {product_name=}"
+        )
+
+    return templates_path
+
+
+@pytest.fixture
+async def product_templates_in_db(
+    connection: SAConnection,
+    make_products_table: Callable,
+    products_names: list[str],
+    templates_names: list[str],
+):
+    await make_products_table(connection)
+
+    # one version of all tempaltes
+    for template_name in templates_names:
+        await connection.execute(
+            jinja2_templates.insert().values(
+                name=template_name, content="fake template in database"
+            )
+        )
+
+        # only even products have templates
+        for product_name in products_names[0::2]:
+            await connection.execute(
+                products_to_templates.insert().values(
+                    template_name=template_name, product_name=product_name
+                )
+            )
+
+
+async def test_export_and_import_table(
+    connection: SAConnection,
+    product_templates_in_db: None,
+):
+    exported_values = []
+    excluded_names = {"created", "modified", "group_id"}
+    async for row in connection.execute(
+        sa.select(*(c for c in products.c if c.name not in excluded_names))
+    ):
+        assert row
+        exported_values.append(dict(row))
+
+    # now just upsert them
+    for values in exported_values:
+        values["display_name"] += "-changed"
+        await connection.execute(
+            pg_insert(products)
+            .values(**values)
+            .on_conflict_do_update(index_elements=[products.c.name], set_=values)
+        )
+
+
+async def test_create_templates_products_folder(
+    connection: SAConnection,
+    templates_dir: Path,
+    products_names: list[str],
+    tmp_path: Path,
+    templates_names: list[str],
+    product_templates_in_db: None,
+):
+    download_path = tmp_path / "downloaded" / "templates"
+    assert templates_dir != download_path
+
+    for product_name in products_names:
+        product_folder = download_path / product_name
+        product_folder.mkdir(parents=True, exist_ok=True)
+
+        # takes common as defaults
+        for p in (templates_dir / "common").iterdir():
+            if p.is_file():
+                shutil.copy(p, product_folder / p.name, follow_symlinks=False)
+
+        # overrides with customs in-place
+        if (templates_dir / product_name).exists():
+            for p in (templates_dir / product_name).iterdir():
+                if p.is_file():
+                    shutil.copy(p, product_folder / p.name, follow_symlinks=False)
+
+        # overrides if with files in database
+        async for row in connection.execute(
+            sa.select(
+                products_to_templates.c.product_name,
+                jinja2_templates.c.name,
+                jinja2_templates.c.content,
+            )
+            .select_from(products_to_templates.join(jinja2_templates))
+            .where(products_to_templates.c.product_name == product_name)
+        ):
+            assert row
+
+            template_path = product_folder / row.name
+            template_path.write_text(row.content)
+
+        assert sorted(
+            product_folder / template_name for template_name in templates_names
+        ) == sorted(product_folder.rglob("*.*"))