✨ add project tags to RUT listing/export (#6722)

Author: matusdrobuliak66

packages/models-library/src/models_library/api_schemas_resource_usage_tracker/service_runs.py

@@ -20,6 +20,7 @@ class ServiceRunGet(BaseModel):
     user_email: str
     project_id: ProjectID
     project_name: str
+    project_tags: list[str]
     node_id: NodeID
     node_name: str
     root_parent_project_id: ProjectID

packages/postgres-database/src/simcore_postgres_database/migration/versions/8e1f83486be7_enhance_projects_tags_for_rut.py

@@ -0,0 +1,90 @@
+"""enhance projects_tags for RUT
+
+Revision ID: 8e1f83486be7
+Revises: 8bfe65a5e294
+Create Date: 2024-11-15 09:12:57.789183+00:00
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "8e1f83486be7"
+down_revision = "8bfe65a5e294"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "projects_tags", sa.Column("project_uuid_for_rut", sa.String(), nullable=True)
+    )
+
+    # Migrate
+    op.execute(
+        sa.DDL(
+            """
+                UPDATE projects_tags
+                SET project_uuid_for_rut = projects.uuid
+                FROM projects
+                WHERE projects_tags.project_id = projects.id;
+            """
+        )
+    )
+
+    op.alter_column(
+        "projects_tags",
+        "project_uuid_for_rut",
+        existing_type=sa.String(),
+        nullable=False,
+    )
+    op.alter_column(
+        "projects_tags", "project_id", existing_type=sa.BIGINT(), nullable=True
+    )
+    op.drop_constraint(
+        "study_tags_study_id_tag_id_key", "projects_tags", type_="unique"
+    )
+    op.create_unique_constraint(
+        "project_tags_project_uuid_unique",
+        "projects_tags",
+        ["project_uuid_for_rut", "tag_id"],
+    )
+    op.drop_constraint("study_tags_study_id_fkey", "projects_tags", type_="foreignkey")
+    op.create_foreign_key(
+        "project_tags_project_id_fkey",
+        "projects_tags",
+        "projects",
+        ["project_id"],
+        ["id"],
+        onupdate="CASCADE",
+        ondelete="SET NULL",
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(
+        "project_tags_project_id_fkey", "projects_tags", type_="foreignkey"
+    )
+    op.create_foreign_key(
+        "study_tags_study_id_fkey",
+        "projects_tags",
+        "projects",
+        ["project_id"],
+        ["id"],
+        onupdate="CASCADE",
+        ondelete="CASCADE",
+    )
+    op.drop_constraint(
+        "project_tags_project_uuid_unique", "projects_tags", type_="unique"
+    )
+    op.create_unique_constraint(
+        "study_tags_study_id_tag_id_key", "projects_tags", ["project_id", "tag_id"]
+    )
+    op.alter_column(
+        "projects_tags", "project_id", existing_type=sa.BIGINT(), nullable=False
+    )
+    op.drop_column("projects_tags", "project_uuid_for_rut")
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/models/projects_tags.py

@@ -13,15 +13,27 @@
     sa.Column(
         "project_id",
         sa.BigInteger,
-        sa.ForeignKey(projects.c.id, onupdate="CASCADE", ondelete="CASCADE"),
-        nullable=False,
-        doc="NOTE that project.c.id != project.c.uuid",
+        sa.ForeignKey(
+            projects.c.id,
+            onupdate="CASCADE",
+            ondelete="SET NULL",
+            name="project_tags_project_id_fkey",
+        ),
+        nullable=True,  # <-- NULL means that project was deleted
+        doc="NOTE that project.c.id != project.c.uuid. If project is deleted, we do not delete project in this table, we just set this column to NULL. Why? Because the `project_uuid_for_rut` is still used by resource usage tracker",
     ),
     sa.Column(
         "tag_id",
         sa.BigInteger,
         sa.ForeignKey(tags.c.id, onupdate="CASCADE", ondelete="CASCADE"),
         nullable=False,
     ),
-    sa.UniqueConstraint("project_id", "tag_id"),
+    sa.Column(
+        "project_uuid_for_rut",
+        sa.String,
+        nullable=False,
+    ),
+    sa.UniqueConstraint(
+        "project_uuid_for_rut", "tag_id", name="project_tags_project_uuid_unique"
+    ),
 )

packages/postgres-database/src/simcore_postgres_database/utils_tags_sql.py

@@ -1,4 +1,5 @@
 import functools
+from uuid import UUID
 
 import sqlalchemy as sa
 from simcore_postgres_database.models.groups import user_to_groups
@@ -60,7 +61,7 @@ def get_tag_stmt(
             # aggregation ensures MOST PERMISSIVE policy of access-rights
             sa.func.bool_or(tags_access_rights.c.read).label("read"),
             sa.func.bool_or(tags_access_rights.c.write).label("write"),
-            sa.func.bool_or(tags_access_rights.c.delete).label("delete")
+            sa.func.bool_or(tags_access_rights.c.delete).label("delete"),
         )
         .select_from(
             _join_user_to_given_tag(
@@ -80,7 +81,7 @@ def list_tags_stmt(*, user_id: int):
             # aggregation ensures MOST PERMISSIVE policy of access-rights
             sa.func.bool_or(tags_access_rights.c.read).label("read"),
             sa.func.bool_or(tags_access_rights.c.write).label("write"),
-            sa.func.bool_or(tags_access_rights.c.delete).label("delete")
+            sa.func.bool_or(tags_access_rights.c.delete).label("delete"),
         )
         .select_from(
             _join_user_to_tags(
@@ -104,7 +105,7 @@ def count_groups_with_given_access_rights_stmt(
     tag_id: int,
     read: bool | None,
     write: bool | None,
-    delete: bool | None
+    delete: bool | None,
 ):
     """
     How many groups (from this user_id) are given EXACTLY these access permissions
@@ -192,12 +193,15 @@ def get_tags_for_project_stmt(*, project_index: int):
     )
 
 
-def add_tag_to_project_stmt(*, project_index: int, tag_id: int):
+def add_tag_to_project_stmt(
+    *, project_index: int, tag_id: int, project_uuid_for_rut: UUID
+):
     return (
         pg_insert(projects_tags)
         .values(
             project_id=project_index,
             tag_id=tag_id,
+            project_uuid_for_rut=f"{project_uuid_for_rut}",
         )
         .on_conflict_do_nothing()
     )

packages/postgres-database/tests/test_utils_tags.py

@@ -668,6 +668,7 @@ def _check(func_smt, **kwargs):
     user_id = 425  # 4
     tag_id = 4
     project_index = 1
+    project_uuid = "106f8b4b-ffb6-459a-a27b-981c779e6d3f"
     service_key = "simcore/services/comp/isolve"
     service_version = "2.0.85"
 
@@ -726,6 +727,7 @@ def _check(func_smt, **kwargs):
         add_tag_to_project_stmt,
         project_index=project_index,
         tag_id=tag_id,
+        project_uuid_for_rut=project_uuid,
     )
 
     _check(

services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/service_runs.py

@@ -101,6 +101,7 @@ class Config:
 class ServiceRunWithCreditsDB(ServiceRunDB):
     osparc_credits: Decimal | None
     transaction_status: CreditTransactionStatus | None
+    project_tags: list[str]
 
     class Config:
         orm_mode = True

services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/resource_tracker.py

@@ -28,6 +28,7 @@
 from models_library.users import UserID
 from models_library.wallets import WalletID
 from pydantic import PositiveInt
+from simcore_postgres_database.models.projects_tags import projects_tags
 from simcore_postgres_database.models.resource_tracker_credit_transactions import (
     resource_tracker_credit_transactions,
 )
@@ -46,6 +47,7 @@
 from simcore_postgres_database.models.resource_tracker_service_runs import (
     resource_tracker_service_runs,
 )
+from simcore_postgres_database.models.tags import tags
 from sqlalchemy.dialects.postgresql import ARRAY, INTEGER
 
 from .....exceptions.errors import (
@@ -212,6 +214,15 @@ async def get_service_run_by_id(
             return None
         return ServiceRunDB.from_orm(row)
 
+    _project_tags_subquery = (
+        sa.select(
+            projects_tags.c.project_uuid_for_rut,
+            sa.func.array_agg(tags.c.name).label("project_tags"),
+        )
+        .select_from(projects_tags.join(tags, projects_tags.c.tag_id == tags.c.id))
+        .group_by(projects_tags.c.project_uuid_for_rut)
+    ).subquery("project_tags_subquery")
+
     async def list_service_runs_by_product_and_user_and_wallet(
         self,
         product_name: ProductName,
@@ -260,6 +271,10 @@ async def list_service_runs_by_product_and_user_and_wallet(
                     resource_tracker_service_runs.c.missed_heartbeat_counter,
                     resource_tracker_credit_transactions.c.osparc_credits,
                     resource_tracker_credit_transactions.c.transaction_status,
+                    sa.func.coalesce(
+                        self._project_tags_subquery.c.project_tags,
+                        sa.cast(sa.text("'{}'"), sa.ARRAY(sa.String)),
+                    ).label("project_tags"),
                 )
                 .select_from(
                     resource_tracker_service_runs.join(
@@ -273,6 +288,11 @@ async def list_service_runs_by_product_and_user_and_wallet(
                             == resource_tracker_credit_transactions.c.service_run_id
                         ),
                         isouter=True,
+                    ).join(
+                        self._project_tags_subquery,
+                        resource_tracker_service_runs.c.project_id
+                        == self._project_tags_subquery.c.project_uuid_for_rut,
+                        isouter=True,
                     )
                 )
                 .where(resource_tracker_service_runs.c.product_name == product_name)
@@ -436,7 +456,9 @@ async def export_service_runs_table_to_s3(
                     resource_tracker_service_runs.c.service_run_id,
                     resource_tracker_service_runs.c.wallet_name,
                     resource_tracker_service_runs.c.user_email,
-                    resource_tracker_service_runs.c.project_name,
+                    resource_tracker_service_runs.c.root_parent_project_name.label(
+                        "project_name"
+                    ),
                     resource_tracker_service_runs.c.node_name,
                     resource_tracker_service_runs.c.service_key,
                     resource_tracker_service_runs.c.service_version,
@@ -445,13 +467,22 @@ async def export_service_runs_table_to_s3(
                     resource_tracker_service_runs.c.stopped_at,
                     resource_tracker_credit_transactions.c.osparc_credits,
                     resource_tracker_credit_transactions.c.transaction_status,
+                    sa.func.coalesce(
+                        self._project_tags_subquery.c.project_tags,
+                        sa.cast(sa.text("'{}'"), sa.ARRAY(sa.String)),
+                    ).label("project_tags"),
                 )
                 .select_from(
                     resource_tracker_service_runs.join(
                         resource_tracker_credit_transactions,
                         resource_tracker_service_runs.c.service_run_id
                         == resource_tracker_credit_transactions.c.service_run_id,
                         isouter=True,
+                    ).join(
+                        self._project_tags_subquery,
+                        resource_tracker_service_runs.c.project_id
+                        == self._project_tags_subquery.c.project_uuid_for_rut,
+                        isouter=True,
                     )
                 )
                 .where(resource_tracker_service_runs.c.product_name == product_name)

services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/service_runs.py

@@ -121,6 +121,7 @@ async def list_service_runs(
                 user_email=service.user_email,
                 project_id=service.project_id,
                 project_name=service.project_name,
+                project_tags=service.project_tags,
                 root_parent_project_id=service.root_parent_project_id,
                 root_parent_project_name=service.root_parent_project_name,
                 node_id=service.node_id,

services/resource-usage-tracker/tests/unit/with_dbs/conftest.py

@@ -51,6 +51,7 @@ def mock_env(monkeypatch: pytest.MonkeyPatch) -> EnvVarsDict:
         "SC_BOOT_MODE": "production",
         "POSTGRES_CLIENT_NAME": "postgres_test_client",
         "RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_CHECK_ENABLED": "0",
+        "RESOURCE_USAGE_TRACKER_TRACING": "null",
     }
     setenvs_from_dict(monkeypatch, env_vars)
     return env_vars

services/web/server/src/simcore_service_webserver/projects/db.py

@@ -403,7 +403,9 @@ async def list_projects(  # pylint: disable=too-many-arguments,too-many-statemen
                 sa.select(
                     projects_tags.c.project_id,
                     sa.func.array_agg(projects_tags.c.tag_id).label("tags"),
-                ).group_by(projects_tags.c.project_id)
+                )
+                .where(projects_tags.c.project_id.is_not(None))
+                .group_by(projects_tags.c.project_id)
             ).subquery("project_tags_subquery")
 
             ###
@@ -1218,6 +1220,7 @@ async def add_tag(
                     projects_tags.insert().values(
                         project_id=project["id"],
                         tag_id=tag_id,
+                        project_uuid_for_rut=project["uuid"],
                     )
                 )
                 project_tags.append(tag_id)