Users search improvments (#270)

* Users search improvments

* Users to org moved to backend

* Update users with additional data

* Migrate users when refreshing

* BE request for users change

* PR comments

* submodules updated

* removed unused code

* PR comments

* Submodules merge

Author: lumburovskalina

alembic/versions/7aa933ec5de9_kratos_data.py

@@ -0,0 +1,39 @@
+"""Kratos data
+
+Revision ID: 7aa933ec5de9
+Revises: 05bbef1eec3f
+Create Date: 2024-11-19 16:04:00.539685
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "7aa933ec5de9"
+down_revision = "05bbef1eec3f"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("user", sa.Column("email", sa.String(), nullable=True))
+    op.add_column("user", sa.Column("verified", sa.Boolean(), nullable=True))
+    op.add_column("user", sa.Column("created_at", sa.DateTime(), nullable=True))
+    op.add_column("user", sa.Column("metadata_public", sa.JSON(), nullable=True))
+    op.add_column("user", sa.Column("sso_provider", sa.String(), nullable=True))
+    op.create_unique_constraint(None, "user", ["email"])
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, "user", type_="unique")
+    op.drop_column("user", "sso_provider")
+    op.drop_column("user", "metadata_public")
+    op.drop_column("user", "created_at")
+    op.drop_column("user", "verified")
+    op.drop_column("user", "email")
+    # ### end Alembic commands ###

app.py

@@ -43,6 +43,7 @@
 from starlette.routing import Route, Mount
 
 from controller.project.manager import check_in_deletion_projects
+from controller.user.manager import migrate_kratos_users
 from route_prefix import (
     PREFIX_ORGANIZATION,
     PREFIX_PROJECT,
@@ -70,6 +71,7 @@
 logger = logging.getLogger(__name__)
 
 init_config()
+migrate_kratos_users()
 fastapi_app = FastAPI()
 
 

controller/auth/kratos.py

@@ -5,6 +5,9 @@
 from datetime import datetime, timedelta
 from urllib.parse import quote
 
+from controller.user import manager
+
+
 logging.basicConfig(level=logging.INFO)
 logger: logging.Logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
@@ -17,19 +20,19 @@
 KRATOS_IDENTITY_CACHE_TIMEOUT = timedelta(minutes=30)
 
 
-def __get_cached_values() -> Dict[str, Dict[str, Any]]:
+def get_cached_values(update_db_users: bool = True) -> Dict[str, Dict[str, Any]]:
     global KRATOS_IDENTITY_CACHE
     if not KRATOS_IDENTITY_CACHE or len(KRATOS_IDENTITY_CACHE) == 0:
-        __refresh_identity_cache()
+        __refresh_identity_cache(update_db_users)
     elif (
         KRATOS_IDENTITY_CACHE["collected"] + KRATOS_IDENTITY_CACHE_TIMEOUT
         < datetime.now()
     ):
-        __refresh_identity_cache()
+        __refresh_identity_cache(update_db_users)
     return KRATOS_IDENTITY_CACHE
 
 
-def __refresh_identity_cache():
+def __refresh_identity_cache(update_db_users: bool = True) -> None:
     global KRATOS_IDENTITY_CACHE
     request = requests.get(f"{KRATOS_ADMIN_URL}/identities")
     if request.ok:
@@ -54,6 +57,9 @@ def __refresh_identity_cache():
     else:
         KRATOS_IDENTITY_CACHE = {}
 
+    if update_db_users:
+        manager.migrate_kratos_users()
+
 
 def __get_link_from_kratos_request(request: requests.Response) -> str:
     # rel=next only if there is more than 1 page
@@ -71,7 +77,7 @@ def __get_link_from_kratos_request(request: requests.Response) -> str:
 def __get_identity(user_id: str, only_simple: bool = True) -> Dict[str, Any]:
     if not isinstance(user_id, str):
         user_id = str(user_id)
-    cache = __get_cached_values()
+    cache = get_cached_values()
     if user_id in cache:
         if only_simple:
             return cache[user_id]["simple"]
@@ -117,7 +123,7 @@ def __parse_identity_to_simple(identity: Dict[str, Any]) -> Dict[str, str]:
 
 
 def get_userid_from_mail(user_mail: str) -> str:
-    values = __get_cached_values()
+    values = get_cached_values()
     for key in values:
         if key == "collected":
             continue

controller/user/manager.py

@@ -1,5 +1,5 @@
-from typing import Any, Dict, List
-from submodules.model import User, enums
+from typing import Any, Dict, List, Optional
+from submodules.model import User, daemon, enums
 from submodules.model.business_objects import user, user_activity, general
 from controller.auth import kratos
 from submodules.model.exceptions import EntityNotFoundException
@@ -91,11 +91,17 @@ def remove_organization_from_user(user_mail: str) -> None:
     user.remove_organization(user_id, with_commit=True)
 
 
-def get_active_users(minutes: int, order_by_interaction: bool) -> User:
+def get_active_users_filtered(
+    minutes: Optional[int] = None,
+    sort_key: Optional[str] = None,
+    sort_direction: Optional[str] = None,
+    offset: Optional[int] = None,
+    limit: Optional[int] = None,
+) -> User:
     now = datetime.now()
     last_interaction_range = (now - timedelta(minutes=minutes)) if minutes > 0 else None
-    return user_activity.get_active_users_in_range(
-        last_interaction_range, order_by_interaction
+    return user.get_active_users_after_filter(
+        last_interaction_range, sort_key, sort_direction, offset, limit
     )
 
 
@@ -104,53 +110,53 @@ def update_last_interaction(user_id: str) -> None:
     user_activity.update_last_interaction(user_id)
 
 
-def get_mapped_sorted_paginated_users(
-    active_users: Dict[str, Any],
-    sort_key: str,
-    sort_direction: int,
-    offset: int,
-    limit: int,
-) -> List[Dict[str, Any]]:
-
-    final_users = []
-    save_len_final_users = 0
-
-    # mapping users with the users in kratos
-    active_users_ids = list(active_users.keys())
-
-    for user_id in active_users_ids:
-        get_user = kratos.__get_identity(user_id, False)["identity"]
-        if get_user and get_user["traits"]["email"] is not None:
-            get_user["email"] = get_user["traits"]["email"]
-            get_user["verified"] = get_user["verifiable_addresses"][0]["verified"]
-            active_user_by_id = active_users[user_id]
-            get_user["last_interaction"] = active_user_by_id["last_interaction"]
-            get_user["role"] = active_user_by_id["role"]
-            get_user["organization"] = active_user_by_id["organizationName"]
-
-            public_meta = get_user["metadata_public"]
-            get_user["sso_provider"] = (
-                public_meta.get("registration_scope", {}).get("provider_id", None)
-                if public_meta
-                else None
-            )
-
-            final_users.append(get_user)
-            save_len_final_users += 1
-
-    final_users = sorted(
-        final_users,
-        key=lambda x: (x[sort_key] is None, x.get(sort_key, "")),
-        reverse=sort_direction == -1,
-    )
-
-    # paginating users
-    final_users = final_users[offset : offset + limit]
-
-    return final_users, save_len_final_users
-
-
 def delete_user(user_id: str) -> None:
     user.delete(user_id, with_commit=True)
     user_activity.delete_user_activity(user_id, with_commit=True)
     kratos.__refresh_identity_cache()
+
+
+def migrate_kratos_users() -> None:
+    # this is only supposed to be called during startup of the application
+    daemon.run_with_db_token(__migrate_kratos_users)
+
+
+def __migrate_kratos_users():
+    users_kratos = kratos.get_cached_values(False)
+    users_database = user.get_all()
+
+    for user_database in users_database:
+        user_id = str(user_database.id)
+        user_identity = users_kratos[user_id]["identity"]
+
+        if user_database.email != user_identity["traits"]["email"]:
+            user_database.email = user_identity["traits"]["email"]
+        if (
+            user_database.verified
+            != user_identity["verifiable_addresses"][0]["verified"]
+        ):
+            user_database.verified = user_identity["verifiable_addresses"][0][
+                "verified"
+            ]
+        if (
+            user_database.created_at
+            != user_identity["verifiable_addresses"][0]["created_at"]
+        ):
+            user_database.created_at = user_identity["verifiable_addresses"][0][
+                "created_at"
+            ]
+        if user_database.metadata_public != user_identity["metadata_public"]:
+            user_database.metadata_public = user_identity["metadata_public"]
+        sso_provider = (
+            (
+                user_identity["metadata_public"]
+                .get("registration_scope", {})
+                .get("provider_id", None)
+            )
+            if user_identity["metadata_public"]
+            else None
+        )
+        if user_database.sso_provider != sso_provider:
+            user_database.sso_provider = sso_provider
+
+    general.commit()

fast_api/models.py

@@ -440,3 +440,7 @@ class UpdateCustomerButton(BaseModel):
     location: Optional[CustomerButtonLocation] = None
     visible: Optional[StrictBool] = None
     config: Optional[Dict[StrictStr, Any]] = None
+
+
+class MissingUsersBody(BaseModel):
+    user_ids: List[StrictStr]

fast_api/routes/organization.py

@@ -10,6 +10,7 @@
     DeleteOrganizationBody,
     DeleteUserBody,
     MappedSortedPaginatedUsers,
+    MissingUsersBody,
     RemoveUserToOrganizationBody,
     UserLanguageDisplay,
 )
@@ -24,7 +25,8 @@
 from controller.user import manager as user_manager
 
 from fast_api.routes.client_response import get_silent_success, pack_json_result
-from submodules.model.business_objects import organization
+from submodules.model import events
+from submodules.model.business_objects import organization, user
 from submodules.model.util import sql_alchemy_to_dict
 from util import notification
 
@@ -273,31 +275,32 @@ def get_mapped_sorted_paginated_users(
     request: Request, body: MappedSortedPaginatedUsers = Body(...)
 ):
     auth_manager.check_admin_access(request.state.info)
-    active_users = user_manager.get_active_users(body.filter_minutes, None)
+    count_users = user_manager.get_active_users_filtered(body.filter_minutes)
+    active_users = user_manager.get_active_users_filtered(
+        body.filter_minutes, body.sort_key, body.sort_direction, body.offset, body.limit
+    )
     active_users = [
         {
             "id": str(user.id),
             "last_interaction": (
                 user.last_interaction.isoformat() if user.last_interaction else None
             ),
             "role": user.role,
-            "organizationName": (
-                organization_manager.get_organization_by_id(str(user.organization_id))[
-                    "name"
-                ]
-                if user.organization_id
-                else ""
-            ),
+            "organization": user.organization_name,
+            "email": user.email,
+            "verified": user.verified,
+            "created_at": user.created_at.isoformat(),
+            "metadata_public": user.metadata_public,
+            "sso_provider": user.sso_provider,
         }
         for user in active_users
     ]
-    active_users = {user["id"]: user for user in active_users}
 
-    data, final_len = user_manager.get_mapped_sorted_paginated_users(
-        active_users, body.sort_key, body.sort_direction, body.offset, body.limit
-    )
     return pack_json_result(
-        {"mappedSortedPaginatedUsers": data, "fullCountUsers": final_len},
+        {
+            "mappedSortedPaginatedUsers": active_users,
+            "fullCountUsers": len(count_users),
+        },
         wrap_for_frontend=False,  # needed because it's used like this on the frontend (kratos values)
     )
 
@@ -307,3 +310,17 @@ def delete_user(request: Request, body: DeleteUserBody = Body(...)):
     auth_manager.check_admin_access(request.state.info)
     user_manager.delete_user(body.user_id)
     return get_silent_success()
+
+
+@router.post("/missing-users-interaction")
+def get_missing_users_interaction(request: Request, body: MissingUsersBody = Body(...)):
+    auth_manager.check_admin_access(request.state.info)
+    data = user.get_missing_users(body.user_ids)
+    return pack_json_result(data, wrap_for_frontend=False)
+
+
+@router.get("/user-to-organization")
+def get_user_to_organization(request: Request):
+    auth_manager.check_admin_access(request.state.info)
+    data = user.get_user_to_organization()
+    return pack_json_result(data, wrap_for_frontend=False)