Admin queries (#308)

* Admin queries

* Excel download for admin queries

* PR comments

* small change

* revert user language chagne (handeled in different pr)

* Submodule update

Author: JWittmeyer

controller/misc/manager.py

@@ -1,13 +1,21 @@
 from typing import Any, Dict, List
 from fast_api.types import ServiceVersionResult
-from submodules.model.global_objects import customer_button
+from submodules.model.global_objects import (
+    customer_button,
+    admin_queries as admin_queries_db_go,
+)
 from datetime import datetime
 import os
 from controller.auth import kratos
 from submodules.model.util import sql_alchemy_to_dict
 from submodules.model import enums
 import requests
 from urllib.parse import urlparse, urlencode, parse_qsl, parse_qs
+from util.tmp_export_file_cleanup import add_cleanup_task
+from uuid import uuid4
+import pandas as pd
+from submodules.model.util import ensure_sql_text
+from submodules.model.business_objects import general
 
 import base64
 
@@ -169,3 +177,16 @@ def __patch_url(url: str, **kwargs):
         ._replace(query=urlencode(dict(parse_qsl(urlparse(url).query), **kwargs)))
         .geturl()
     )
+
+
+def create_admin_query_excel(
+    query: enums.AdminQueries, parameters: Dict[str, Any]
+) -> str:
+
+    q = admin_queries_db_go.get_result_admin_query(query, parameters, as_query=True)
+
+    df = pd.read_sql(ensure_sql_text(q), con=general.get_bind())
+    tmp_filename = f"tmp/feedback_{uuid4()}.xlsx"
+    df.to_excel(tmp_filename, index=False)
+    add_cleanup_task(tmp_filename, 5)
+    return tmp_filename

fast_api/models.py

@@ -513,5 +513,9 @@ class CheckInviteUsersBody(BaseModel):
     emails: List[StrictStr]
 
 
+class AdminQueryFilterBody(BaseModel):
+    parameters: Optional[Dict[str, Any]] = None
+
+
 class RecordDeletion(BaseModel):
     record_ids: List[str]

fast_api/routes/misc.py

@@ -1,6 +1,6 @@
 from exceptions.exceptions import AuthManagerError
 from fastapi import APIRouter, Body, Request, status
-from fastapi.responses import PlainTextResponse
+from fastapi.responses import PlainTextResponse, FileResponse
 from fast_api.models import (
     CancelTaskBody,
     CheckInviteUsersBody,
@@ -9,6 +9,7 @@
     ModelProviderDownloadModelBody,
     CreateCustomerButton,
     UpdateCustomerButton,
+    AdminQueryFilterBody,
 )
 from fast_api.routes.client_response import (
     pack_json_result,
@@ -22,12 +23,16 @@
 from controller.model_provider import manager as model_provider_manager
 from controller.task_master import manager as task_master_manager
 from submodules.model import enums
-from submodules.model.global_objects import customer_button as customer_button_db_go
+from submodules.model.global_objects import (
+    customer_button as customer_button_db_go,
+    admin_queries as admin_queries_db_go,
+)
 from submodules.model.util import sql_alchemy_to_dict
 from submodules.model.enums import (
     try_parse_enum_value,
     CustomerButtonType,
     CustomerButtonLocation,
+    AdminQueries,
 )
 from submodules.model.business_objects import task_queue as task_queue_bo
 
@@ -296,3 +301,33 @@ def check_valid_emails(request: Request, body: CheckInviteUsersBody = Body(...))
         raise AuthManagerError("Full admin access required")
     data = auth.check_valid_emails(body.emails)
     return pack_json_result(data)
+
+
+# post to allow for a body to be sent
+@router.post("/admin-query/{query}")
+def get_admin_queries(
+    request: Request, query: AdminQueries, body: AdminQueryFilterBody = Body(...)
+):
+    auth.check_admin_access(request.state.info)
+    if not auth.check_is_full_admin(request):
+        raise AuthManagerError("Full admin access required")
+    data = admin_queries_db_go.get_result_admin_query(query, body.parameters)
+    return pack_json_result(data)
+
+
+@router.post("/admin-query/{query}/download")
+def get_admin_query_excel(
+    request: Request, query: AdminQueries, body: AdminQueryFilterBody = Body(...)
+):
+    auth.check_admin_access(request.state.info)
+    if not auth.check_is_full_admin(request):
+        raise AuthManagerError("Full admin access required")
+
+    file_path = manager.create_admin_query_excel(query, body.parameters)
+
+    if file_path is None:
+        return GENERIC_FAILURE_RESPONSE
+    return FileResponse(
+        file_path,
+        media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    )

submodules/model

@@ -1,5 +1,6 @@
 from submodules.model.business_objects import upload_task
-import os, shutil
+import os
+import shutil
 
 
 def clean_up_database() -> None:

util/clean_up.py

@@ -0,0 +1,56 @@
+from datetime import datetime, timedelta
+from submodules.model.daemon import run_without_db_token
+import glob
+import os
+import traceback
+import time
+
+__cleanup_tasks = None
+
+
+def add_cleanup_task(tmp_filename: str, minutes: int) -> None:
+    global __cleanup_tasks
+    if __cleanup_tasks is None:
+        add_existing_feedback_and_consumption_files_to_cleanup(tmp_filename)
+        run_without_db_token(start_cleanup)
+    else:
+        __cleanup_tasks[tmp_filename] = datetime.now() + timedelta(minutes=minutes)
+
+
+TMP_PATTERN = [
+    "tmp/feedback_*.xlsx",
+    "tmp/consumption_*.zip",
+    "tmp/consumption_detailed_*.zip",
+    "tmp/consumption_summary_*.xlsx",
+]
+
+
+def add_existing_feedback_and_consumption_files_to_cleanup(tmp_filename: str) -> None:
+    # failsafe for shutdown before deletion time ran out
+    # should only be called for feedback_cleanup_tasks = None
+    global __cleanup_tasks
+    if __cleanup_tasks is not None:
+        raise ValueError("someone didn't read the comment")
+    __cleanup_tasks = {}
+    for pattern in TMP_PATTERN:
+        for filename in glob.glob(pattern):
+            minutes = 1  # after a restart they most likely are not valid anymore so we delete them sooner
+            if filename == tmp_filename:
+                minutes = 60
+            __cleanup_tasks[filename] = datetime.now() + timedelta(minutes=minutes)
+
+
+def start_cleanup() -> None:
+    global __cleanup_tasks
+    while __cleanup_tasks is not None and len(__cleanup_tasks) > 0:
+        try:
+            for filename, delete_time in list(__cleanup_tasks.items()):
+                if datetime.now() > delete_time:
+                    if os.path.isfile(filename):
+                        os.remove(filename)
+                    del __cleanup_tasks[filename]
+        except Exception:
+            print("Error in cleanup task", flush=True)
+            print(traceback.format_exc(), flush=True)
+        finally:
+            time.sleep(60)