refactor: log add workspace_id

Author: wxg0103

apps/application/views/application.py

@@ -50,7 +50,7 @@ class ApplicationAPI(APIView):
                      RoleConstants.WORKSPACE_MANAGE.get_workspace_role())
     @log(menu='Application', operate='Create an application',
          get_operation_object=lambda r, k: {'name': r.data.get('name')},
-         workspace_id=lambda r, k: k.get('workspace_id'))
+         )
     def post(self, request: Request, workspace_id: str):
         return result.success(
             ApplicationSerializer(data={'workspace_id': workspace_id, 'user_id': request.user.id}).insert(request.data))
@@ -103,7 +103,7 @@ class Import(APIView):
             tags=[_('Application')]  # type: ignore
         )
         @has_permissions(PermissionConstants.APPLICATION_READ, RoleConstants.WORKSPACE_MANAGE.get_workspace_role())
-        @log(menu='Application', operate="Import Application", workspace_id=lambda r, k: k.get('workspace_id'))
+        @log(menu='Application', operate="Import Application", )
         def post(self, request: Request, workspace_id: str):
             return result.success(ApplicationSerializer(
                 data={'user_id': request.user.id, 'workspace_id': workspace_id,
@@ -126,7 +126,7 @@ class Export(APIView):
                          RoleConstants.WORKSPACE_MANAGE.get_workspace_role())
         @log(menu='Application', operate="Export Application",
              get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')),
-             workspace_id=lambda r, k: k.get('workspace_id'))
+             )
         def post(self, request: Request, workspace_id: str, application_id: str):
             return ApplicationOperateSerializer(
                 data={'application_id': application_id,
@@ -148,7 +148,7 @@ class Operate(APIView):
                          RoleConstants.WORKSPACE_MANAGE.get_workspace_role())
         @log(menu='Application', operate='Deleting application',
              get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')),
-             workspace_id=lambda r, k: k.get('workspace_id')
+             
              )
         def delete(self, request: Request, workspace_id: str, application_id: str):
             return result.success(ApplicationOperateSerializer(
@@ -169,7 +169,7 @@ def delete(self, request: Request, workspace_id: str, application_id: str):
                          RoleConstants.WORKSPACE_MANAGE.get_workspace_role())
         @log(menu='Application', operate="Modify the application",
              get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')),
-             workspace_id=lambda r, k: k.get('workspace_id'))
+             )
         def put(self, request: Request, workspace_id: str, application_id: str):
             return result.success(
                 ApplicationOperateSerializer(
@@ -207,7 +207,7 @@ class Publish(APIView):
         )
         @log(menu='Application', operate='Publishing an application',
              get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')),
-             workspace_id=lambda r, k: k.get('workspace_id')
+             
              )
         def put(self, request: Request, workspace_id: str, application_id: str):
             return result.success(

apps/application/views/application_api_key.py

@@ -38,7 +38,7 @@ class ApplicationKey(APIView):
     )
     @log(menu='Application', operate="Add ApiKey",
          get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')),
-         workspace_id=lambda r, k: k.get('workspace_id'))
+         )
     @has_permissions(PermissionConstants.APPLICATION_OVERVIEW_API_KEY.get_workspace_application_permission(),
                      RoleConstants.WORKSPACE_MANAGE.get_workspace_role()
                      )
@@ -80,7 +80,7 @@ class Operate(APIView):
                          RoleConstants.WORKSPACE_MANAGE.get_workspace_role())
         @log(menu='Application', operate="Modify application API_KEY",
              get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')),
-             workspace_id=lambda r, k: k.get('workspace_id'))
+             )
         def put(self, request: Request, workspace_id: str, application_id: str, api_key_id: str):
             return result.success(
                 ApplicationKeySerializer.Operate(
@@ -102,7 +102,7 @@ def put(self, request: Request, workspace_id: str, application_id: str, api_key_
                          RoleConstants.WORKSPACE_MANAGE.get_workspace_role())
         @log(menu='Application', operate="Delete application API_KEY",
              get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')),
-             workspace_id=lambda r, k: k.get('workspace_id'))
+             )
         def delete(self, request: Request, workspace_id: str, application_id: str, api_key_id: str):
             return result.success(
                 ApplicationKeySerializer.Operate(

apps/application/views/application_version.py

@@ -97,7 +97,7 @@ def get(self, request: Request, workspace_id: str, application_id: str, work_flo
                          RoleConstants.WORKSPACE_MANAGE.get_workspace_role())
         @log(menu='Application', operate="Modify application version information",
              get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')),
-             workspace_id=lambda r, k: k.get('workspace_id'))
+             )
         def put(self, request: Request, workspace_id: str, application_id: str, work_flow_version_id: str):
             return result.success(
                 ApplicationVersionSerializer.Operate(

apps/common/job/__init__.py

@@ -0,0 +1,17 @@
+# coding=utf-8
+"""
+    @project: maxkb
+    @Author：虎
+    @file： __init__.py
+    @date：2024/3/14 11:54
+    @desc:
+"""
+#from .client_access_num_job import *
+from .clean_chat_job import *
+from .clean_debug_file_job import *
+
+
+def run():
+    #client_access_num_job.run()
+    clean_chat_job.run()
+    clean_debug_file_job.run()

apps/common/job/clean_chat_job.py

@@ -0,0 +1,81 @@
+# coding=utf-8
+
+import logging
+import datetime
+
+from django.db import transaction
+from django.utils import timezone
+from apscheduler.schedulers.background import BackgroundScheduler
+from django_apscheduler.jobstores import DjangoJobStore
+from application.models import Application, Chat, ChatRecord
+from django.db.models import Q, Max
+from common.lock.impl.file_lock import FileLock
+
+from knowledge.models import File
+
+scheduler = BackgroundScheduler()
+scheduler.add_jobstore(DjangoJobStore(), "default")
+lock = FileLock()
+
+
+def clean_chat_log_job():
+    from django.utils.translation import gettext_lazy as _
+    logging.getLogger("max_kb").info(_('start clean chat log'))
+    now = timezone.now()
+
+    applications = Application.objects.all().values('id', 'clean_time')
+    cutoff_dates = {
+        app['id']: now - datetime.timedelta(days=app['clean_time'] or 180)
+        for app in applications
+    }
+
+    query_conditions = Q()
+    for app_id, cutoff_date in cutoff_dates.items():
+        query_conditions |= Q(chat__application_id=app_id, create_time__lt=cutoff_date)
+    batch_size = 500
+    while True:
+        with transaction.atomic():
+            chat_records = ChatRecord.objects.filter(query_conditions).select_related('chat').only('id', 'chat_id',
+                                                                                                   'create_time')[
+                           :batch_size]
+            if not chat_records:
+                break
+            chat_record_ids = [record.id for record in chat_records]
+            chat_ids = {record.chat_id for record in chat_records}
+
+            # 计算每个 chat_id 的最大 create_time
+            max_create_times = ChatRecord.objects.filter(id__in=chat_record_ids).values('chat_id').annotate(
+                max_create_time=Max('create_time'))
+
+            # 收集需要删除的文件
+            files_to_delete = []
+            for record in chat_records:
+                max_create_time = next(
+                    (item['max_create_time'] for item in max_create_times if item['chat_id'] == record.chat_id), None)
+                if max_create_time:
+                    files_to_delete.extend(
+                        File.objects.filter(meta__chat_id=str(record.chat_id), create_time__lt=max_create_time)
+                    )
+            # 删除 ChatRecord
+            deleted_count = ChatRecord.objects.filter(id__in=chat_record_ids).delete()[0]
+
+            # 删除没有关联 ChatRecord 的 Chat
+            Chat.objects.filter(chatrecord__isnull=True, id__in=chat_ids).delete()
+            File.objects.filter(loid__in=[file.loid for file in files_to_delete]).delete()
+
+            if deleted_count < batch_size:
+                break
+
+    logging.getLogger("max_kb").info(_('end clean chat log'))
+
+
+def run():
+    if lock.try_lock('clean_chat_log_job', 30 * 30):
+        try:
+            scheduler.start()
+            existing_job = scheduler.get_job(job_id='clean_chat_log')
+            if existing_job is not None:
+                existing_job.remove()
+            scheduler.add_job(clean_chat_log_job, 'cron', hour='0', minute='5', id='clean_chat_log')
+        finally:
+            lock.un_lock('clean_chat_log_job')

apps/common/job/clean_debug_file_job.py

@@ -0,0 +1,37 @@
+# coding=utf-8
+
+import logging
+from datetime import timedelta
+
+from apscheduler.schedulers.background import BackgroundScheduler
+from django.db.models import Q
+from django.utils import timezone
+from django_apscheduler.jobstores import DjangoJobStore
+
+from common.lock.impl.file_lock import FileLock
+from knowledge.models import File
+
+scheduler = BackgroundScheduler()
+scheduler.add_jobstore(DjangoJobStore(), "default")
+lock = FileLock()
+
+
+def clean_debug_file():
+    from django.utils.translation import gettext_lazy as _
+    logging.getLogger("max_kb").info(_('start clean debug file'))
+    two_hours_ago = timezone.now() - timedelta(hours=2)
+    # 删除对应的文件
+    File.objects.filter(Q(create_time__lt=two_hours_ago) & Q(meta__debug=True)).delete()
+    logging.getLogger("max_kb").info(_('end clean debug file'))
+
+
+def run():
+    if lock.try_lock('clean_debug_file', 30 * 30):
+        try:
+            scheduler.start()
+            clean_debug_file_job = scheduler.get_job(job_id='clean_debug_file')
+            if clean_debug_file_job is not None:
+                clean_debug_file_job.remove()
+            scheduler.add_job(clean_debug_file, 'cron', hour='2', minute='0', second='0', id='clean_debug_file')
+        finally:
+            lock.un_lock('clean_debug_file')

apps/common/log/log.py

@@ -57,7 +57,7 @@ def _get_details(request):
 
 
 def _get_workspace_id(request, kwargs):
-    return kwargs.get('workspace_id', None)
+    return kwargs.get('workspace_id', 'None')
 
 
 def log(menu: str, operate, get_user=_get_user, get_ip_address=_get_ip_address, get_details=_get_details,

apps/folders/views/folder.py

@@ -44,7 +44,7 @@ class FolderView(APIView):
     @log(
         menu='folder', operate='Create folder',
         get_operation_object=lambda r, k: {'name': r.data.get('name')},
-        workspace_id=lambda r, k: k.get('workspace_id')
+        
     )
     def post(self, request: Request, workspace_id: str, source: str):
         return result.success(FolderSerializer.Create(
@@ -93,7 +93,7 @@ class Operate(APIView):
         @log(
             menu='folder', operate='Edit folder',
             get_operation_object=lambda r, k: get_folder_operation_object(k.get('folder_id'), k.get('source')),
-            workspace_id=lambda r, k: k.get('workspace_id')
+            
         )
         def put(self, request: Request, workspace_id: str, source: str, folder_id: str):
             return result.success(FolderSerializer.Operate(
@@ -136,7 +136,7 @@ def get(self, request: Request, workspace_id: str, source: str, folder_id: str):
         @log(
             menu='folder', operate='Delete folder',
             get_operation_object=lambda r, k: get_folder_operation_object(k.get('folder_id'), k.get('source')),
-            workspace_id=lambda r, k: k.get('workspace_id')
+            
         )
         def delete(self, request: Request, workspace_id: str, source: str, folder_id: str):
             return result.success(FolderSerializer.Operate(