refactor: replace logging with maxkb_logger for consistent logging across modules

Author: liuruibin

apps/application/flow/step_node/ai_chat_step_node/impl/base_chat_node.py

@@ -25,6 +25,7 @@
 from application.flow.tools import Reasoning
 from models_provider.models import Model
 from models_provider.tools import get_model_credential, get_model_instance_by_model_workspace_id
+from common.utils.logger import maxkb_logger
 
 tool_message_template = """
 <details>
@@ -126,7 +127,7 @@ def mcp_response_generator(chat_model, message_list, mcp_servers):
             except StopAsyncIteration:
                 break
     except Exception as e:
-        logging.getLogger("max_kb").error(f'Exception: {e}')
+        maxkb_logger.error(f'Exception: {e}')
     finally:
         loop.close()
 

apps/common/event/listener_manage.py

@@ -6,7 +6,6 @@
     @date：2023/10/20 14:01
     @desc:
 """
-import logging
 import os
 import threading
 import datetime
@@ -22,13 +21,13 @@
 from common.db.search import native_search, get_dynamics_model, native_update
 from common.utils.common import get_file_content
 from common.utils.lock import try_lock, un_lock
+from common.utils.logger import maxkb_logger, maxkb_error_logger
 from common.utils.page_utils import page_desc
 from knowledge.models import Paragraph, Status, Document, ProblemParagraphMapping, TaskType, State,SourceType, SearchMode
 from maxkb.conf import (PROJECT_DIR)
 from django.utils.translation import gettext_lazy as _
 
-max_kb_error = logging.getLogger(__file__)
-max_kb = logging.getLogger(__file__)
+
 lock = threading.Lock()
 
 
@@ -87,12 +86,12 @@ def embedding_by_paragraph_list(paragraph_id_list, embedding_model: Embeddings):
             ListenerManagement.embedding_by_paragraph_data_list(data_list, paragraph_id_list=paragraph_id_list,
                                                                 embedding_model=embedding_model)
         except Exception as e:
-            max_kb_error.error(_('Query vector data: {paragraph_id_list} error {error} {traceback}').format(
+            maxkb_error_logger.error(_('Query vector data: {paragraph_id_list} error {error} {traceback}').format(
                 paragraph_id_list=paragraph_id_list, error=str(e), traceback=traceback.format_exc()))
 
     @staticmethod
     def embedding_by_paragraph_data_list(data_list, paragraph_id_list, embedding_model: Embeddings):
-        max_kb.info(_('Start--->Embedding paragraph: {paragraph_id_list}').format(paragraph_id_list=paragraph_id_list))
+        maxkb_logger.info(_('Start--->Embedding paragraph: {paragraph_id_list}').format(paragraph_id_list=paragraph_id_list))
         status = Status.success
         try:
             # 删除段落
@@ -104,12 +103,12 @@ def is_save_function():
             # 批量向量化
             VectorStore.get_embedding_vector().batch_save(data_list, embedding_model, is_save_function)
         except Exception as e:
-            max_kb_error.error(_('Vectorized paragraph: {paragraph_id_list} error {error} {traceback}').format(
+            maxkb_error_logger.error(_('Vectorized paragraph: {paragraph_id_list} error {error} {traceback}').format(
                 paragraph_id_list=paragraph_id_list, error=str(e), traceback=traceback.format_exc()))
             status = Status.error
         finally:
             QuerySet(Paragraph).filter(id__in=paragraph_id_list).update(**{'status': status})
-            max_kb.info(
+            maxkb_logger.info(
                 _('End--->Embedding paragraph: {paragraph_id_list}').format(paragraph_id_list=paragraph_id_list))
 
     @staticmethod
@@ -119,7 +118,7 @@ def embedding_by_paragraph(paragraph_id, embedding_model: Embeddings):
         @param paragraph_id:    段落id
         @param embedding_model:  向量模型
         """
-        max_kb.info(_('Start--->Embedding paragraph: {paragraph_id}').format(paragraph_id=paragraph_id))
+        maxkb_logger.info(_('Start--->Embedding paragraph: {paragraph_id}').format(paragraph_id=paragraph_id))
         # 更新到开始状态
         ListenerManagement.update_status(QuerySet(Paragraph).filter(id=paragraph_id), TaskType.EMBEDDING, State.STARTED)
         try:
@@ -144,12 +143,12 @@ def is_the_task_interrupted():
             ListenerManagement.update_status(QuerySet(Paragraph).filter(id=paragraph_id), TaskType.EMBEDDING,
                                              State.SUCCESS)
         except Exception as e:
-            max_kb_error.error(_('Vectorized paragraph: {paragraph_id} error {error} {traceback}').format(
+            maxkb_error_logger.error(_('Vectorized paragraph: {paragraph_id} error {error} {traceback}').format(
                 paragraph_id=paragraph_id, error=str(e), traceback=traceback.format_exc()))
             ListenerManagement.update_status(QuerySet(Paragraph).filter(id=paragraph_id), TaskType.EMBEDDING,
                                              State.FAILURE)
         finally:
-            max_kb.info(_('End--->Embedding paragraph: {paragraph_id}').format(paragraph_id=paragraph_id))
+            maxkb_logger.info(_('End--->Embedding paragraph: {paragraph_id}').format(paragraph_id=paragraph_id))
 
     @staticmethod
     def embedding_by_data_list(data_list: List, embedding_model: Embeddings):
@@ -265,7 +264,7 @@ def is_the_task_interrupted():
 
             if is_the_task_interrupted():
                 return
-            max_kb.info(_('Start--->Embedding document: {document_id}').format(document_id=document_id)
+            maxkb_logger.info(_('Start--->Embedding document: {document_id}').format(document_id=document_id)
                         )
             # 批量修改状态为PADDING
             ListenerManagement.update_status(QuerySet(Document).filter(id=document_id), TaskType.EMBEDDING,
@@ -285,12 +284,12 @@ def is_the_task_interrupted():
                                                                            document_id)),
                       is_the_task_interrupted)
         except Exception as e:
-            max_kb_error.error(_('Vectorized document: {document_id} error {error} {traceback}').format(
+            maxkb_error_logger.error(_('Vectorized document: {document_id} error {error} {traceback}').format(
                 document_id=document_id, error=str(e), traceback=traceback.format_exc()))
         finally:
             ListenerManagement.post_update_document_status(document_id, TaskType.EMBEDDING)
             ListenerManagement.get_aggregation_document_status(document_id)()
-            max_kb.info(_('End--->Embedding document: {document_id}').format(document_id=document_id))
+            maxkb_logger.info(_('End--->Embedding document: {document_id}').format(document_id=document_id))
             un_lock('embedding' + str(document_id))
 
     @staticmethod
@@ -301,18 +300,18 @@ def embedding_by_knowledge(knowledge_id, embedding_model: Embeddings):
         @param embedding_model 向量模型
         :return: None
         """
-        max_kb.info(_('Start--->Embedding knowledge: {knowledge_id}').format(knowledge_id=knowledge_id))
+        maxkb_logger.info(_('Start--->Embedding knowledge: {knowledge_id}').format(knowledge_id=knowledge_id))
         try:
             ListenerManagement.delete_embedding_by_knowledge(knowledge_id)
             document_list = QuerySet(Document).filter(knowledge_id=knowledge_id)
-            max_kb.info(_('Start--->Embedding document: {document_list}').format(document_list=document_list))
+            maxkb_logger.info(_('Start--->Embedding document: {document_list}').format(document_list=document_list))
             for document in document_list:
                 ListenerManagement.embedding_by_document(document.id, embedding_model=embedding_model)
         except Exception as e:
-            max_kb_error.error(_('Vectorized knowledge: {knowledge_id} error {error} {traceback}').format(
+            maxkb_error_logger.error(_('Vectorized knowledge: {knowledge_id} error {error} {traceback}').format(
                 knowledge_id=knowledge_id, error=str(e), traceback=traceback.format_exc()))
         finally:
-            max_kb.info(_('End--->Embedding knowledge: {knowledge_id}').format(knowledge_id=knowledge_id))
+            maxkb_logger.info(_('End--->Embedding knowledge: {knowledge_id}').format(knowledge_id=knowledge_id))
 
     @staticmethod
     def delete_embedding_by_document(document_id):

apps/common/handle/impl/table/csv_parse_table_handle.py

@@ -4,8 +4,7 @@
 from charset_normalizer import detect
 
 from common.handle.base_parse_table_handle import BaseParseTableHandle
-
-max_kb = logging.getLogger("max_kb")
+from common.utils.logger import maxkb_logger, maxkb_error_logger
 
 
 class CsvParseTableHandle(BaseParseTableHandle):
@@ -20,7 +19,7 @@ def handle(self, file, get_buffer, save_image):
         try:
             content = buffer.decode(detect(buffer)['encoding'])
         except BaseException as e:
-            max_kb.error(f'csv split handle error: {e}')
+            maxkb_logger.error(f'csv split handle error: {e}')
             return [{'name': file.name, 'paragraphs': []}]
 
         csv_model = content.split('\n')
@@ -40,5 +39,5 @@ def get_content(self, file, save_image):
         try:
             return buffer.decode(detect(buffer)['encoding'])
         except BaseException as e:
-            max_kb.error(f'csv split handle error: {e}')
+            maxkb_error_logger.error(f'csv split handle error: {e}')
             return f'error: {e}'

apps/common/handle/impl/table/xls_parse_table_handle.py

@@ -4,8 +4,7 @@
 import xlrd
 
 from common.handle.base_parse_table_handle import BaseParseTableHandle
-
-max_kb = logging.getLogger("max_kb")
+from common.utils.logger import maxkb_error_logger
 
 
 class XlsParseTableHandle(BaseParseTableHandle):
@@ -56,7 +55,7 @@ def handle(self, file, get_buffer, save_image):
                 result.append({'name': sheet.name, 'paragraphs': paragraphs})
 
         except BaseException as e:
-            max_kb.error(f'excel split handle error: {e}')
+            maxkb_error_logger.error(f'excel split handle error: {e}')
             return [{'name': file.name, 'paragraphs': []}]
         return result
 

apps/common/handle/impl/table/xlsx_parse_table_handle.py

@@ -6,8 +6,7 @@
 
 from common.handle.base_parse_table_handle import BaseParseTableHandle
 from common.handle.impl.common_handle import xlsx_embed_cells_images
-
-max_kb = logging.getLogger("max_kb")
+from common.utils.logger import maxkb_error_logger
 
 
 class XlsxParseTableHandle(BaseParseTableHandle):
@@ -74,7 +73,7 @@ def handle(self, file, get_buffer, save_image):
                 result.append({'name': sheetname, 'paragraphs': paragraphs})
 
         except BaseException as e:
-            max_kb.error(f'excel split handle error: {e}')
+            maxkb_error_logger.error(f'excel split handle error: {e}')
             return [{'name': file.name, 'paragraphs': []}]
         return result
 
@@ -87,7 +86,7 @@ def get_content(self, file, save_image):
                 if len(image_dict) > 0:
                     save_image(image_dict.values())
             except Exception as e:
-                logging.getLogger("max_kb").error(f'Exception: {e}')
+                maxkb_error_logger.error(f'Exception: {e}')
                 image_dict = {}
             md_tables = ''
             # 如果未指定 sheet_name，则使用第一个工作表

apps/common/handle/impl/text/pdf_split_handle.py

@@ -19,6 +19,7 @@
 from langchain_community.document_loaders import PyPDFLoader
 
 from common.handle.base_split_handle import BaseSplitHandle
+from common.utils.logger import maxkb_error_logger, maxkb_logger
 from common.utils.split_model import SplitModel
 
 default_pattern_list = [re.compile('(?<=^)# .*|(?<=\\n)# .*'),
@@ -29,7 +30,6 @@
                         re.compile("(?<=\\n)(?<!#)###### (?!#).*|(?<=^)(?<!#)###### (?!#).*"),
                         re.compile("(?<!\n)\n\n+")]
 
-max_kb = logging.getLogger("max_kb")
 
 
 def check_links_in_pdf(doc):
@@ -72,7 +72,7 @@ def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_bu
             else:
                 split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit)
         except BaseException as e:
-            max_kb.error(f"File: {file.name}, error: {e}")
+            maxkb_error_logger.error(f"File: {file.name}, error: {e}")
             return {'name': file.name,
                     'content': []}
         finally:
@@ -109,7 +109,7 @@ def handle_pdf_content(file, pdf_document):
                     raise e
                 except BaseException as e:
                     # 当页出错继续进行下一页，防止一个页面出错导致整个文件解析失败
-                    max_kb.error(f"File: {file.name}, Page: {page_num + 1}, error: {e}")
+                    maxkb_error_logger.error(f"File: {file.name}, Page: {page_num + 1}, error: {e}")
                     continue
                 finally:
                     os.remove(page_num_pdf)
@@ -120,7 +120,7 @@ def handle_pdf_content(file, pdf_document):
             content = content.replace('\0', '')
 
             elapsed_time = time.time() - start_time
-            max_kb.debug(
+            maxkb_logger.debug(
                 f"File: {file.name}, Page: {page_num + 1}, Time : {elapsed_time: .3f}s,   content-length: {len(page_content)}")
 
         return content
@@ -278,7 +278,7 @@ def handle_links(doc, pattern_list, with_filter, limit):
                     pre_toc[i]['content'] = re.sub(r'(?<!。)\n+', '', pre_toc[i]['content'])
                     pre_toc[i]['content'] = re.sub(r'(?<!.)\n+', '', pre_toc[i]['content'])
             except BaseException as e:
-                max_kb.error(_('This document has no preface and is treated as ordinary text: {e}').format(e=e))
+                maxkb_error_logger.error(_('This document has no preface and is treated as ordinary text: {e}').format(e=e))
                 if pattern_list is not None and len(pattern_list) > 0:
                     split_model = SplitModel(pattern_list, with_filter, limit)
                 else:

apps/common/job/clean_chat_job.py

@@ -10,6 +10,7 @@
 from application.models import Application, Chat, ChatRecord
 from django.db.models import Q, Max
 from common.lock.impl.file_lock import FileLock
+from common.utils.logger import maxkb_logger
 
 from knowledge.models import File
 
@@ -20,7 +21,7 @@
 
 def clean_chat_log_job():
     from django.utils.translation import gettext_lazy as _
-    logging.getLogger("max_kb").info(_('start clean chat log'))
+    maxkb_logger.info(_('start clean chat log'))
     now = timezone.now()
 
     applications = Application.objects.all().values('id', 'clean_time')
@@ -66,7 +67,7 @@ def clean_chat_log_job():
             if deleted_count < batch_size:
                 break
 
-    logging.getLogger("max_kb").info(_('end clean chat log'))
+    maxkb_logger.info(_('end clean chat log'))
 
 
 def run():

apps/common/job/clean_debug_file_job.py

@@ -9,6 +9,7 @@
 from django_apscheduler.jobstores import DjangoJobStore
 
 from common.lock.impl.file_lock import FileLock
+from common.utils.logger import maxkb_logger
 from knowledge.models import File
 
 scheduler = BackgroundScheduler()
@@ -18,11 +19,11 @@
 
 def clean_debug_file():
     from django.utils.translation import gettext_lazy as _
-    logging.getLogger("max_kb").info(_('start clean debug file'))
+    maxkb_logger.info(_('start clean debug file'))
     two_hours_ago = timezone.now() - timedelta(hours=2)
     # 删除对应的文件
     File.objects.filter(Q(create_time__lt=two_hours_ago) & Q(meta__debug=True)).delete()
-    logging.getLogger("max_kb").info(_('end clean debug file'))
+    maxkb_logger.info(_('end clean debug file'))
 
 
 def run():

apps/common/utils/fork.py

@@ -10,6 +10,8 @@
 import requests
 from bs4 import BeautifulSoup
 
+from common.utils.logger import maxkb_error_logger, maxkb_logger
+
 requests.packages.urllib3.disable_warnings()
 
 
@@ -145,7 +147,7 @@ def get_beautiful_soup(response):
                 try:
                     html_content = response.content.decode(charset)
                 except Exception as e:
-                    logging.getLogger("max_kb").error(f'{e}')
+                    maxkb_error_logger.error(f'{e}')
                 return BeautifulSoup(html_content, "html.parser")
         return beautiful_soup
 
@@ -156,14 +158,14 @@ def fork(self):
                 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36'
             }
 
-            logging.getLogger("max_kb").info(f'fork:{self.base_fork_url}')
+            maxkb_logger.info(f'fork:{self.base_fork_url}')
             response = requests.get(self.base_fork_url, verify=False, headers=headers)
             if response.status_code != 200:
-                logging.getLogger("max_kb").error(f"url: {self.base_fork_url} code:{response.status_code}")
+                maxkb_error_logger.error(f"url: {self.base_fork_url} code:{response.status_code}")
                 return Fork.Response.error(f"url: {self.base_fork_url} code:{response.status_code}")
             bf = self.get_beautiful_soup(response)
         except Exception as e:
-            logging.getLogger("max_kb_error").error(f'{str(e)}:{traceback.format_exc()}')
+            maxkb_error_logger.error(f'{str(e)}:{traceback.format_exc()}')
             return Fork.Response.error(str(e))
         bf = self.reset_beautiful_soup(bf)
         link_list = self.get_child_link_list(bf)
@@ -173,6 +175,6 @@ def fork(self):
 
 
 def handler(base_url, response: Fork.Response):
-    logging.getLogger("max_kb").info(base_url.url, base_url.tag.text if base_url.tag else None, response.content)
+    maxkb_logger.info(base_url.url, base_url.tag.text if base_url.tag else None, response.content)
 
 # ForkManage('https://bbs.fit2cloud.com/c/de/6', ['.md-content']).fork(3, set(), handler)

apps/knowledge/serializers/knowledge.py

@@ -30,6 +30,7 @@
 from common.exception.app_exception import AppApiException
 from common.utils.common import valid_license, post, get_file_content, parse_image
 from common.utils.fork import Fork, ChildLink
+from common.utils.logger import maxkb_logger
 from common.utils.split_model import get_split_model
 from knowledge.models import Knowledge, KnowledgeScope, KnowledgeType, Document, Paragraph, Problem, \
     ProblemParagraphMapping, TaskType, State, SearchMode, KnowledgeFolder
@@ -630,7 +631,7 @@ def handler(child_link: ChildLink, response: Fork.Response):
                         document_name = child_link.tag.text if child_link.tag is not None and len(
                             child_link.tag.text.strip()) > 0 else child_link.url
                         paragraphs = get_split_model('web.md').parse(response.content)
-                        logging.getLogger("max_kb").info(child_link.url.strip())
+                        maxkb_logger.info(child_link.url.strip())
                         first = QuerySet(Document).filter(
                             meta__source_url=child_link.url.strip(),
                             knowledge=knowledge