Feat/2.3.0 beta3 (#1832)

Author: zgqgit

.gitmodules

@@ -1,6 +0,0 @@
-[submodule "src/bisheng-unstructured"]
-	path = src/bisheng-unstructured
-	url = https://github.com/dataelement/bisheng-unstructured.git
-[submodule "src/bisheng-rt"]
-	path = src/bisheng-rt
-	url = https://github.com/dataelement/bisheng-rt.git

docker/docker-compose.yml

@@ -40,7 +40,7 @@ services:
 
   backend:
     container_name: bisheng-backend
-    image: dataelement/bisheng-backend:v2.3.0-beta2
+    image: dataelement/bisheng-backend:v2.3.0-beta3
     ports:
       - "7860:7860"
     environment:
@@ -78,7 +78,7 @@ services:
 
   backend_worker:
     container_name: bisheng-backend-worker
-    image: dataelement/bisheng-backend:v2.3.0-beta2
+    image: dataelement/bisheng-backend:v2.3.0-beta3
     environment:
       TZ: Asia/Shanghai
       BS_MILVUS_CONNECTION_ARGS: '{"host":"milvus","port":"19530","user":"","password":"","secure":false}'
@@ -109,7 +109,7 @@ services:
 
   frontend:
     container_name: bisheng-frontend
-    image: dataelement/bisheng-frontend:v2.3.0-beta2
+    image: dataelement/bisheng-frontend:v2.3.0-beta3
     ports:
       - "3001:3001"
     environment:

src/backend/bisheng/__init__.py

@@ -7,7 +7,7 @@
 
 try:
     # 通过ci去自动修改
-    __version__ = '2.3.0-beta2'
+    __version__ = '2.3.0-beta3'
 except metadata.PackageNotFoundError:
     # Case where package metadata is not available.
     __version__ = ''

src/backend/bisheng/api/services/assistant.py

@@ -1,5 +1,4 @@
 from datetime import datetime
-from datetime import datetime
 from typing import Any, List, Optional, Union
 
 from fastapi import Request
@@ -297,7 +296,7 @@ async def update_assistant(cls, request: Request, login_user: UserPayload, req:
                 raise AssistantNameRepeatError()
             assistant.name = req.name
         assistant.desc = req.desc
-        assistant.logo = req.logo
+        assistant.logo = req.logo if req.logo else assistant.logo
         assistant.prompt = req.prompt
         assistant.guide_word = req.guide_word
         assistant.guide_question = req.guide_question

src/backend/bisheng/api/services/audit_log.py

@@ -1,7 +1,7 @@
 import csv
 from datetime import datetime
 from tempfile import NamedTemporaryFile
-from typing import Any, List, Optional
+from typing import Any, List, Optional, Dict
 
 from loguru import logger
 
@@ -49,18 +49,21 @@ def get_audit_log(cls, login_user: UserPayload, group_ids, operator_ids, start_t
         return resp_200(data={'data': data, 'total': total})
 
     @classmethod
-    def get_all_operators(cls, login_user: UserPayload) -> Any:
+    def get_all_operators(cls, login_user: UserPayload) -> List[Dict]:
         groups = []
         if not login_user.is_admin():
             groups = [one.group_id for one in UserGroupDao.get_user_admin_group(login_user.user_id)]
+            # not any group admin
+            if not groups:
+                raise UnAuthorizedError()
 
         data = AuditLogDao.get_all_operators(groups)
         res = {}
         for one in data:
             if not one[1]:
                 continue
             res[one[0]] = {'user_id': one[0], 'user_name': one[1]}
-        return resp_200(data=list(res.values()))
+        return list(res.values())
 
     @classmethod
     def _chat_log(cls, user: UserPayload, ip_address: str, event_type: EventType, object_type: ObjectType,
@@ -599,8 +602,7 @@ def export_session_messages(cls, user: UserPayload, flow_ids: List[str], user_id
             minio_client.put_object_sync(object_name=tmp_object_name, file=tmp_file.name,
                                          content_type='application/text',
                                          bucket_name=minio_client.tmp_bucket)
-        share_url = minio_client.get_share_link(tmp_object_name, minio_client.tmp_bucket)
-        return minio_client.clear_minio_share_host(share_url)
+        return minio_client.get_share_link_sync(tmp_object_name, minio_client.tmp_bucket)
 
     @classmethod
     def get_chat_messages(cls, chat_list: List[AppChatList]) -> List[AppChatList]:

src/backend/bisheng/api/services/base.py

@@ -12,7 +12,7 @@ def get_logo_share_link(cls, logo_path: str):
         redis_client = get_redis_client_sync()
         if not logo_path:
             return ''
-        cache_key = f'logo_cache:{logo_path}'
+        cache_key = f'logo_cache_new:{logo_path}'
         # 先从内存中获取
         share_url = cls.LogoMemoryCache.get(cache_key)
         if share_url:
@@ -25,9 +25,7 @@ def get_logo_share_link(cls, logo_path: str):
             return share_url
 
         minio_client = get_minio_storage_sync()
-        share_url = minio_client.get_share_link(logo_path)
-        # 去除前缀通过nginx访问，防止访问不到文件
-        share_url = minio_client.clear_minio_share_host(share_url)
+        share_url = minio_client.get_share_link_sync(logo_path)
 
         # 缓存5天， 临时链接有效期为7天
         redis_client.set(cache_key, share_url, 3600 * 120)
@@ -40,7 +38,7 @@ async def get_logo_share_link_async(cls, logo_path: str):
         redis_client = await get_redis_client()
         if not logo_path:
             return ''
-        cache_key = f'logo_cache:{logo_path}'
+        cache_key = f'logo_cache_new:{logo_path}'
         # 先从内存中获取
         share_url = cls.LogoMemoryCache.get(cache_key)
         if share_url:
@@ -53,11 +51,9 @@ async def get_logo_share_link_async(cls, logo_path: str):
             return share_url
 
         minio_client = await get_minio_storage()
-        share_url = minio_client.get_share_link(logo_path)
-        # 去除前缀通过nginx访问，防止访问不到文件
-        share_url = minio_client.clear_minio_share_host(share_url)
+        share_url = await minio_client.get_share_link(logo_path)
 
         # 缓存5天， 临时链接有效期为7天
         await redis_client.aset(cache_key, share_url, 3600 * 120)
         cls.LogoMemoryCache.set(cache_key, share_url)
-        return share_url
+        return share_url

src/backend/bisheng/api/services/dataset_service.py

@@ -16,7 +16,7 @@ class DatasetService(BaseService):
     def build_dataset_list(cls,
                            page: int,
                            limit: int,
-                           keyword: Optional[str] = None) -> List[Dict]:
+                           keyword: Optional[str] = None) -> (List[Dict], int):
         """补全list 数据"""
 
         dataset_list = DatasetDao.filter_dataset_by_ids(dataset_ids=[],
@@ -31,12 +31,11 @@ def build_dataset_list(cls,
         user_ids = [one.user_id for one in dataset_list]
         user_list = UserDao.get_user_by_ids(user_ids)
         user_dict = {one.user_id: one for one in user_list}
-        res = [DatasetRead.validate(one) for one in dataset_list]
-        minio_client = get_minio_storage_sync()
+        res = [DatasetRead.model_validate(one) for one in dataset_list]
         for one in res:
             one.user_name = user_dict[one.user_id].user_name
             if one.object_name:
-                one.url = minio_client.get_share_link(one.object_name)
+                one.url = one.object_name
 
         return res, total_count
 
@@ -74,3 +73,7 @@ def delete_dataset(cls, dataset_id: int):
             minio_client.remove_object_sync(object_name=object_name)
         DatasetDao.delete(dataset)
         return True
+
+    @classmethod
+    async def get_one_by_object_name(cls, object_name: str) -> Optional[Dataset]:
+        dataset = await DatasetDao.aget_dataset_by_object_name(object_name)

src/backend/bisheng/api/services/knowledge.py

@@ -78,6 +78,7 @@ async def get_knowledge(
             login_user: UserPayload,
             knowledge_type: KnowledgeTypeEnum,
             name: str = None,
+            sort_by: str = "update_time",
             page: int = 1,
             limit: int = 10,
     ) -> (List[KnowledgeRead], int):
@@ -96,6 +97,7 @@ async def get_knowledge(
                 knowledge_id_extra,
                 knowledge_type,
                 name,
+                sort_by,
                 page,
                 limit,
             )
@@ -104,7 +106,7 @@ async def get_knowledge(
             )
         else:
             res = await KnowledgeDao.aget_all_knowledge(
-                name, knowledge_type, page=page, limit=limit
+                name, knowledge_type, sort_by, page=page, limit=limit
             )
             total = await KnowledgeDao.acount_all_knowledge(name, knowledge_type)
 
@@ -461,7 +463,7 @@ async def get_preview_file_chunk(
             new_file_name = KnowledgeUtils.get_tmp_preview_file_object_name(filepath)
             minio_client = await get_minio_storage()
             if await minio_client.object_exists(minio_client.tmp_bucket, new_file_name):
-                file_share_url = minio_client.get_share_link(
+                file_share_url = await minio_client.get_share_link(
                     new_file_name, minio_client.tmp_bucket
                 )
 
@@ -623,9 +625,7 @@ async def rebuild_knowledge_file(cls, request: Request,
         db_file.updater_name = login_user.user_name
         db_file = await KnowledgeFileDao.async_update(db_file)
 
-        file_path, _ = cls.get_file_share_url(db_file.id)
-
-        preview_cache_key = cls.get_preview_cache_key(req_data.knowledge_id, file_path=file_path)
+        preview_cache_key = cls.get_preview_cache_key(req_data.knowledge_id, file_path=req_data.file_path)
         file_worker.retry_knowledge_file_celery.delay(db_file.id, preview_cache_key, req_data.callback_url)
 
         return db_file.model_dump()
@@ -1172,13 +1172,13 @@ def get_file_share_url(cls, file_id: int) -> (str, str):
         """ 获取文件原始下载地址 和 对应的预览文件下载地址 """
         file = KnowledgeFileDao.get_file_by_ids([file_id])
         if not file:
-            raise NotFoundError.http_exception()
+            raise NotFoundError()
         file = file[0]
         minio_client = get_minio_storage_sync()
         # 130版本以前的文件解析
         if file.parse_type in [ParseType.LOCAL.value, ParseType.UNS.value]:
-            original_url = minio_client.get_share_link(cls.get_knowledge_file_object_name(file.id, file.file_name))
-            preview_url = minio_client.get_share_link(str(file.id))
+            original_url = minio_client.get_share_link_sync(cls.get_knowledge_file_object_name(file.id, file.file_name))
+            preview_url = minio_client.get_share_link_sync(str(file.id))
         else:
             original_url = cls.get_file_share_url_with_empty(file.object_name)
             preview_url = ""
@@ -1197,7 +1197,7 @@ def get_file_share_url_with_empty(cls, object_name: str) -> str:
         """
         minio_client = get_minio_storage_sync()
         if minio_client.object_exists_sync(minio_client.bucket, object_name):
-            return minio_client.get_share_link(object_name, minio_client.bucket)
+            return minio_client.get_share_link_sync(object_name)
         return ""
 
     @classmethod

src/backend/bisheng/api/services/knowledge_imp.py

@@ -560,7 +560,7 @@ def add_file_embedding(
         f"start download original file={db_file.id} file_name={db_file.file_name}"
     )
 
-    file_url = minio_client.get_share_link(db_file.object_name)
+    file_url = minio_client.get_share_link_sync(db_file.object_name, clear_host=False)
     filepath, _ = file_download(file_url)
 
     # Convert split_rule string to dict if needed

src/backend/bisheng/api/services/libreoffice_converter.py

@@ -1,6 +1,7 @@
 import os
 import shutil  # For checking if the executable is in PATH
 import subprocess
+import tempfile
 
 from loguru import logger
 
@@ -85,22 +86,25 @@ def convert_doc_to_docx(input_doc_path, output_dir=None):
     file_name_no_ext = os.path.splitext(base_name)[0]
     output_docx_path = os.path.join(output_dir, f"{file_name_no_ext}.docx")
 
-    command = [
-        soffice_path,
-        "--headless",  # Run in headless mode (no GUI)
-        "--convert-to",
-        "docx",  # Specify the output format
-        "--outdir",
-        output_dir,  # Specify the output directory
-        input_doc_path,  # The input file
-    ]
-
-    logger.debug(f"Executing command: {' '.join(command)}")
-
     try:
-        process = subprocess.run(
-            command, check=True, capture_output=True, text=True, timeout=120
-        )  # 120 seconds timeout
+        with tempfile.TemporaryDirectory() as temp_dir:
+            command = [
+                soffice_path,
+                "--headless",  # Run in headless mode (no GUI)
+                "--norestore",
+                "--invisible",
+                "-env:SingleAppInstance=false",
+                f"-env:UserInstallation=file://{temp_dir}",
+                "--convert-to",
+                "docx:Office Open XML Text",  # Specify the output format
+                "--outdir",
+                output_dir,  # Specify the output directory
+                input_doc_path,  # The input file
+            ]
+            logger.debug(f"Executing command: {' '.join(command)}")
+            process = subprocess.run(
+                command, check=True, capture_output=True, text=True, timeout=120
+            )  # 120 seconds timeout
         logger.debug(f"LibreOffice STDOUT: {process.stdout}")
         if (
                 process.stderr
@@ -192,23 +196,27 @@ def convert_ppt_to_pdf(input_path, output_dir=None):
     pdf_name = os.path.splitext(base_name)[0] + ".pdf"
     expected_pdf_path = os.path.join(output_dir, pdf_name)
 
-    command = [
-        soffice_path,
-        "--headless",
-        "--convert-to",
-        "pdf",
-        "--outdir",
-        output_dir,
-        input_path,
-    ]
-
     try:
-        logger.debug(f"Converting {input_path} to PDF using {soffice_path}...")
-        # LibreOffice can sometimes be slow to start up and convert.
-        # It may also not provide much stdout/stderr unless there's a significant error.
-        process = subprocess.run(
-            command, capture_output=True, text=True, check=True, timeout=180
-        )  # 180 seconds timeout
+        with tempfile.TemporaryDirectory() as temp_dir:
+            command = [
+                soffice_path,
+                "--headless",
+                "--norestore",
+                "--invisible",
+                "-env:SingleAppInstance=false",
+                f"-env:UserInstallation=file://{temp_dir}",
+                "--convert-to",
+                "pdf",
+                "--outdir",
+                output_dir,
+                input_path,
+            ]
+            logger.debug(f"Converting {input_path} to PDF using {soffice_path}...")
+            # LibreOffice can sometimes be slow to start up and convert.
+            # It may also not provide much stdout/stderr unless there's a significant error.
+            process = subprocess.run(
+                command, capture_output=True, text=True, check=True, timeout=180
+            )  # 180 seconds timeout
 
         if process.stdout:
             logger.debug(f"soffice stdout: {process.stdout}")  # Often empty on success