Merge branch 'v2-c' into knowledge_workflow

Author: liuruibin

apps/chat/serializers/chat.py

@@ -173,8 +173,6 @@ def generate_prompt(self, instance: dict):
 
         message = messages[-1]['content']
         q = prompt.replace("{userInput}", message)
-        q = q.replace("{application_name}", application.name)
-        q = q.replace("{detail}", application.desc)
 
         messages[-1]['content'] = q
         SUPPORTED_MODEL_TYPES = ["LLM", "IMAGE"]
@@ -185,13 +183,11 @@ def generate_prompt(self, instance: dict):
         if not model_exist:
             raise Exception(_("Model does not exists or is not an LLM model"))
 
-        system_content = SYSTEM_ROLE.format(application_name=application.name, detail=application.desc)
-
         def process():
             model = get_model_instance_by_model_workspace_id(model_id=model_id, workspace_id=workspace_id,
                                                              **application.model_params_setting)
             try:
-                for r in model.stream([SystemMessage(content=system_content),
+                for r in model.stream([SystemMessage(content=SYSTEM_ROLE),
                                        *[HumanMessage(content=m.get('content')) if m.get(
                                            'role') == 'user' else AIMessage(
                                            content=m.get('content')) for m in messages]]):

apps/chat/template/generate_prompt_system

@@ -62,5 +62,3 @@
 
 输出时不得包含任何解释或附加说明，只能返回符合以上格式的内容。
 
-智能体名称: {application_name}
-功能描述: {detail}

apps/common/utils/tool_code.py

@@ -112,6 +112,7 @@ def exec_code(self, code_str, keywords, function_name=None):
         lines = subprocess_result.stdout.splitlines()
         result_line = [line for line in lines if line.startswith(_id)]
         if not result_line:
+            maxkb_logger.error("\n".join(lines))
             raise Exception("No result found.")
         result = json.loads(base64.b64decode(result_line[-1].split(":", 1)[1]).decode())
         if result.get('code') == 200:

apps/ops/celery/heartbeat.py

@@ -6,20 +6,20 @@
 @heartbeat_sent.connect
 def heartbeat(sender, **kwargs):
     worker_name = sender.eventer.hostname.split('@')[0]
-    heartbeat_path = Path('/tmp/worker_heartbeat_{}'.format(worker_name))
+    heartbeat_path = Path('/opt/maxkb-app/tmp/worker_heartbeat_{}'.format(worker_name))
     heartbeat_path.touch()
 
 
 @worker_ready.connect
 def worker_ready(sender, **kwargs):
     worker_name = sender.hostname.split('@')[0]
-    ready_path = Path('/tmp/worker_ready_{}'.format(worker_name))
+    ready_path = Path('/opt/maxkb-app/tmp/worker_ready_{}'.format(worker_name))
     ready_path.touch()
 
 
 @worker_shutdown.connect
 def worker_shutdown(sender, **kwargs):
     worker_name = sender.hostname.split('@')[0]
     for signal in ['ready', 'heartbeat']:
-        path = Path('/tmp/worker_{}_{}'.format(signal, worker_name))
+        path = Path('/opt/maxkb-app/tmp/worker_{}_{}'.format(signal, worker_name))
         path.unlink(missing_ok=True)

apps/oss/serializers/file.py

@@ -1,14 +1,20 @@
 # coding=utf-8
+import base64
+import ipaddress
 import re
+import socket
 import urllib
+from urllib.parse import urlparse
 
+import requests
 import uuid_utils.compat as uuid
 from django.db.models import QuerySet
 from django.http import HttpResponse
 from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 
-from common.exception.app_exception import NotFound404
+from application.models import Application
+from common.exception.app_exception import NotFound404, AppApiException
 from knowledge.models import File, FileSourceType
 from tools.serializers.tool import UploadedFileField
 
@@ -158,3 +164,80 @@ def delete(self):
             if file is not None:
                 file.delete()
             return True
+
+
+def get_url_content(url, application_id: str):
+    application = Application.objects.filter(id=application_id).first()
+    if application is None:
+        return AppApiException(500, _('Application does not exist'))
+    if not application.file_upload_enable:
+        return AppApiException(500, _('File upload is not enabled'))
+    file_limit = 50 * 1024 * 1024
+    if application.file_upload_setting and application.file_upload_setting.file_limit:
+        file_limit = application.file_upload_setting.file_limit * 1024 * 1024
+    parsed = validate_url(url)
+
+    response = requests.get(
+        url,
+        timeout=3,
+        allow_redirects=False
+    )
+    final_host = urlparse(response.url).hostname
+    if is_private_ip(final_host):
+        raise ValueError("Blocked unsafe redirect to internal host")
+    # 判断文件大小
+    if response.headers.get('Content-Length', 0) > file_limit:
+        return AppApiException(500, _('File size exceeds limit'))
+    # 返回状态码 响应内容大小  响应的contenttype 还有字节流
+    content_type = response.headers.get('Content-Type', '')
+    # 根据内容类型决定如何处理
+    if 'text' in content_type or 'json' in content_type:
+        content = response.text
+    else:
+        # 二进制内容使用Base64编码
+        content = base64.b64encode(response.content).decode('utf-8')
+
+    return {
+        'status_code': response.status_code,
+        'Content-Length': response.headers.get('Content-Length', 0),
+        'Content-Type': content_type,
+        'content': content,
+    }
+
+
+def is_private_ip(host: str) -> bool:
+    """检测 IP 是否属于内网、环回、云 metadata 的危险地址"""
+    try:
+        ip = ipaddress.ip_address(socket.gethostbyname(host))
+        return (
+                ip.is_private or
+                ip.is_loopback or
+                ip.is_reserved or
+                ip.is_link_local or
+                ip.is_multicast
+        )
+    except Exception:
+        return True
+
+
+def validate_url(url: str):
+    """验证 URL 是否安全"""
+    if not url:
+        raise ValueError("URL is required")
+
+    parsed = urlparse(url)
+
+    # 仅允许 http / https
+    if parsed.scheme not in ("http", "https"):
+        raise ValueError("Only http and https are allowed")
+
+    host = parsed.hostname
+    # 域名不能为空
+    if not host:
+        raise ValueError("Invalid URL")
+
+    # 禁止访问内部、保留、环回、云 metadata
+    if is_private_ip(host):
+        raise ValueError("Access to internal IP addresses is blocked")
+
+    return parsed

apps/oss/views/file.py

@@ -1,21 +1,14 @@
 # coding=utf-8
-import base64
-import ipaddress
-import socket
-from urllib.parse import urlparse
-
-import requests
 from django.utils.translation import gettext_lazy as _
 from drf_spectacular.utils import extend_schema
 from rest_framework.parsers import MultiPartParser
 from rest_framework.views import APIView
 from rest_framework.views import Request
-
 from common.auth import TokenAuth
 from common.log.log import log
 from common.result import result
 from knowledge.api.file import FileUploadAPI, FileGetAPI
-from oss.serializers.file import FileSerializer
+from oss.serializers.file import FileSerializer, get_url_content
 
 
 class FileRetrievalView(APIView):
@@ -84,71 +77,7 @@ class GetUrlView(APIView):
         operation_id=_('Get url'),  # type: ignore
         tags=[_('Chat')]  # type: ignore
     )
-    def get(self, request: Request):
+    def get(self, request: Request, application_id: str):
         url = request.query_params.get('url')
-        parsed = validate_url(url)
-
-        response = requests.get(
-            url,
-            timeout=3,
-            allow_redirects=False
-        )
-        final_host = urlparse(response.url).hostname
-        if is_private_ip(final_host):
-            raise ValueError("Blocked unsafe redirect to internal host")
-
-        # 返回状态码 响应内容大小  响应的contenttype 还有字节流
-        content_type = response.headers.get('Content-Type', '')
-        # 根据内容类型决定如何处理
-        if 'text' in content_type or 'json' in content_type:
-            content = response.text
-        else:
-            # 二进制内容使用Base64编码
-            content = base64.b64encode(response.content).decode('utf-8')
-
-        return result.success({
-            'status_code': response.status_code,
-            'Content-Length': response.headers.get('Content-Length', 0),
-            'Content-Type': content_type,
-            'content': content,
-        })
-
-
-def is_private_ip(host: str) -> bool:
-    """检测 IP 是否属于内网、环回、云 metadata 的危险地址"""
-    try:
-        ip = ipaddress.ip_address(socket.gethostbyname(host))
-        return (
-                ip.is_private or
-                ip.is_loopback or
-                ip.is_reserved or
-                ip.is_link_local or
-                ip.is_multicast
-        )
-    except Exception:
-        return True
-
-
-def validate_url(url: str):
-    """验证 URL 是否安全"""
-    if not url:
-        raise ValueError("URL is required")
-
-    parsed = urlparse(url)
-
-    # 仅允许 http / https
-    if parsed.scheme not in ("http", "https"):
-        raise ValueError("Only http and https are allowed")
-
-    host = parsed.hostname
-    path = parsed.path
-
-    # 域名不能为空
-    if not host:
-        raise ValueError("Invalid URL")
-
-    # 禁止访问内部、保留、环回、云 metadata
-    if is_private_ip(host):
-        raise ValueError("Access to internal IP addresses is blocked")
-
-    return parsed
+        result_data = get_url_content(url, application_id)
+        return result.success(result_data)