Merge branch 'v2-c' into knowledge_workflow

Author: liuruibin

apps/common/utils/tool_code.py

@@ -7,6 +7,8 @@
 import socket
 import subprocess
 import sys
+import signal
+import time
 import uuid_utils.compat as uuid
 from common.utils.logger import maxkb_logger
 from django.utils.translation import gettext_lazy as _
@@ -84,13 +86,14 @@ def exec_code(self, code_str, keywords, function_name=None):
         python_paths = CONFIG.get_sandbox_python_package_paths().split(',')
         _exec_code = f"""
 try:
-    import sys, json, base64, builtins
+    import os, sys, json, base64, builtins
     path_to_exclude = ['/opt/py3/lib/python3.11/site-packages', '/opt/maxkb-app/apps']
     sys.path = [p for p in sys.path if p not in path_to_exclude]
     sys.path += {python_paths}
     locals_v={'{}'}
     keywords={keywords}
     globals_v={'{}'}
+    os.environ.clear()
     exec({dedent(code_str)!a}, globals_v, locals_v)
     f_name, f = {action_function}
     for local in locals_v:
@@ -182,16 +185,14 @@ def generate_mcp_server_code(self, code_str, params):
         python_paths = CONFIG.get_sandbox_python_package_paths().split(',')
         code = self._generate_mcp_server_code(code_str, params)
         return f"""
-import os
-import sys
-import logging
+import os, sys, logging
 logging.basicConfig(level=logging.WARNING)
 logging.getLogger("mcp").setLevel(logging.ERROR)
 logging.getLogger("mcp.server").setLevel(logging.ERROR)
-
 path_to_exclude = ['/opt/py3/lib/python3.11/site-packages', '/opt/maxkb-app/apps']
 sys.path = [p for p in sys.path if p not in path_to_exclude]
 sys.path += {python_paths}
+os.environ.clear()
 exec({dedent(code)!a})
 """
 
@@ -223,24 +224,40 @@ def get_tool_mcp_config(self, code, params):
         return tool_config
 
     def _exec_sandbox(self, _code):
-        kwargs = {'cwd': BASE_DIR}
-        kwargs['env'] = {
+        kwargs = {'cwd': BASE_DIR, 'env': {
             'LD_PRELOAD': self.sandbox_so_path,
-        }
+        }}
         maxkb_logger.debug(f"Sandbox execute code: {_code}")
         compressed_and_base64_encoded_code_str = base64.b64encode(gzip.compress(_code.encode())).decode()
+        cmd = [
+            'su', '-s', python_directory, '-c',
+            f'import base64,gzip; exec(gzip.decompress(base64.b64decode(\'{compressed_and_base64_encoded_code_str}\')).decode())',
+            self.user
+        ]
         try:
-            subprocess_result = subprocess.run(
-                ['su', '-s', python_directory, '-c',
-                 f'import base64,gzip; exec(gzip.decompress(base64.b64decode(\'{compressed_and_base64_encoded_code_str}\')).decode())',
-                 self.user],
+            proc = subprocess.Popen(
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
                 text=True,
-                capture_output=True,
-                timeout=self.process_timeout_seconds,
-                **kwargs)
+                **kwargs,
+                start_new_session=True
+            )
+            proc.wait(timeout=self.process_timeout_seconds)
+            return subprocess.CompletedProcess(
+                proc.args,
+                proc.returncode,
+                proc.stdout.read(),
+                proc.stderr.read()
+            )
         except subprocess.TimeoutExpired:
+            pgid = os.getpgid(proc.pid)
+            os.killpg(pgid, signal.SIGTERM) #温和终止
+            time.sleep(1) #留出短暂时间让进程清理
+            if proc.poll() is None: #如果仍未终止，强制终止
+                os.killpg(pgid, signal.SIGKILL)
+            proc.wait()
             raise Exception(_("Sandbox process execution timeout, consider increasing MAXKB_SANDBOX_PYTHON_PROCESS_TIMEOUT_SECONDS."))
-        return subprocess_result
 
     def validate_mcp_transport(self, code_str):
         servers = json.loads(code_str)

apps/oss/views/file.py

@@ -1,5 +1,8 @@
 # coding=utf-8
 import base64
+import ipaddress
+import socket
+from urllib.parse import urlparse
 
 import requests
 from django.utils.translation import gettext_lazy as _
@@ -83,7 +86,17 @@ class GetUrlView(APIView):
     )
     def get(self, request: Request):
         url = request.query_params.get('url')
-        response = requests.get(url)
+        parsed = validate_url(url)
+
+        response = requests.get(
+            url,
+            timeout=3,
+            allow_redirects=False
+        )
+        final_host = urlparse(response.url).hostname
+        if is_private_ip(final_host):
+            raise ValueError("Blocked unsafe redirect to internal host")
+
         # 返回状态码 响应内容大小  响应的contenttype 还有字节流
         content_type = response.headers.get('Content-Type', '')
         # 根据内容类型决定如何处理
@@ -99,3 +112,43 @@ def get(self, request: Request):
             'Content-Type': content_type,
             'content': content,
         })
+
+
+def is_private_ip(host: str) -> bool:
+    """检测 IP 是否属于内网、环回、云 metadata 的危险地址"""
+    try:
+        ip = ipaddress.ip_address(socket.gethostbyname(host))
+        return (
+                ip.is_private or
+                ip.is_loopback or
+                ip.is_reserved or
+                ip.is_link_local or
+                ip.is_multicast
+        )
+    except Exception:
+        return True
+
+
+def validate_url(url: str):
+    """验证 URL 是否安全"""
+    if not url:
+        raise ValueError("URL is required")
+
+    parsed = urlparse(url)
+
+    # 仅允许 http / https
+    if parsed.scheme not in ("http", "https"):
+        raise ValueError("Only http and https are allowed")
+
+    host = parsed.hostname
+    path = parsed.path
+
+    # 域名不能为空
+    if not host:
+        raise ValueError("Invalid URL")
+
+    # 禁止访问内部、保留、环回、云 metadata
+    if is_private_ip(host):
+        raise ValueError("Access to internal IP addresses is blocked")
+
+    return parsed

apps/system_manage/serializers/user_resource_permission.py

@@ -9,6 +9,7 @@
 import json
 import os
 
+from django.contrib.postgres.fields import ArrayField
 from django.core.cache import cache
 from django.db import models
 from django.db.models import QuerySet, Q, TextField
@@ -343,10 +344,13 @@ def get_queryset(self, instance, is_x_pack_ee: bool):
             "role": models.CharField(),
             "role_setting.type": models.CharField(),
             "user_role_relation.workspace_id": models.CharField(),
+            'tmp.type_list': ArrayField(models.CharField()),
+            'tmp.role_name_list_str': models.CharField()
 
         }))
         nick_name = instance.get('nick_name')
         username = instance.get('username')
+        role_name = instance.get('role')
         permission = instance.get('permission')
         query_p_list = [None if p == "NOT_AUTH" else p for p in permission]
 
@@ -375,15 +379,31 @@ def get_queryset(self, instance, is_x_pack_ee: bool):
                 **{"u.id__in": QuerySet(workspace_user_role_mapping_model).filter(
                     workspace_id=self.data.get('workspace_id')).values("user_id")})
         if is_x_pack_ee:
-            user_query_set = user_query_set.filter(
-                **{'role_setting.type': "USER", 'user_role_relation.workspace_id': self.data.get('workspace_id')})
+            user_query_set = user_query_set.filter(**{
+                "tmp.type_list__contains": ["USER"]
+            })
+            role_name_and_type_query_set = QuerySet(model=get_dynamics_model({
+            'user_role_relation.workspace_id': models.CharField(),
+        })).filter(**{
+                "user_role_relation.workspace_id": self.data.get('workspace_id'),
+            })
+            if role_name:
+                user_query_set = user_query_set.filter(
+                    **{'tmp.role_name_list_str__icontains': str(role_name)}
+                )
+
+            return {
+                'workspace_user_resource_permission_query_set': workspace_user_resource_permission_query_set,
+                'user_query_set': user_query_set,
+                'role_name_and_type_query_set': role_name_and_type_query_set
+            }
         else:
             user_query_set = user_query_set.filter(
                 **{'role': "USER"})
-        return {
-            'workspace_user_resource_permission_query_set': workspace_user_resource_permission_query_set,
-            'user_query_set': user_query_set
-        }
+            return {
+                'workspace_user_resource_permission_query_set': workspace_user_resource_permission_query_set,
+                'user_query_set': user_query_set
+            }
 
     def list(self, instance, with_valid=True):
         if with_valid:

apps/system_manage/sql/get_resource_user_permission_detail_ee.sql

@@ -1,34 +1,41 @@
 SELECT
-    distinct(u.id),
+    DISTINCT u.id,
     u.nick_name,
     u.username,
-    case
-		when
-	      wurp."permission" is null then 'NOT_AUTH'
-		else wurp."permission"
-	end
+    tmp.role_name_list AS role_name,
+    CASE
+        WHEN wurp."permission" IS NULL THEN 'NOT_AUTH'
+        ELSE wurp."permission"
+    END AS permission
 FROM
     public."user" u
 LEFT JOIN (
     SELECT
-        user_id ,
-	(case
-		when  auth_type = 'ROLE'
-		and  'ROLE' = any( permission_list) then 'ROLE'
-			when  auth_type = 'RESOURCE_PERMISSION_GROUP'
-			and 'MANAGE'= any(permission_list)   then 'MANAGE'
-			  when  auth_type = 'RESOURCE_PERMISSION_GROUP'
-			and 'VIEW' = any( permission_list) then 'VIEW'
-			else null
-		end) as "permission"
+        user_id,
+        CASE
+            WHEN auth_type = 'ROLE'
+                 AND 'ROLE' = ANY(permission_list) THEN 'ROLE'
+            WHEN auth_type = 'RESOURCE_PERMISSION_GROUP'
+                 AND 'MANAGE' = ANY(permission_list) THEN 'MANAGE'
+            WHEN auth_type = 'RESOURCE_PERMISSION_GROUP'
+                 AND 'VIEW' = ANY(permission_list) THEN 'VIEW'
+            ELSE NULL
+        END AS "permission"
     FROM
         workspace_user_resource_permission
-        ${workspace_user_resource_permission_query_set}
-        ) wurp
-ON
-    u.id = wurp.user_id
-left join user_role_relation user_role_relation
-on user_role_relation.user_id = u.id
-left join role_setting role_setting
-on role_setting.id = user_role_relation.role_id
+    ${workspace_user_resource_permission_query_set}
+) wurp ON u.id = wurp.user_id
+LEFT JOIN (
+    SELECT
+        ARRAY_AGG(role_setting.role_name) AS role_name_list,
+        ARRAY_AGG(role_setting.role_name)::text AS role_name_list_str,
+        ARRAY_AGG(role_setting.type) AS type_list,
+        user_role_relation.user_id
+    FROM user_role_relation user_role_relation
+    LEFT JOIN role_setting role_setting
+    ON role_setting.id = user_role_relation.role_id
+    ${role_name_and_type_query_set}
+    GROUP BY
+    user_role_relation.user_id) tmp
+ON u.id = tmp.user_id
 ${user_query_set}

apps/system_manage/views/user_resource_permission.py

@@ -196,6 +196,7 @@ def get(self, request: Request, workspace_id: str, target: str, resource: str, c
             return result.success(ResourceUserPermissionSerializer(
                 data={'workspace_id': workspace_id, "target": target, 'auth_target_type': resource, }
             ).page({'username': request.query_params.get("username"),
+                    'role': request.query_params.get("role"),
                     'nick_name': request.query_params.get("nick_name"),
                     'permission': request.query_params.getlist("permission[]")}, current_page, page_size,
                    ))

apps/tools/serializers/tool.py

@@ -431,7 +431,7 @@ def get_field_value(debug_field_list, name, is_required):
 
         @staticmethod
         def convert_value(name: str, value: str, _type: str, is_required: bool):
-            if not is_required and value is None:
+            if not is_required and (value is None or (isinstance(value, str) and len(value.strip()) == 0)):
                 return None
             try:
                 if _type == 'int':