Merge branch 'develop' into patch-1

Author: WMC001

backend/agents/create_agent_info.py

@@ -9,6 +9,7 @@
 from nexent.core.agents.agent_model import AgentRunInfo, ModelConfig, AgentConfig, ToolConfig
 from nexent.memory.memory_service import search_memory_in_levels
 
+from services.file_management_service import get_llm_model
 from services.vectordatabase_service import (
     ElasticSearchService,
     get_vector_db_core,
@@ -17,13 +18,15 @@
 from services.tenant_config_service import get_selected_knowledge_list
 from services.remote_mcp_service import get_remote_mcp_server_list
 from services.memory_config_service import build_memory_context
+from services.image_service import get_vlm_model
 from database.agent_db import search_agent_info_by_agent_id, query_sub_agents_id_list
 from database.tool_db import search_tools_for_sub_agent
 from database.model_management_db import get_model_records, get_model_by_model_id
+from database.client import minio_client
 from utils.model_name_utils import add_repo_to_name
 from utils.prompt_template_utils import get_agent_prompt_template
 from utils.config_utils import tenant_config_manager, get_model_name_from_config
-from consts.const import LOCAL_MCP_SERVER, MODEL_CONFIG_MAPPING, LANGUAGE
+from consts.const import LOCAL_MCP_SERVER, MODEL_CONFIG_MAPPING, LANGUAGE, DATA_PROCESS_SERVICE
 
 logger = logging.getLogger("create_agent_info")
 logger.setLevel(logging.DEBUG)
@@ -236,6 +239,18 @@ async def create_tool_config_list(agent_id, tenant_id, user_id):
                 "vdb_core": get_vector_db_core(),
                 "embedding_model": get_embedding_model(tenant_id=tenant_id),
             }
+        elif tool_config.class_name == "AnalyzeTextFileTool":
+            tool_config.metadata = {
+                "llm_model": get_llm_model(tenant_id=tenant_id),
+                "storage_client": minio_client,
+                "data_process_service_url": DATA_PROCESS_SERVICE
+            }
+        elif tool_config.class_name == "AnalyzeImageTool":
+            tool_config.metadata = {
+                "vlm_model": get_vlm_model(tenant_id=tenant_id),
+                "storage_client": minio_client,
+            }
+
         tool_config_list.append(tool_config)
 
     return tool_config_list

backend/consts/const.py

@@ -279,7 +279,7 @@ class VectorDatabaseType(str, Enum):
     os.getenv("LLM_SLOW_TOKEN_RATE_THRESHOLD", "10.0"))  # tokens per second
 
 # APP Version
-APP_VERSION = "v1.7.6"
+APP_VERSION = "v1.7.7"
 
 DEFAULT_ZH_TITLE = "新对话"
 DEFAULT_EN_TITLE = "New Conversation"

backend/services/file_management_service.py

@@ -10,7 +10,7 @@
 from fastapi import UploadFile
 
 from agents.preprocess_manager import preprocess_manager
-from consts.const import UPLOAD_FOLDER, MAX_CONCURRENT_UPLOADS, DATA_PROCESS_SERVICE, LANGUAGE
+from consts.const import UPLOAD_FOLDER, MAX_CONCURRENT_UPLOADS, DATA_PROCESS_SERVICE, LANGUAGE, MODEL_CONFIG_MAPPING
 from database.attachment_db import (
     upload_fileobj,
     get_file_url,
@@ -19,11 +19,15 @@
     delete_file,
     list_files
 )
-from utils.attachment_utils import convert_image_to_text, convert_long_text_to_text
 from services.vectordatabase_service import ElasticSearchService, get_vector_db_core
+from utils.attachment_utils import convert_image_to_text, convert_long_text_to_text
+from utils.config_utils import tenant_config_manager, get_model_name_from_config
 from utils.prompt_template_utils import get_file_processing_messages_template
 from utils.file_management_utils import save_upload_file
 
+from nexent import MessageObserver
+from nexent.core.models import OpenAILongContextModel
+
 # Create upload directory
 upload_dir = Path(UPLOAD_FOLDER)
 upload_dir.mkdir(exist_ok=True)
@@ -405,3 +409,16 @@ def get_file_description(files: List[UploadFile]) -> str:
         else:
             description += f"- File {file.filename or ''}\n"
     return description
+
+def get_llm_model(tenant_id: str):
+    # Get the tenant config
+    main_model_config = tenant_config_manager.get_model_config(
+        key=MODEL_CONFIG_MAPPING["llm"], tenant_id=tenant_id)
+    long_text_to_text_model = OpenAILongContextModel(
+        observer=MessageObserver(),
+        model_id=get_model_name_from_config(main_model_config),
+        api_base=main_model_config.get("base_url"),
+        api_key=main_model_config.get("api_key"),
+        max_context_tokens=main_model_config.get("max_tokens")
+    )
+    return long_text_to_text_model

backend/services/image_service.py

@@ -4,6 +4,11 @@
 import aiohttp
 
 from consts.const import DATA_PROCESS_SERVICE
+from consts.const import MODEL_CONFIG_MAPPING
+from utils.config_utils import tenant_config_manager, get_model_name_from_config
+
+from nexent import MessageObserver
+from nexent.core.models import OpenAIVLModel
 
 logger = logging.getLogger("image_service")
 
@@ -23,3 +28,19 @@ async def proxy_image_impl(decoded_url: str):
 
             result = await response.json()
             return result
+
+def get_vlm_model(tenant_id: str):
+    # Get the tenant config
+    vlm_model_config = tenant_config_manager.get_model_config(
+        key=MODEL_CONFIG_MAPPING["vlm"], tenant_id=tenant_id)
+    return OpenAIVLModel(
+                observer=MessageObserver(),
+                model_id=get_model_name_from_config(
+                    vlm_model_config) if vlm_model_config else "",
+                api_base=vlm_model_config.get("base_url", ""),
+                api_key=vlm_model_config.get("api_key", ""),
+                temperature=0.7,
+                top_p=0.7,
+                frequency_penalty=0.5,
+                max_tokens=512
+            )

backend/services/tool_configuration_service.py

@@ -11,7 +11,7 @@
 import jsonref
 from mcpadapt.smolagents_adapter import _sanitize_function_name
 
-from consts.const import DEFAULT_USER_ID, LOCAL_MCP_SERVER
+from consts.const import DEFAULT_USER_ID, LOCAL_MCP_SERVER, DATA_PROCESS_SERVICE
 from consts.exceptions import MCPConnectionError, ToolExecutionException, NotFoundException
 from consts.model import ToolInstanceInfoRequest, ToolInfo, ToolSourceEnum, ToolValidateRequest
 from database.remote_mcp_db import get_mcp_records_by_tenant, get_mcp_server_by_name_and_tenant
@@ -23,8 +23,11 @@
     search_last_tool_instance_by_tool_id,
 )
 from database.user_tenant_db import get_all_tenant_ids
+from services.file_management_service import get_llm_model
 from services.vectordatabase_service import get_embedding_model, get_vector_db_core
 from services.tenant_config_service import get_selected_knowledge_list
+from database.client import minio_client
+from services.image_service import get_vlm_model
 
 logger = logging.getLogger("tool_configuration_service")
 
@@ -613,6 +616,27 @@ def _validate_local_tool(
                 'embedding_model': embedding_model,
             }
             tool_instance = tool_class(**params)
+        elif tool_name == "analyze_image":
+            if not tenant_id or not user_id:
+                raise ToolExecutionException(f"Tenant ID and User ID are required for {tool_name} validation")
+            image_to_text_model = get_vlm_model(tenant_id=tenant_id)
+            params = {
+                **instantiation_params,
+                'vlm_model': image_to_text_model,
+                'storage_client': minio_client
+            }
+            tool_instance = tool_class(**params)
+        elif tool_name == "analyze_text_file":
+            if not tenant_id or not user_id:
+                raise ToolExecutionException(f"Tenant ID and User ID are required for {tool_name} validation")
+            long_text_to_text_model = get_llm_model(tenant_id=tenant_id)
+            params = {
+                **instantiation_params,
+                'llm_model': long_text_to_text_model,
+                'storage_client': minio_client,
+                "data_process_service_url": DATA_PROCESS_SERVICE
+            }
+            tool_instance = tool_class(**params)
         else:
             tool_instance = tool_class(**instantiation_params)
 

doc/docs/en/opensource-memorial-wall.md

@@ -16,12 +16,12 @@ Each message should include your name/handle and date.
 Keep messages respectful and in line with our Code of Conduct.
 -->
 
-::: info techbro_kevin - 2025-06-15
-first time doing open source, nexent is pretty cool! natural language to create agents, way easier than i thought
+::: tip product_guy - 2025-02-07
+no more PRDs!! just describe in plain english and devs get it instantly
 :::
 
-::: warning Dr. Chen - AI Algorithm Engineer - 2025-06-25
-Moving from LangChain to the MCP ecosystem was definitely the right choice. Nexent's MCP tool integration makes our team's workflow much more standardized, especially when handling enterprise applications. The "USB-C of AI" analogy is so apt! Now our agents can seamlessly connect to various external services - no more headaches over different API formats.
+::: info techbro_kevin - 2025-06-15
+first time doing open source, nexent is pretty cool! natural language to create agents, way easier than i thought
 :::
 
 ::: tip startup_dev - 2025-06-18
@@ -36,6 +36,10 @@ working on multi-agent stuff, the knowledge tracing feature caught my eye. built
 first contribution was fixing a typo lmao... but community is super friendly! now using nexent for personal knowledge management, feels like having a second brain
 :::
 
+::: warning Dr. Chen - AI Algorithm Engineer - 2025-06-25
+Moving from LangChain to the MCP ecosystem was definitely the right choice. Nexent's MCP tool integration makes our team's workflow much more standardized, especially when handling enterprise applications. The "USB-C of AI" analogy is so apt! Now our agents can seamlessly connect to various external services - no more headaches over different API formats.
+:::
+
 ::: warning mldev2025 - 2025-06-25
 moved from langchain to MCP, good choice. the "USB-C of AI" analogy is spot on
 :::
@@ -44,10 +48,6 @@ moved from langchain to MCP, good choice. the "USB-C of AI" analogy is spot on
 zero-code actually works! just described what i wanted and boom, got an agent. handles pdfs and everything
 :::
 
-::: tip product_guy - 2025-02-07
-no more PRDs!! just describe in plain english and devs get it instantly
-:::
-
 ::: info Tom Park - University of Toronto CS Student - 2025-08-03
 International student here! Started contributing to Nexent as part of my open source class assignment, but ended up loving the project. The documentation is so well-written that even non-native English speakers like me can easily understand and contribute. I helped translate some docs and built a study group coordination agent for our international student community. The multimodal support works great for students who prefer different communication styles!
 :::
@@ -69,5 +69,5 @@ Nexent represents the future of intelligent agent creation — simple, elegant,
 :::
 
 ::: info uu - 2024-01-15
-华为ICT智能体，感谢nexent平台支持！
+Huawei ICT Agent, thanks to the nexent platform for its support!
 :::