feat: refactor agent tools management and add UI integration

- Added endpoint to list agent tools with metadata, excluding hidden tools.
- Updated NewChatRequest and RegenerateRequest schemas to include disabled tools.
- Integrated disabled tools management in the NewChatPage and Composer components.
- Improved tool instructions and visibility in the system prompt.
- Refactored tool registration to support hidden tools and default enabled states.
- Enhanced document chunk creation to handle strict zip behavior.
- Cleaned up imports and formatting across various files for consistency.

Author: MODSetter

surfsense_backend/app/agents/new_chat/chat_deepagent.py

@@ -303,21 +303,27 @@ async def create_surfsense_deep_agent(
         len(tools),
     )
 
-    # Build system prompt based on agent_config
+    # Build system prompt based on agent_config, scoped to the tools actually enabled
     _t0 = time.perf_counter()
     _sandbox_enabled = sandbox_backend is not None
+    _enabled_tool_names = {t.name for t in tools}
+    _user_disabled_tool_names = set(disabled_tools) if disabled_tools else set()
     if agent_config is not None:
         system_prompt = build_configurable_system_prompt(
             custom_system_instructions=agent_config.system_instructions,
             use_default_system_instructions=agent_config.use_default_system_instructions,
             citations_enabled=agent_config.citations_enabled,
             thread_visibility=thread_visibility,
             sandbox_enabled=_sandbox_enabled,
+            enabled_tool_names=_enabled_tool_names,
+            disabled_tool_names=_user_disabled_tool_names,
         )
     else:
         system_prompt = build_surfsense_system_prompt(
             thread_visibility=thread_visibility,
             sandbox_enabled=_sandbox_enabled,
+            enabled_tool_names=_enabled_tool_names,
+            disabled_tool_names=_user_disabled_tool_names,
         )
     _perf_log.info(
         "[create_agent] System prompt built in %.3fs", time.perf_counter() - _t0

surfsense_backend/app/agents/new_chat/system_prompt.py

@@ -97,6 +97,7 @@ class ToolDefinition:
     factory: Callable[[dict[str, Any]], BaseTool]
     requires: list[str] = field(default_factory=list)
     enabled_by_default: bool = True
+    hidden: bool = False
 
 
 # =============================================================================
@@ -139,7 +140,7 @@ class ToolDefinition:
     # need to call search_knowledge_base separately before generating.
     ToolDefinition(
         name="generate_report",
-        description="Generate a structured Markdown report from provided content",
+        description="Generate a structured report from provided content and export it",
         factory=lambda deps: create_generate_report_tool(
             search_space_id=deps["search_space_id"],
             thread_id=deps["thread_id"],
@@ -234,7 +235,7 @@ class ToolDefinition:
         requires=["user_id", "search_space_id", "db_session", "thread_visibility"],
     ),
     # =========================================================================
-    # LINEAR TOOLS - create, update, delete issues
+    # LINEAR TOOLS - create, update, delete issues (WIP - hidden from UI)
     # =========================================================================
     ToolDefinition(
         name="create_linear_issue",
@@ -245,6 +246,8 @@ class ToolDefinition:
             user_id=deps["user_id"],
         ),
         requires=["db_session", "search_space_id", "user_id"],
+        enabled_by_default=False,
+        hidden=True,
     ),
     ToolDefinition(
         name="update_linear_issue",
@@ -255,6 +258,8 @@ class ToolDefinition:
             user_id=deps["user_id"],
         ),
         requires=["db_session", "search_space_id", "user_id"],
+        enabled_by_default=False,
+        hidden=True,
     ),
     ToolDefinition(
         name="delete_linear_issue",
@@ -265,9 +270,11 @@ class ToolDefinition:
             user_id=deps["user_id"],
         ),
         requires=["db_session", "search_space_id", "user_id"],
+        enabled_by_default=False,
+        hidden=True,
     ),
     # =========================================================================
-    # NOTION TOOLS - create, update, delete pages
+    # NOTION TOOLS - create, update, delete pages (WIP - hidden from UI)
     # =========================================================================
     ToolDefinition(
         name="create_notion_page",
@@ -278,6 +285,8 @@ class ToolDefinition:
             user_id=deps["user_id"],
         ),
         requires=["db_session", "search_space_id", "user_id"],
+        enabled_by_default=False,
+        hidden=True,
     ),
     ToolDefinition(
         name="update_notion_page",
@@ -288,6 +297,8 @@ class ToolDefinition:
             user_id=deps["user_id"],
         ),
         requires=["db_session", "search_space_id", "user_id"],
+        enabled_by_default=False,
+        hidden=True,
     ),
     ToolDefinition(
         name="delete_notion_page",
@@ -298,9 +309,11 @@ class ToolDefinition:
             user_id=deps["user_id"],
         ),
         requires=["db_session", "search_space_id", "user_id"],
+        enabled_by_default=False,
+        hidden=True,
     ),
     # =========================================================================
-    # GOOGLE DRIVE TOOLS - create files, delete files
+    # GOOGLE DRIVE TOOLS - create files, delete files (WIP - hidden from UI)
     # =========================================================================
     ToolDefinition(
         name="create_google_drive_file",
@@ -311,6 +324,8 @@ class ToolDefinition:
             user_id=deps["user_id"],
         ),
         requires=["db_session", "search_space_id", "user_id"],
+        enabled_by_default=False,
+        hidden=True,
     ),
     ToolDefinition(
         name="delete_google_drive_file",
@@ -321,6 +336,8 @@ class ToolDefinition:
             user_id=deps["user_id"],
         ),
         requires=["db_session", "search_space_id", "user_id"],
+        enabled_by_default=False,
+        hidden=True,
     ),
 ]
 
@@ -344,7 +361,7 @@ def get_all_tool_names() -> list[str]:
 
 
 def get_default_enabled_tools() -> list[str]:
-    """Get names of tools that are enabled by default."""
+    """Get names of tools that are enabled by default (excludes hidden tools)."""
     return [tool_def.name for tool_def in BUILTIN_TOOLS if tool_def.enabled_by_default]
 
 
@@ -393,10 +410,10 @@ def build_tools(
     if disabled_tools:
         tool_names_to_use -= set(disabled_tools)
 
-    # Build the tools
+    # Build the tools (skip hidden/WIP tools unconditionally)
     tools: list[BaseTool] = []
     for tool_def in BUILTIN_TOOLS:
-        if tool_def.name not in tool_names_to_use:
+        if tool_def.hidden or tool_def.name not in tool_names_to_use:
             continue
 
         # Check that all required dependencies are provided

surfsense_backend/app/agents/new_chat/tools/registry.py

@@ -12,7 +12,6 @@
 
 from bs4 import BeautifulSoup
 from markdownify import markdownify as md
-
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.future import select
 from sqlalchemy.orm import selectinload
@@ -139,7 +138,9 @@ def _html_to_markdown(html: str) -> str:
         soup = BeautifulSoup(html, "html.parser")
         for tag in soup.find_all(["style", "script", "img"]):
             tag.decompose()
-        for tag in soup.find_all(["table", "thead", "tbody", "tfoot", "tr", "td", "th"]):
+        for tag in soup.find_all(
+            ["table", "thead", "tbody", "tfoot", "tr", "td", "th"]
+        ):
             tag.unwrap()
         return md(str(soup)).strip()
 

surfsense_backend/app/connectors/composio_gmail_connector.py

@@ -10,11 +10,10 @@
 from typing import Any
 
 from bs4 import BeautifulSoup
-from markdownify import markdownify as md
-
 from google.auth.transport.requests import Request
 from google.oauth2.credentials import Credentials
 from googleapiclient.discovery import build
+from markdownify import markdownify as md
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.future import select
 from sqlalchemy.orm.attributes import flag_modified
@@ -356,7 +355,9 @@ def _html_to_markdown(html: str) -> str:
         soup = BeautifulSoup(html, "html.parser")
         for tag in soup.find_all(["style", "script", "img"]):
             tag.decompose()
-        for tag in soup.find_all(["table", "thead", "tbody", "tfoot", "tr", "td", "th"]):
+        for tag in soup.find_all(
+            ["table", "thead", "tbody", "tfoot", "tr", "td", "th"]
+        ):
             tag.unwrap()
         return md(str(soup)).strip()
 

surfsense_backend/app/connectors/google_gmail_connector.py

@@ -960,7 +960,10 @@ class Chunk(BaseModel, TimestampMixin):
     embedding = Column(Vector(config.embedding_model_instance.dimension))
 
     document_id = Column(
-        Integer, ForeignKey("documents.id", ondelete="CASCADE"), nullable=False, index=True
+        Integer,
+        ForeignKey("documents.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
     )
     document = relationship("Document", back_populates="chunks")
 

surfsense_backend/app/db.py

@@ -211,7 +211,7 @@ async def index(
 
             chunks = [
                 Chunk(content=text, embedding=emb)
-                for text, emb in zip(chunk_texts, chunk_embeddings)
+                for text, emb in zip(chunk_texts, chunk_embeddings, strict=False)
             ]
             perf.info(
                 "[indexing] chunk+embed doc=%d chunks=%d in %.3fs",

surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py

@@ -35,6 +35,7 @@
     shielded_async_session,
 )
 from app.schemas.new_chat import (
+    AgentToolInfo,
     NewChatMessageRead,
     NewChatRequest,
     NewChatThreadCreate,
@@ -1024,6 +1025,32 @@ async def list_messages(
         ) from None
 
 
+# =============================================================================
+# Agent Tools Endpoint
+# =============================================================================
+
+
+@router.get("/agent/tools", response_model=list[AgentToolInfo])
+async def list_agent_tools(
+    _user: User = Depends(current_active_user),
+):
+    """Return the list of built-in agent tools with their metadata.
+
+    Hidden (WIP) tools are excluded from the response.
+    """
+    from app.agents.new_chat.tools.registry import BUILTIN_TOOLS
+
+    return [
+        AgentToolInfo(
+            name=t.name,
+            description=t.description,
+            enabled_by_default=t.enabled_by_default,
+        )
+        for t in BUILTIN_TOOLS
+        if not t.hidden
+    ]
+
+
 # =============================================================================
 # Chat Streaming Endpoint
 # =============================================================================
@@ -1108,6 +1135,7 @@ async def handle_new_chat(
                 needs_history_bootstrap=thread.needs_history_bootstrap,
                 thread_visibility=thread.visibility,
                 current_user_display_name=user.display_name or "A team member",
+                disabled_tools=request.disabled_tools,
             ),
             media_type="text/event-stream",
             headers={
@@ -1344,6 +1372,7 @@ async def stream_with_cleanup():
                     needs_history_bootstrap=thread.needs_history_bootstrap,
                     thread_visibility=thread.visibility,
                     current_user_display_name=user.display_name or "A team member",
+                    disabled_tools=request.disabled_tools,
                 ):
                     yield chunk
                 streaming_completed = True

surfsense_backend/app/routes/new_chat_routes.py

@@ -18,6 +18,7 @@
 Non-OAuth connectors (BookStack, GitHub, etc.) are limited to one per search space.
 """
 
+import asyncio
 import logging
 from contextlib import suppress
 from datetime import UTC, datetime, timedelta
@@ -52,8 +53,6 @@
     SearchSourceConnectorRead,
     SearchSourceConnectorUpdate,
 )
-import asyncio
-
 from app.services.composio_service import ComposioService, get_composio_service
 from app.services.notification_service import NotificationService
 from app.tasks.connector_indexers import (
@@ -3124,7 +3123,9 @@ async def get_drive_picker_token(
                 detail="Composio connected account not found. Please reconnect.",
             )
         service = get_composio_service()
-        access_token = await asyncio.to_thread(service.get_access_token, composio_account_id)
+        access_token = await asyncio.to_thread(
+            service.get_access_token, composio_account_id
+        )
         return {
             "access_token": access_token,
             "client_id": config.GOOGLE_OAUTH_CLIENT_ID,

surfsense_backend/app/routes/search_source_connectors_routes.py

@@ -172,6 +172,9 @@ class NewChatRequest(BaseModel):
     mentioned_surfsense_doc_ids: list[int] | None = (
         None  # Optional SurfSense documentation IDs mentioned with @ in the chat
     )
+    disabled_tools: list[str] | None = (
+        None  # Optional list of tool names the user has disabled from the UI
+    )
 
 
 class RegenerateRequest(BaseModel):
@@ -191,6 +194,20 @@ class RegenerateRequest(BaseModel):
     )
     mentioned_document_ids: list[int] | None = None
     mentioned_surfsense_doc_ids: list[int] | None = None
+    disabled_tools: list[str] | None = None
+
+
+# =============================================================================
+# Agent Tools Schemas
+# =============================================================================
+
+
+class AgentToolInfo(BaseModel):
+    """Schema for a single agent tool's public metadata."""
+
+    name: str
+    description: str
+    enabled_by_default: bool
 
 
 class ResumeDecision(BaseModel):