Merge pull request #740 from MODSetter/dev

feat(0.0.12): bumped version, added composio connectors and various fixes

Author: MODSetter

README.md

@@ -29,8 +29,7 @@ SurfSense is a highly customizable AI research agent, connected to external sour
 
 # Video 
 
-https://github.com/user-attachments/assets/42a29ea1-d4d8-4213-9c69-972b5b806d58
-
+https://github.com/user-attachments/assets/cc0c84d3-1f2f-4f7a-b519-2ecce22310b1
 
 
 ## Podcast Sample
@@ -52,15 +51,18 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
 - Interact in Natural Language and get cited answers.
 ### 📄 **Cited Answers**
 - Get Cited answers just like Perplexity.
+### 🧩 **Universal Compatibility**
+- Connect virtually any inference provider via the OpenAI spec and LiteLLM.
 ### 🔔 **Privacy & Local LLM Support**
-- Works Flawlessly with Ollama local LLMs.
+- Works Flawlessly with local LLMs like vLLM and Ollama.
 ### 🏠 **Self Hostable**
 - Open source and easy to deploy locally.
 ### 👥 **Team Collaboration with RBAC**
 - Role-Based Access Control for Search Spaces
 - Invite team members with customizable roles (Owner, Admin, Editor, Viewer)
 - Granular permissions for documents, chats, connectors, and settings
 - Share knowledge bases securely within your organization
+- Team chats update in real-time and "Chat about the chat" in comment threads
 ### 🎙️ Podcasts 
 - Blazingly fast podcast generation agent. (Creates a 3-minute podcast in under 20 seconds.)
 - Convert your chat conversations into engaging audio content
@@ -237,6 +239,8 @@ Before self-hosting installation, make sure to complete the [prerequisite setup
 
  ### **BackEnd** 
 
+-  **LiteLLM**: Universal LLM integration supporting 100+ models (OpenAI, Anthropic, Ollama, etc.)
+
 -  **FastAPI**: Modern, fast web framework for building APIs with Python
 
 -  **PostgreSQL with pgvector**: Database with vector search capabilities for similarity searches
@@ -253,8 +257,6 @@ Before self-hosting installation, make sure to complete the [prerequisite setup
 
 -  **LangChain**: Framework for developing AI-powered applications.
 
--  **LiteLLM**: Universal LLM integration supporting 100+ models (OpenAI, Anthropic, Ollama, etc.)
-
 -  **Rerankers**: Advanced result ranking for improved search relevance
 
 -  **Hybrid Search**: Combines vector similarity and full-text search for optimal results using Reciprocal Rank Fusion (RRF)

surfsense_backend/alembic/versions/79_add_composio_connector_enums.py

@@ -0,0 +1,95 @@
+"""Add Composio connector types to SearchSourceConnectorType and DocumentType enums
+
+Revision ID: 79
+Revises: 78
+
+This migration adds the Composio connector enum values to both:
+- searchsourceconnectortype (for connector type tracking)
+- documenttype (for document type tracking)
+
+Composio is a managed OAuth integration service that allows connecting
+to various third-party services (Google Drive, Gmail, Calendar, etc.)
+without requiring separate OAuth app verification.
+
+This migration adds three specific connector types:
+- COMPOSIO_GOOGLE_DRIVE_CONNECTOR
+- COMPOSIO_GMAIL_CONNECTOR
+- COMPOSIO_GOOGLE_CALENDAR_CONNECTOR
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "79"
+down_revision: str | None = "78"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+# Define the ENUM type names and the new values
+CONNECTOR_ENUM = "searchsourceconnectortype"
+CONNECTOR_NEW_VALUES = [
+    "COMPOSIO_GOOGLE_DRIVE_CONNECTOR",
+    "COMPOSIO_GMAIL_CONNECTOR",
+    "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR",
+]
+DOCUMENT_ENUM = "documenttype"
+DOCUMENT_NEW_VALUES = [
+    "COMPOSIO_GOOGLE_DRIVE_CONNECTOR",
+    "COMPOSIO_GMAIL_CONNECTOR",
+    "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR",
+]
+
+
+def upgrade() -> None:
+    """Upgrade schema - add Composio connector types to connector and document enums safely."""
+    # Add each Composio connector type to searchsourceconnectortype only if not exists
+    for value in CONNECTOR_NEW_VALUES:
+        op.execute(
+            f"""
+            DO $$
+            BEGIN
+                IF NOT EXISTS (
+                    SELECT 1 FROM pg_enum e
+                    JOIN pg_type t ON e.enumtypid = t.oid
+                    WHERE t.typname = '{CONNECTOR_ENUM}' AND e.enumlabel = '{value}'
+                ) THEN
+                    ALTER TYPE {CONNECTOR_ENUM} ADD VALUE '{value}';
+                END IF;
+            END$$;
+        """
+        )
+
+    # Add each Composio connector type to documenttype only if not exists
+    for value in DOCUMENT_NEW_VALUES:
+        op.execute(
+            f"""
+            DO $$
+            BEGIN
+                IF NOT EXISTS (
+                    SELECT 1 FROM pg_enum e
+                    JOIN pg_type t ON e.enumtypid = t.oid
+                    WHERE t.typname = '{DOCUMENT_ENUM}' AND e.enumlabel = '{value}'
+                ) THEN
+                    ALTER TYPE {DOCUMENT_ENUM} ADD VALUE '{value}';
+                END IF;
+            END$$;
+        """
+        )
+
+
+def downgrade() -> None:
+    """Downgrade schema - remove Composio connector types from connector and document enums.
+
+    Note: PostgreSQL does not support removing enum values directly.
+    To properly downgrade, you would need to:
+    1. Delete any rows using the Composio connector type values
+    2. Create new enums without the Composio connector types
+    3. Alter the columns to use the new enums
+    4. Drop the old enums
+
+    This is left as a no-op since removing enum values is complex
+    and typically not needed in practice.
+    """
+    pass

surfsense_backend/app/agents/new_chat/chat_deepagent.py

@@ -7,6 +7,7 @@
 """
 
 from collections.abc import Sequence
+from typing import Any
 
 from deepagents import create_deep_agent
 from langchain_core.tools import BaseTool
@@ -23,6 +24,90 @@
 from app.agents.new_chat.tools.registry import build_tools_async
 from app.services.connector_service import ConnectorService
 
+# =============================================================================
+# Connector Type Mapping
+# =============================================================================
+
+# Maps SearchSourceConnectorType enum values to the searchable document/connector types
+# used by the knowledge_base tool. Some connectors map to different document types.
+_CONNECTOR_TYPE_TO_SEARCHABLE: dict[str, str] = {
+    # Direct mappings (connector type == searchable type)
+    "TAVILY_API": "TAVILY_API",
+    "SEARXNG_API": "SEARXNG_API",
+    "LINKUP_API": "LINKUP_API",
+    "BAIDU_SEARCH_API": "BAIDU_SEARCH_API",
+    "SLACK_CONNECTOR": "SLACK_CONNECTOR",
+    "TEAMS_CONNECTOR": "TEAMS_CONNECTOR",
+    "NOTION_CONNECTOR": "NOTION_CONNECTOR",
+    "GITHUB_CONNECTOR": "GITHUB_CONNECTOR",
+    "LINEAR_CONNECTOR": "LINEAR_CONNECTOR",
+    "DISCORD_CONNECTOR": "DISCORD_CONNECTOR",
+    "JIRA_CONNECTOR": "JIRA_CONNECTOR",
+    "CONFLUENCE_CONNECTOR": "CONFLUENCE_CONNECTOR",
+    "CLICKUP_CONNECTOR": "CLICKUP_CONNECTOR",
+    "GOOGLE_CALENDAR_CONNECTOR": "GOOGLE_CALENDAR_CONNECTOR",
+    "GOOGLE_GMAIL_CONNECTOR": "GOOGLE_GMAIL_CONNECTOR",
+    "GOOGLE_DRIVE_CONNECTOR": "GOOGLE_DRIVE_FILE",  # Connector type differs from document type
+    "AIRTABLE_CONNECTOR": "AIRTABLE_CONNECTOR",
+    "LUMA_CONNECTOR": "LUMA_CONNECTOR",
+    "ELASTICSEARCH_CONNECTOR": "ELASTICSEARCH_CONNECTOR",
+    "WEBCRAWLER_CONNECTOR": "CRAWLED_URL",  # Maps to document type
+    "BOOKSTACK_CONNECTOR": "BOOKSTACK_CONNECTOR",
+    "CIRCLEBACK_CONNECTOR": "CIRCLEBACK",  # Connector type differs from document type
+    "OBSIDIAN_CONNECTOR": "OBSIDIAN_CONNECTOR",
+    # Composio connectors
+    "COMPOSIO_GOOGLE_DRIVE_CONNECTOR": "COMPOSIO_GOOGLE_DRIVE_CONNECTOR",
+    "COMPOSIO_GMAIL_CONNECTOR": "COMPOSIO_GMAIL_CONNECTOR",
+    "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR": "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR",
+}
+
+# Document types that don't come from SearchSourceConnector but should always be searchable
+_ALWAYS_AVAILABLE_DOC_TYPES: list[str] = [
+    "EXTENSION",  # Browser extension data
+    "FILE",  # Uploaded files
+    "NOTE",  # User notes
+    "YOUTUBE_VIDEO",  # YouTube videos
+]
+
+
+def _map_connectors_to_searchable_types(
+    connector_types: list[Any],
+) -> list[str]:
+    """
+    Map SearchSourceConnectorType enums to searchable document/connector types.
+
+    This function:
+    1. Converts connector type enums to their searchable counterparts
+    2. Includes always-available document types (EXTENSION, FILE, NOTE, YOUTUBE_VIDEO)
+    3. Deduplicates while preserving order
+
+    Args:
+        connector_types: List of SearchSourceConnectorType enum values
+
+    Returns:
+        List of searchable connector/document type strings
+    """
+    result_set: set[str] = set()
+    result_list: list[str] = []
+
+    # Add always-available document types first
+    for doc_type in _ALWAYS_AVAILABLE_DOC_TYPES:
+        if doc_type not in result_set:
+            result_set.add(doc_type)
+            result_list.append(doc_type)
+
+    # Map each connector type to its searchable equivalent
+    for ct in connector_types:
+        # Handle both enum and string types
+        ct_str = ct.value if hasattr(ct, "value") else str(ct)
+        searchable = _CONNECTOR_TYPE_TO_SEARCHABLE.get(ct_str)
+        if searchable and searchable not in result_set:
+            result_set.add(searchable)
+            result_list.append(searchable)
+
+    return result_list
+
+
 # =============================================================================
 # Deep Agent Factory
 # =============================================================================
@@ -116,13 +201,40 @@ async def create_surfsense_deep_agent(
             additional_tools=[my_custom_tool]
         )
     """
+    # Discover available connectors and document types for this search space
+    # This enables dynamic tool docstrings that inform the LLM about what's actually available
+    available_connectors: list[str] | None = None
+    available_document_types: list[str] | None = None
+
+    try:
+        # Get enabled search source connectors for this search space
+        connector_types = await connector_service.get_available_connectors(
+            search_space_id
+        )
+        if connector_types:
+            # Convert enum values to strings and also include mapped document types
+            available_connectors = _map_connectors_to_searchable_types(connector_types)
+
+        # Get document types that have at least one document indexed
+        available_document_types = await connector_service.get_available_document_types(
+            search_space_id
+        )
+    except Exception as e:
+        # Log but don't fail - fall back to all connectors if discovery fails
+        import logging
+
+        logging.warning(f"Failed to discover available connectors/document types: {e}")
+
     # Build dependencies dict for the tools registry
     dependencies = {
         "search_space_id": search_space_id,
         "db_session": db_session,
         "connector_service": connector_service,
         "firecrawl_api_key": firecrawl_api_key,
         "user_id": user_id,  # Required for memory tools
+        # Dynamic connector/document type discovery for knowledge base tool
+        "available_connectors": available_connectors,
+        "available_document_types": available_document_types,
     }
 
     # Build tools using the async registry (includes MCP tools)

surfsense_backend/app/agents/new_chat/tools/__init__.py

@@ -19,6 +19,7 @@
 # Tool factory exports (for direct use)
 from .display_image import create_display_image_tool
 from .knowledge_base import (
+    CONNECTOR_DESCRIPTIONS,
     create_search_knowledge_base_tool,
     format_documents_for_context,
     search_knowledge_base_async,
@@ -40,6 +41,8 @@
 __all__ = [
     # Registry
     "BUILTIN_TOOLS",
+    # Knowledge base utilities
+    "CONNECTOR_DESCRIPTIONS",
     "ToolDefinition",
     "build_tools",
     # Tool factories
@@ -51,7 +54,6 @@
     "create_scrape_webpage_tool",
     "create_search_knowledge_base_tool",
     "create_search_surfsense_docs_tool",
-    # Knowledge base utilities
     "format_documents_for_context",
     "get_all_tool_names",
     "get_default_enabled_tools",