diff --git a/.github/workflows/main_webcatgpt.yml b/.github/workflows/main_webcatgpt.yml.disabled similarity index 100% rename from .github/workflows/main_webcatgpt.yml rename to .github/workflows/main_webcatgpt.yml.disabled diff --git a/README.md b/README.md index 725c510..ba1db2e 100644 --- a/README.md +++ b/README.md @@ -105,18 +105,27 @@ make mcp # Start MCP server | Variable | Default | Description | |----------|---------|-------------| | `SERPER_API_KEY` | *(none)* | Serper API key for premium search (optional, falls back to DuckDuckGo if not set) | +| `PERPLEXITY_API_KEY` | *(none)* | Perplexity API key for deep research tool (optional, get at https://www.perplexity.ai/settings/api) | | `WEBCAT_API_KEY` | *(none)* | Bearer token for authentication (optional, if set all requests must include `Authorization: Bearer `) | | `PORT` | `8000` | Server port | | `LOG_LEVEL` | `INFO` | Logging level (DEBUG, INFO, WARNING, ERROR) | | `LOG_DIR` | `/tmp` | Log file directory | +| `MAX_CONTENT_LENGTH` | `1000000` | Maximum characters to return per scraped article | -### Get a Serper API Key +### Get API Keys +**Serper API (for web search):** 1. Visit [serper.dev](https://serper.dev) 2. Sign up for free tier (2,500 searches/month) 3. Copy your API key 4. Add to `.env` file: `SERPER_API_KEY=your_key` +**Perplexity API (for deep research):** +1. Visit [perplexity.ai/settings/api](https://www.perplexity.ai/settings/api) +2. Sign up and get your API key +3. Copy your API key +4. Add to `.env` file: `PERPLEXITY_API_KEY=your_key` + ### Enable Authentication (Optional) To require bearer token authentication for all MCP tool calls: diff --git a/docker/.env.example b/docker/.env.example index c412dd5..d87937c 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -2,6 +2,11 @@ # If not set, DuckDuckGo fallback will be used SERPER_API_KEY= +# Perplexity API key for deep research (optional) +# Required for the deep_research MCP tool +# Get your API key at: https://www.perplexity.ai/settings/api +PERPLEXITY_API_KEY= + # WebCat API key for bearer token authentication (optional) # If set, all requests must include: Authorization: Bearer # If not set, no authentication is required @@ -12,6 +17,9 @@ PORT=8000 LOG_LEVEL=INFO LOG_DIR=/tmp +# Content limits +MAX_CONTENT_LENGTH=1000000 + # Rate limiting RATE_LIMIT_WINDOW=60 RATE_LIMIT_MAX_REQUESTS=10 diff --git a/docker/README.md b/docker/README.md index 91e7a4b..e1d9f63 100644 --- a/docker/README.md +++ b/docker/README.md @@ -50,9 +50,11 @@ docker-compose up ### Environment Variables - `SERPER_API_KEY`: **Optional** - Serper API key for premium search (falls back to DuckDuckGo if not set) +- `PERPLEXITY_API_KEY`: **Optional** - Perplexity API key for deep research tool (get key at https://www.perplexity.ai/settings/api) - `PORT`: Port to run the server on (default: 8000) - `LOG_LEVEL`: Logging level (default: INFO) - `LOG_DIR`: Directory for log files (default: /tmp) +- `MAX_CONTENT_LENGTH`: Maximum characters to return per scraped article (default: 1000000) ### Simplified Setup @@ -78,7 +80,14 @@ The server runs on **FastMCP** and exposes MCP protocol endpoints: - Falls back to DuckDuckGo automatically - Returns full webpage content in markdown format -2. **`health_check`** - Check server health status +2. **`deep_research`** - Comprehensive deep research (NEW!) + - Uses Perplexity AI's sonar-deep-research model + - Performs dozens of searches and reads hundreds of sources + - Synthesizes findings into comprehensive reports + - Takes 2-4 minutes (what would take humans many hours) + - Configurable research effort: low, medium, high + +3. **`health_check`** - Check server health status ## Testing the Server diff --git a/docker/clients/perplexity_client.py b/docker/clients/perplexity_client.py new file mode 100644 index 0000000..27aa51d --- /dev/null +++ b/docker/clients/perplexity_client.py @@ -0,0 +1,93 @@ +# Copyright (c) 2024 Travis Frisinger +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +"""Perplexity API client - deep research search using Perplexity's sonar models.""" + +import logging +from typing import List, Literal + +from perplexity import Perplexity + +logger = logging.getLogger(__name__) + + +def fetch_perplexity_deep_research( + query: str, + api_key: str, + max_results: int = 5, + research_effort: Literal["low", "medium", "high"] = "high", +) -> tuple[str, List[str]]: + """ + Fetch deep research results from Perplexity AI using official SDK. + + Uses Perplexity's sonar-deep-research model which performs dozens of searches, + reads hundreds of sources, and reasons through material to deliver comprehensive + research reports. This is ideal for in-depth analysis and multi-source synthesis. + + Args: + query: The research query/question + api_key: Perplexity API key + max_results: Maximum number of results to return (default: 5) + research_effort: Computational effort level - "low" (fast), "medium" (balanced), + or "high" (deepest research). Only applies to sonar-deep-research. + + Returns: + Tuple of (research_report: str, citations: List[str]) + - research_report: Full synthesized research content in markdown + - citations: List of citation URLs used in the research + """ + try: + logger.info( + f"Fetching Perplexity deep research (effort: {research_effort}): {query}" + ) + + # Initialize Perplexity client with 10-minute timeout for deep research + client = Perplexity(api_key=api_key, timeout=600.0) + + # Create chat completion with deep research + # Note: Official SDK may not support all parameters, using minimal set + response = client.chat.completions.create( + model="sonar-deep-research", + messages=[ + { + "role": "system", + "content": ( + "You are a comprehensive research assistant. Provide detailed, " + "well-researched answers with clear structure and citations. " + f"Focus on returning {max_results} most relevant sources." + ), + }, + {"role": "user", "content": query}, + ], + ) + + # Extract research content + research_report = "" + citation_urls: List[str] = [] + + if response.choices and len(response.choices) > 0: + research_report = response.choices[0].message.content or "" + + # Extract citations from response + if hasattr(response, "citations") and response.citations: + citation_urls = [ + citation if isinstance(citation, str) else citation.get("url", "") + for citation in response.citations + if citation + ][:max_results] + + token_count = ( + response.usage.total_tokens if hasattr(response, "usage") else "N/A" + ) + logger.info( + f"Perplexity deep research completed: {len(research_report)} chars, " + f"{len(citation_urls)} citations, {token_count} tokens" + ) + + return research_report, citation_urls + + except Exception as e: + logger.exception(f"Error fetching Perplexity deep research: {str(e)}") + return "", [] diff --git a/docker/constants.py b/docker/constants.py index 54a30a2..73f10b0 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -5,6 +5,8 @@ """Constants for WebCat application.""" +import os + # Application version VERSION = "2.3.1" @@ -22,7 +24,10 @@ ] # Content limits -MAX_CONTENT_LENGTH = 8000 +try: + MAX_CONTENT_LENGTH = int(os.environ.get("MAX_CONTENT_LENGTH", "1000000")) +except ValueError: + MAX_CONTENT_LENGTH = 1000000 DEFAULT_SEARCH_RESULTS = 5 # Timeout settings diff --git a/docker/mcp_server.py b/docker/mcp_server.py index 9e6cdca..9823104 100644 --- a/docker/mcp_server.py +++ b/docker/mcp_server.py @@ -53,14 +53,19 @@ async def search_tool() -> dict: logger = setup_logging("webcat.log") # Import tools AFTER loading .env so they can access environment variables +from tools.deep_research_tool import deep_research_tool # noqa: E402 from tools.health_check_tool import health_check_tool # noqa: E402 from tools.search_tool import search_tool # noqa: E402 # Log configuration status SERPER_API_KEY = os.environ.get("SERPER_API_KEY", "") +PERPLEXITY_API_KEY = os.environ.get("PERPLEXITY_API_KEY", "") logging.info( f"SERPER API key: {'Set' if SERPER_API_KEY else 'Not set (using DuckDuckGo fallback)'}" ) +logging.info( + f"PERPLEXITY API key: {'Set' if PERPLEXITY_API_KEY else 'Not set (deep_research tool unavailable)'}" +) # Create FastMCP instance @@ -72,6 +77,16 @@ async def search_tool() -> dict: description="Search the web for information using Serper API or DuckDuckGo fallback", )(search_tool) +mcp_server.tool( + name="deep_research", + description=( + "Perform comprehensive deep research on a topic using Perplexity AI. " + "This tool performs dozens of searches, reads hundreds of sources, and " + "synthesizes findings into a comprehensive report. Ideal for in-depth " + "analysis and multi-source research. Takes 2-4 minutes to complete." + ), +)(deep_research_tool) + mcp_server.tool(name="health_check", description="Check the health of the server")( health_check_tool ) diff --git a/docker/test_deep_research_mcp.py b/docker/test_deep_research_mcp.py new file mode 100755 index 0000000..225846d --- /dev/null +++ b/docker/test_deep_research_mcp.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024 Travis Frisinger +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +"""Test deep_research tool via MCP server.""" + +import json + +import requests + +MCP_SERVER_URL = "http://localhost:8000/mcp" + +# Create session with required headers +session = requests.Session() +session.headers.update({"Accept": "application/json, text/event-stream"}) + +# Step 1: Initialize +print("šŸ”§ Initializing MCP session...") +init_payload = { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "clientInfo": {"name": "test-client", "version": "1.0.0"}, + }, +} + +init_resp = session.post(MCP_SERVER_URL, json=init_payload) +session_id = init_resp.headers.get("mcp-session-id") +print(f"āœ… Session ID: {session_id}\n") + +# Step 2: Send initialized notification +print("šŸ”” Sending initialized notification...") +initialized_payload = {"jsonrpc": "2.0", "method": "notifications/initialized"} +session.post( + MCP_SERVER_URL, json=initialized_payload, headers={"mcp-session-id": session_id} +) + +# Step 3: Call deep_research tool +print("\nšŸ” Calling deep_research tool...") +print("Query: What is Python programming language?") +print("Effort: low") +print("\nThis will take ~1-2 minutes...\n") + +tool_payload = { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": { + "name": "deep_research", + "arguments": { + "query": "What is Python programming language?", + "research_effort": "low", + "max_results": 5, + }, + }, +} + +tool_resp = session.post( + MCP_SERVER_URL, + json=tool_payload, + headers={"mcp-session-id": session_id}, + timeout=300, +) + +print(f"Status: {tool_resp.status_code}\n") + +# Parse SSE response +if tool_resp.status_code == 200: + lines = tool_resp.text.strip().split("\n") + for line in lines: + if line.startswith("data: "): + data_json = line[6:] # Remove "data: " prefix + result = json.loads(data_json) + + if "result" in result and "content" in result["result"]: + content = result["result"]["content"] + if isinstance(content, list) and len(content) > 0: + text_content = content[0].get("text", "") + # Parse the JSON string inside text + research_result = json.loads(text_content) + + print("=" * 80) + print("DEEP RESEARCH RESULT:") + print("=" * 80) + print(f"Query: {research_result.get('query')}") + print(f"Source: {research_result.get('search_source')}") + print(f"Results: {len(research_result.get('results', []))}") + + if research_result.get("results"): + print("\n" + "=" * 80) + print("RESEARCH REPORT:") + print("=" * 80) + print(research_result["results"][0].get("content", "")) + + break +else: + print(f"āŒ Error: {tool_resp.text}") diff --git a/docker/test_perplexity.py b/docker/test_perplexity.py new file mode 100755 index 0000000..46d2381 --- /dev/null +++ b/docker/test_perplexity.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024 Travis Frisinger +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +"""Test script for Perplexity deep research functionality.""" + +import asyncio +import json +import os +import sys + +# Add docker directory to path +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +# Load environment variables before importing tools +from dotenv import load_dotenv # noqa: E402 + +load_dotenv() + +from tools.deep_research_tool import deep_research_tool # noqa: E402 + +# Check if API key is set +PERPLEXITY_API_KEY = os.environ.get("PERPLEXITY_API_KEY", "") + +if not PERPLEXITY_API_KEY: + print("āŒ PERPLEXITY_API_KEY not set in .env file") + print("Please add your API key to docker/.env:") + print("PERPLEXITY_API_KEY=your_api_key_here") + print("\nGet your API key at: https://www.perplexity.ai/settings/api") + sys.exit(1) + +print(f"āœ… PERPLEXITY_API_KEY found: {PERPLEXITY_API_KEY[:8]}...") + + +async def test_deep_research(): + """Test the deep_research tool with a sample query.""" + print("\n" + "=" * 80) + print("Testing Perplexity Deep Research Tool") + print("=" * 80 + "\n") + + query = "What are the key differences between GPT-4 and Claude 3.5 Sonnet?" + print(f"Query: {query}") + print("Research Effort: low (for faster testing)") + print("\nThis will take ~1 minute...\n") + + try: + # Call the deep research tool + result = await deep_research_tool( + query=query, + research_effort="low", # Use low for faster testing + max_results=5, + ) + + print("āœ… Deep research completed!\n") + print("=" * 80) + print("RESULTS:") + print("=" * 80 + "\n") + + # Pretty print the result + print(json.dumps(result, indent=2)) + + # Extract and display key information + if "results" in result and result["results"]: + first_result = result["results"][0] + print("\n" + "=" * 80) + print("FORMATTED RESEARCH REPORT:") + print("=" * 80 + "\n") + print(first_result.get("content", "No content")) + + except Exception as e: + print(f"āŒ Error during deep research: {e}") + import traceback + + traceback.print_exc() + + +if __name__ == "__main__": + asyncio.run(test_deep_research()) diff --git a/docker/test_perplexity_debug.py b/docker/test_perplexity_debug.py new file mode 100755 index 0000000..6176efb --- /dev/null +++ b/docker/test_perplexity_debug.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024 Travis Frisinger +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +"""Debug version - shows exactly what Perplexity returns.""" + +import json +import os +import sys + +import requests +from dotenv import load_dotenv + +load_dotenv() + +PERPLEXITY_API_KEY = os.environ.get("PERPLEXITY_API_KEY", "") + +if not PERPLEXITY_API_KEY: + print("āŒ PERPLEXITY_API_KEY not set") + sys.exit(1) + +print(f"āœ… API Key: {PERPLEXITY_API_KEY[:8]}...\n") + +url = "https://api.perplexity.ai/chat/completions" +headers = { + "Authorization": f"Bearer {PERPLEXITY_API_KEY}", + "Content-Type": "application/json", +} + +payload = { + "model": "sonar-deep-research", + "messages": [{"role": "user", "content": "What is Python programming language?"}], + "reasoning_effort": "low", + "return_citations": True, +} + +print("Making API call (this will cost ~$1)...") +print(f"Query: {payload['messages'][0]['content']}") +print(f"Effort: {payload['reasoning_effort']}\n") + +response = requests.post(url, headers=headers, json=payload, timeout=300) + +print(f"Status: {response.status_code}") +print(f"Response keys: {list(response.json().keys())}\n") + +data = response.json() + +# Show full response structure +print("=" * 80) +print("FULL RESPONSE:") +print("=" * 80) +print(json.dumps(data, indent=2)) diff --git a/docker/test_perplexity_minimal.py b/docker/test_perplexity_minimal.py new file mode 100755 index 0000000..73bc267 --- /dev/null +++ b/docker/test_perplexity_minimal.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024 Travis Frisinger +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +"""Minimal test to verify Perplexity API is being called.""" + +import json +import os +import sys + +import requests +from dotenv import load_dotenv + +# Load environment variables +load_dotenv() + +PERPLEXITY_API_KEY = os.environ.get("PERPLEXITY_API_KEY", "") + +if not PERPLEXITY_API_KEY: + print("āŒ PERPLEXITY_API_KEY not set") + sys.exit(1) + +print(f"āœ… API Key found: {PERPLEXITY_API_KEY[:8]}...") + +# Minimal API call +url = "https://api.perplexity.ai/chat/completions" +headers = { + "Authorization": f"Bearer {PERPLEXITY_API_KEY}", + "Content-Type": "application/json", +} + +# Start with absolute minimal payload +payload = { + "model": "sonar-deep-research", + "messages": [{"role": "user", "content": "What is Python?"}], +} + +print("\nšŸ” Testing minimal Perplexity API call...") +print(f"URL: {url}") +print(f"Payload: {json.dumps(payload, indent=2)}\n") + +try: + print("šŸ“” Making request...") + response = requests.post(url, headers=headers, json=payload, timeout=300) + print(f"āœ… Response status: {response.status_code}") + + if response.status_code == 200: + data = response.json() + print(f"āœ… Success! Got response with {len(json.dumps(data))} chars") + print(f"\nResponse structure: {list(data.keys())}") + if "choices" in data: + print(f"Choices: {len(data['choices'])}") + if data["choices"]: + content = data["choices"][0].get("message", {}).get("content", "") + print(f"Content length: {len(content)} chars") + print(f"\nFirst 500 chars:\n{content[:500]}") + else: + print(f"āŒ Error: {response.status_code}") + print(f"Response: {response.text}") + +except Exception as e: + print(f"āŒ Exception: {e}") + import traceback + + traceback.print_exc() diff --git a/docker/tests/factories/mock_perplexity_response.py b/docker/tests/factories/mock_perplexity_response.py new file mode 100644 index 0000000..a001a76 --- /dev/null +++ b/docker/tests/factories/mock_perplexity_response.py @@ -0,0 +1,87 @@ +# Copyright (c) 2024 Travis Frisinger +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +"""Typed mock for Perplexity API responses.""" + +from typing import List, Optional + + +class MockPerplexityMessage: + """Typed mock for Perplexity message object.""" + + def __init__(self, content: str): + """Initialize message with content. + + Args: + content: Message content (research report) + """ + self.content = content + + +class MockPerplexityChoice: + """Typed mock for Perplexity choice object.""" + + def __init__(self, content: str): + """Initialize choice with message content. + + Args: + content: Message content + """ + self.message = MockPerplexityMessage(content) + + +class MockPerplexityUsage: + """Typed mock for Perplexity usage stats.""" + + def __init__(self, total_tokens: int = 1000): + """Initialize usage stats. + + Args: + total_tokens: Total token count + """ + self.total_tokens = total_tokens + + +class MockPerplexityResponse: + """Typed mock for Perplexity API completion response.""" + + def __init__( + self, + content: str = "", + citations: Optional[List[str]] = None, + total_tokens: int = 1000, + ): + """Initialize mock Perplexity response. + + Args: + content: Research report content + citations: List of citation URLs + total_tokens: Total tokens used + """ + self.choices = [MockPerplexityChoice(content)] if content else [] + self.citations = citations or [] + self.usage = MockPerplexityUsage(total_tokens) + + +class MockPerplexityClient: + """Typed mock for Perplexity client.""" + + def __init__(self, response: MockPerplexityResponse): + """Initialize mock client with predefined response. + + Args: + response: Response to return from API calls + """ + self._response = response + self.chat = self + self.completions = self + + def create(self, **kwargs) -> MockPerplexityResponse: + """Mock create method that returns configured response. + + Returns: + Configured MockPerplexityResponse + """ + return self._response diff --git a/docker/tests/factories/perplexity_response_factory.py b/docker/tests/factories/perplexity_response_factory.py new file mode 100644 index 0000000..6e34e84 --- /dev/null +++ b/docker/tests/factories/perplexity_response_factory.py @@ -0,0 +1,91 @@ +# Copyright (c) 2024 Travis Frisinger +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +"""Factory for creating Perplexity response test doubles.""" + +from typing import List + +from tests.factories.mock_perplexity_response import ( + MockPerplexityClient, + MockPerplexityResponse, +) + + +class PerplexityResponseFactory: + """Factory for creating pre-configured Perplexity response mocks.""" + + @staticmethod + def successful_research( + content: str = "Python is a high-level programming language.", + citations: List[str] = None, + ) -> MockPerplexityResponse: + """Create successful deep research response. + + Args: + content: Research report content + citations: Citation URLs + + Returns: + MockPerplexityResponse with research content + """ + if citations is None: + citations = ["https://python.org", "https://docs.python.org"] + return MockPerplexityResponse(content=content, citations=citations) + + @staticmethod + def empty_response() -> MockPerplexityResponse: + """Create empty response with no content. + + Returns: + MockPerplexityResponse with empty choices + """ + return MockPerplexityResponse(content="") + + @staticmethod + def with_many_citations(num_citations: int = 10) -> MockPerplexityResponse: + """Create response with many citations for testing max_results. + + Args: + num_citations: Number of citations to include + + Returns: + MockPerplexityResponse with specified number of citations + """ + citations = [f"https://url{i}.com" for i in range(1, num_citations + 1)] + return MockPerplexityResponse( + content="Research with many sources", citations=citations + ) + + @staticmethod + def without_citations() -> MockPerplexityResponse: + """Create response without any citations. + + Returns: + MockPerplexityResponse with no citations + """ + return MockPerplexityResponse( + content="Research without citations", citations=[] + ) + + @staticmethod + def api_error() -> Exception: + """Create API error exception. + + Returns: + Exception to be used with side_effect + """ + return Exception("API Error") + + @staticmethod + def client_with_response(response: MockPerplexityResponse) -> MockPerplexityClient: + """Create mock client that returns specific response. + + Args: + response: Response to return from API calls + + Returns: + MockPerplexityClient configured with response + """ + return MockPerplexityClient(response) diff --git a/docker/tests/unit/clients/test_perplexity_client.py b/docker/tests/unit/clients/test_perplexity_client.py new file mode 100644 index 0000000..9841988 --- /dev/null +++ b/docker/tests/unit/clients/test_perplexity_client.py @@ -0,0 +1,126 @@ +# Copyright (c) 2024 Travis Frisinger +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +"""Unit tests for Perplexity API client.""" + +from unittest.mock import patch + +import pytest + +from clients.perplexity_client import fetch_perplexity_deep_research +from tests.factories.perplexity_response_factory import PerplexityResponseFactory + + +@pytest.mark.unit +@patch("clients.perplexity_client.Perplexity") +def test_fetch_perplexity_deep_research_success(mock_perplexity_class): + """Test successful deep research fetch.""" + # Setup using factory + response = PerplexityResponseFactory.successful_research() + mock_client = PerplexityResponseFactory.client_with_response(response) + mock_perplexity_class.return_value = mock_client + + # Call function + report, citations = fetch_perplexity_deep_research( + query="What is Python?", + api_key="test_key", # pragma: allowlist secret + max_results=3, + research_effort="low", + ) + + # Assertions + assert report == "Python is a high-level programming language." + assert len(citations) == 2 + assert citations[0] == "https://python.org" + + # Verify client was initialized with correct timeout + mock_perplexity_class.assert_called_once_with( + api_key="test_key", timeout=600.0 # pragma: allowlist secret + ) + + +@pytest.mark.unit +@patch("clients.perplexity_client.Perplexity") +def test_fetch_perplexity_deep_research_empty_response(mock_perplexity_class): + """Test handling of empty response.""" + # Setup using factory + response = PerplexityResponseFactory.empty_response() + mock_client = PerplexityResponseFactory.client_with_response(response) + mock_perplexity_class.return_value = mock_client + + # Call function + report, citations = fetch_perplexity_deep_research( + query="What is Python?", + api_key="test_key", # pragma: allowlist secret + ) + + # Assertions + assert report == "" + assert citations == [] + + +@pytest.mark.unit +@patch("clients.perplexity_client.Perplexity") +def test_fetch_perplexity_deep_research_api_error(mock_perplexity_class): + """Test handling of API errors.""" + # Setup using factory - client raises exception + mock_client = PerplexityResponseFactory.client_with_response( + PerplexityResponseFactory.empty_response() + ) + mock_client.chat.completions.create = lambda **kwargs: (_ for _ in ()).throw( + PerplexityResponseFactory.api_error() + ) + mock_perplexity_class.return_value = mock_client + + # Call function + report, citations = fetch_perplexity_deep_research( + query="What is Python?", + api_key="test_key", # pragma: allowlist secret + ) + + # Assertions - should return empty values on error + assert report == "" + assert citations == [] + + +@pytest.mark.unit +@patch("clients.perplexity_client.Perplexity") +def test_fetch_perplexity_deep_research_max_results_limit(mock_perplexity_class): + """Test that max_results limits citations returned.""" + # Setup using factory with many citations + response = PerplexityResponseFactory.with_many_citations(num_citations=5) + mock_client = PerplexityResponseFactory.client_with_response(response) + mock_perplexity_class.return_value = mock_client + + # Call function with max_results=2 + report, citations = fetch_perplexity_deep_research( + query="Test query", + api_key="test_key", # pragma: allowlist secret + max_results=2, + ) + + # Assertions - should only return 2 citations + assert len(citations) == 2 + assert citations == ["https://url1.com", "https://url2.com"] + + +@pytest.mark.unit +@patch("clients.perplexity_client.Perplexity") +def test_fetch_perplexity_deep_research_no_citations(mock_perplexity_class): + """Test handling when API returns no citations.""" + # Setup using factory without citations + response = PerplexityResponseFactory.without_citations() + mock_client = PerplexityResponseFactory.client_with_response(response) + mock_perplexity_class.return_value = mock_client + + # Call function + report, citations = fetch_perplexity_deep_research( + query="Test query", + api_key="test_key", # pragma: allowlist secret + ) + + # Assertions + assert report == "Research without citations" + assert citations == [] diff --git a/docker/tests/unit/services/content_scraper/test_edge_cases.py b/docker/tests/unit/services/content_scraper/test_edge_cases.py index 462d7c1..3abcde2 100644 --- a/docker/tests/unit/services/content_scraper/test_edge_cases.py +++ b/docker/tests/unit/services/content_scraper/test_edge_cases.py @@ -19,13 +19,15 @@ class TestContentScraperEdgeCases: @patch("services.content_scraper.trafilatura.extract") @patch("services.content_scraper.requests.get") def test_truncates_content_exceeding_max_length(self, mock_get, mock_trafilatura): - # Arrange - large_content = "" + ("x" * 100000) + "" + # Arrange - content larger than MAX_CONTENT_LENGTH to trigger truncation + large_content = ( + "" + ("x" * (MAX_CONTENT_LENGTH + 10000)) + "" + ) result = a_search_result().build() mock_get.return_value = HttpResponseFactory.success(content=large_content) # Mock trafilatura to return large content - mock_trafilatura.return_value = "x" * 100000 + mock_trafilatura.return_value = "x" * (MAX_CONTENT_LENGTH + 10000) # Act scraped = scrape_search_result(result) diff --git a/docker/tests/unit/tools/test_deep_research_tool.py b/docker/tests/unit/tools/test_deep_research_tool.py new file mode 100644 index 0000000..b0cd5a5 --- /dev/null +++ b/docker/tests/unit/tools/test_deep_research_tool.py @@ -0,0 +1,161 @@ +# Copyright (c) 2024 Travis Frisinger +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +"""Unit tests for deep_research_tool.""" + +from unittest.mock import patch + +import pytest + +from tools.deep_research_tool import deep_research_tool + + +@pytest.mark.unit +@pytest.mark.asyncio +@patch("tools.deep_research_tool.PERPLEXITY_API_KEY", "test_key") +@patch("tools.deep_research_tool.fetch_perplexity_deep_research") +async def test_deep_research_tool_success(mock_fetch): + """Test successful deep research.""" + # Setup mock + mock_fetch.return_value = ( + "Python is a high-level programming language.", + ["https://python.org", "https://docs.python.org"], + ) + + # Call tool + result = await deep_research_tool( + query="What is Python?", + research_effort="low", + max_results=2, + ) + + # Assertions + assert result["query"] == "What is Python?" + assert result["search_source"] == "Perplexity Deep Research (effort: low)" + assert len(result["results"]) == 1 + + # Check content includes report and citations + content = result["results"][0]["content"] + assert "# Deep Research: What is Python?" in content + assert "*Research Effort: Low*" in content + assert "Python is a high-level programming language." in content + assert "## Sources" in content + assert "1. https://python.org" in content + assert "2. https://docs.python.org" in content + + +@pytest.mark.unit +@pytest.mark.asyncio +@patch("tools.deep_research_tool.PERPLEXITY_API_KEY", "") +async def test_deep_research_tool_no_api_key(): + """Test deep research when API key is not configured.""" + # Call tool without API key + result = await deep_research_tool(query="What is Python?") + + # Assertions + assert result["query"] == "What is Python?" + assert "not configured" in result["search_source"] + assert result["results"] == [] + assert "PERPLEXITY_API_KEY" in result["error"] + + +@pytest.mark.unit +@pytest.mark.asyncio +@patch("tools.deep_research_tool.PERPLEXITY_API_KEY", "test_key") +@patch("tools.deep_research_tool.fetch_perplexity_deep_research") +async def test_deep_research_tool_empty_response(mock_fetch): + """Test deep research when API returns empty response.""" + # Setup mock to return empty + mock_fetch.return_value = ("", []) + + # Call tool + result = await deep_research_tool(query="What is Python?") + + # Assertions + assert result["query"] == "What is Python?" + assert result["results"] == [] + assert "failed" in result["error"].lower() + + +@pytest.mark.unit +@pytest.mark.asyncio +@patch("tools.deep_research_tool.PERPLEXITY_API_KEY", "test_key") +@patch("tools.deep_research_tool.fetch_perplexity_deep_research") +async def test_deep_research_tool_default_params(mock_fetch): + """Test deep research with default parameters.""" + # Setup mock + mock_fetch.return_value = ("Research report", ["https://example.com"]) + + # Call tool with only required param + result = await deep_research_tool(query="Test query") + + # Verify defaults were used + mock_fetch.assert_called_once_with( + query="Test query", + api_key="test_key", # pragma: allowlist secret + max_results=5, # default + research_effort="high", # default + ) + + # Check result + assert result["search_source"] == "Perplexity Deep Research (effort: high)" + + +@pytest.mark.unit +@pytest.mark.asyncio +@patch("tools.deep_research_tool.PERPLEXITY_API_KEY", "test_key") +@patch("tools.deep_research_tool.fetch_perplexity_deep_research") +async def test_deep_research_tool_custom_effort_levels(mock_fetch): + """Test deep research with different effort levels.""" + mock_fetch.return_value = ("Report", []) + + # Test low effort + result = await deep_research_tool(query="Test", research_effort="low") + assert "*Research Effort: Low*" in result["results"][0]["content"] + + # Test medium effort + result = await deep_research_tool(query="Test", research_effort="medium") + assert "*Research Effort: Medium*" in result["results"][0]["content"] + + # Test high effort + result = await deep_research_tool(query="Test", research_effort="high") + assert "*Research Effort: High*" in result["results"][0]["content"] + + +@pytest.mark.unit +@pytest.mark.asyncio +@patch("tools.deep_research_tool.PERPLEXITY_API_KEY", "test_key") +@patch("tools.deep_research_tool.fetch_perplexity_deep_research") +async def test_deep_research_tool_no_citations(mock_fetch): + """Test deep research when no citations are returned.""" + # Setup mock with no citations + mock_fetch.return_value = ("Research content without citations", []) + + # Call tool + result = await deep_research_tool(query="Test query") + + # Assertions + content = result["results"][0]["content"] + assert "Research content without citations" in content + assert "## Sources" not in content # Sources section should not be added + + +@pytest.mark.unit +@pytest.mark.asyncio +@patch("tools.deep_research_tool.PERPLEXITY_API_KEY", "test_key") +@patch("tools.deep_research_tool.fetch_perplexity_deep_research") +async def test_deep_research_tool_long_report_snippet(mock_fetch): + """Test that snippet is truncated for long reports.""" + # Setup mock with very long report + long_report = "A" * 1000 + mock_fetch.return_value = (long_report, []) + + # Call tool + result = await deep_research_tool(query="Test") + + # Check snippet is truncated to 500 chars + "..." + snippet = result["results"][0]["snippet"] + assert len(snippet) == 503 # 500 + "..." + assert snippet.endswith("...") diff --git a/docker/tools/deep_research_tool.py b/docker/tools/deep_research_tool.py new file mode 100644 index 0000000..41e286c --- /dev/null +++ b/docker/tools/deep_research_tool.py @@ -0,0 +1,122 @@ +# Copyright (c) 2024 Travis Frisinger +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +"""Deep research tool - comprehensive research using Perplexity's sonar-deep-research.""" + +import logging +import os +from typing import Literal + +from clients.perplexity_client import fetch_perplexity_deep_research +from models.search_response import SearchResponse +from models.search_result import SearchResult + +logger = logging.getLogger(__name__) + +# Get API key from environment +PERPLEXITY_API_KEY = os.environ.get("PERPLEXITY_API_KEY", "") + + +async def deep_research_tool( + query: str, + research_effort: Literal["low", "medium", "high"] = "high", + max_results: int = 5, +) -> dict: + """Perform comprehensive deep research on a topic using Perplexity AI. + + This tool uses Perplexity's sonar-deep-research model which performs dozens of + searches, reads hundreds of sources, and reasons through material to deliver + comprehensive research reports. Ideal for in-depth analysis, multi-source synthesis, + and thorough investigation of complex topics. + + Takes 2-4 minutes to complete what would take a human expert many hours. + + Args: + query: The research question or topic to investigate + research_effort: Computational depth - "low" (fast), "medium" (balanced), + or "high" (deepest, most comprehensive). Default: "high" + max_results: Maximum number of source results to return (default: 5) + + Returns: + Dict representation of SearchResponse with comprehensive research findings + """ + logger.info( + f"Deep research request: {query} (effort: {research_effort}, " + f"max: {max_results})" + ) + + # Check if Perplexity API key is configured + if not PERPLEXITY_API_KEY: + logger.error("PERPLEXITY_API_KEY not configured") + response = SearchResponse( + query=query, + search_source="Perplexity Deep Research (not configured)", + results=[], + error=( + "Perplexity API key not configured. Please set PERPLEXITY_API_KEY " + "environment variable to use deep research functionality." + ), + ) + return response.model_dump() + + # Fetch deep research from Perplexity + research_report, citation_urls = fetch_perplexity_deep_research( + query=query, + api_key=PERPLEXITY_API_KEY, + max_results=max_results, + research_effort=research_effort, + ) + + # Check if we got research content + if not research_report: + logger.warning(f"No deep research results found for query: {query}") + response = SearchResponse( + query=query, + search_source="Perplexity Deep Research", + results=[], + error=( + "Deep research failed. This could be due to API limits, " + "invalid query, or temporary service issues." + ), + ) + return response.model_dump() + + # Format the research report with title and citations + formatted_content = f"# Deep Research: {query}\n\n" + formatted_content += f"*Research Effort: {research_effort.title()}*\n\n" + formatted_content += "---\n\n" + formatted_content += research_report + formatted_content += "\n\n---\n\n" + + # Add citations section + if citation_urls: + formatted_content += "## Sources\n\n" + for i, url in enumerate(citation_urls, 1): + formatted_content += f"{i}. {url}\n" + + # Create a single SearchResult with the full research report + research_result = SearchResult( + title=f"Deep Research: {query}", + url="", # No URL since this is synthesized research + snippet=( + research_report[:500] + "..." + if len(research_report) > 500 + else research_report + ), + content=formatted_content, + ) + + # Build typed response + response = SearchResponse( + query=query, + search_source=f"Perplexity Deep Research (effort: {research_effort})", + results=[research_result], + ) + + logger.info( + f"Deep research completed: {len(research_report)} chars, " + f"{len(citation_urls)} sources cited" + ) + return response.model_dump() diff --git a/pyproject.toml b/pyproject.toml index 35a094a..237c652 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,9 +25,6 @@ classifiers = [ ] dependencies = [ "requests>=2.31.0", - "beautifulsoup4>=4.12.0", - "readability-lxml>=0.8.1", - "lxml_html_clean>=0.1.0", "python-dotenv>=1.0.0", "fastapi>=0.104.0", "uvicorn>=0.24.0", @@ -35,9 +32,9 @@ dependencies = [ "sse-starlette>=1.6.0", "httpx>=0.24.0", "fastmcp>=2.7.0", - "html2text>=2020.1.16", "duckduckgo-search>=3.9.0", "trafilatura>=1.6.0", + "perplexityai>=0.15.0", ] [project.optional-dependencies] @@ -45,8 +42,6 @@ dev = [ # Code formatting "black>=23.0.0", "isort>=5.12.0", - "autoflake>=2.0.0", - "autopep8>=2.0.0", # Linting and static analysis "flake8>=6.0.0", "flake8-bugbear>=23.0.0", @@ -54,38 +49,24 @@ dev = [ "flake8-docstrings>=1.7.0", "flake8-import-order>=0.18.0", "mypy>=1.5.0", - "pylint>=2.17.0", "types-requests>=2.31.0", - "types-PyYAML>=6.0.0", # Testing "pytest>=7.4.0", "pytest-cov>=4.1.0", "pytest-asyncio>=0.21.0", "pytest-mock>=3.11.0", - "pytest-xdist>=3.3.0", "httpx>=0.24.0", - "testcontainers>=3.7.0", # Security "bandit>=1.7.0", "safety>=2.3.0", # Pre-commit hooks "pre-commit>=3.3.0", # Development tools - "ipdb>=0.13.0", - "ipython>=8.14.0", - "jupyter>=1.0.0", "rich>=13.0.0", "watchdog>=3.0.0", # Build and release "build>=0.10.0", "twine>=4.0.0", - "pip-tools>=7.0.0", - "tox>=4.6.0", - # Profiling - "line-profiler>=4.0.0", - "memory-profiler>=0.61.0", - # CLI tools - "click>=8.1.0", ] test = [ "pytest>=7.4.0", @@ -93,7 +74,6 @@ test = [ "pytest-mock>=3.11.0", "pytest-asyncio>=0.21.0", "httpx>=0.24.0", - "testcontainers>=3.7.0", ] docs = [ "sphinx>=7.1.0", @@ -166,11 +146,11 @@ show_error_codes = true [[tool.mypy.overrides]] module = [ - "readability.*", - "html2text.*", "duckduckgo_search.*", "fastmcp.*", "sse_starlette.*", + "trafilatura.*", + "perplexity.*", ] ignore_missing_imports = true