diff --git a/maestro_backend/ai_researcher/agentic_layer/tools/web_search_tool.py b/maestro_backend/ai_researcher/agentic_layer/tools/web_search_tool.py index 2e04da9..d2d6cf8 100644 --- a/maestro_backend/ai_researcher/agentic_layer/tools/web_search_tool.py +++ b/maestro_backend/ai_researcher/agentic_layer/tools/web_search_tool.py @@ -13,7 +13,8 @@ from ai_researcher.dynamic_config import ( get_web_search_provider, get_tavily_api_key, get_linkup_api_key, get_searxng_base_url, get_searxng_categories, get_jina_api_key, get_search_depth, - get_jina_read_full_content, get_jina_fetch_favicons, get_jina_bypass_cache + get_jina_read_full_content, get_jina_fetch_favicons, get_jina_bypass_cache, + get_yacy_base_url, ) logger = logging.getLogger(__name__) @@ -131,6 +132,19 @@ def __init__(self, controller=None): self.client = api_key # Store the API key as the "client" for Jina self.api_key_configured = True logger.info("WebSearchTool initialized with Jina.") + elif self.provider == "yacy": + if not requests: + raise ImportError("YaCy provider selected, but 'requests' library not installed.") + base_url = get_yacy_base_url() + if not base_url: + logger.warning("YaCy base URL not configured in user settings or environment variables.") + self.api_key_configured = False + return + self.client = { + "base_url": base_url.rstrip('/'), + } + self.api_key_configured = True + logger.info("WebSearchTool initialized with YaCy.") else: raise ValueError(f"Unsupported web search provider configured: {self.provider}") except Exception as e: @@ -462,6 +476,93 @@ async def _execute_search( logger.info(f"Jina search returned no results for query: {search_query}") # Don't set error_msg here, just return empty results + elif self.provider == "yacy": + # YaCy search + base_url = self.client["base_url"] + + search_url = f"{base_url}/yacysearch.json" + + # Build YaCy search parameters + params = {"query": search_query, "count": max_results, "format": "json"} + + # Add optional parameters + if from_date: + params["start_date"] = from_date + if to_date: + params["end_date"] = to_date + + if include_domains: + # Add domain filter + params["site"] = ",".join(include_domains) + + if exclude_domains: + # Add exclusion filter + for domain in exclude_domains: + params["exclude"] = (f"{params.get('exclude', '')} -site:{domain}".strip()) + + async with aiohttp.ClientSession() as session: + async with session.get( + search_url, + params=params, + timeout=aiohttp.ClientTimeout(total=30), + ) as response: + if response.status == 401: + error_msg = f"YaCy returns unauthorized. Check your configuration." + logger.error(error_msg) + else: + response.raise_for_status() + search_data = await response.json() + + # Handle YACY response format + if isinstance(search_data, dict): + # YaCy returns results under channels[0].items + channels = search_data.get("channels", []) + if channels and len(channels) > 0: + # Get the first channel's items + first_channel = channels[0] + if isinstance(first_channel, dict): + results = first_channel.get("items", []) + else: + results = [] + else: + # Fallback to other formats + results = search_data.get("results", []) + if not results: + # Try alternative field names + results = search_data.get("search", []) + if not results: + # Another common format + response_data = search_data.get("response", {}) + if isinstance(response_data, dict): + results = response_data.get("results", []) + else: + results = [] + + for result in results[:max_results]: + formatted_results.append({ + "title": result.get('title', result.get('name', 'No Title')), + "snippet": result.get('description', result.get('content', result.get('snippet', 'No Snippet'))), + "url": result.get("url", result.get("link", "#")), + }) + elif isinstance(search_data, list): + # Direct list format + for result in search_data[:max_results]: + formatted_results.append({ + "title": result.get("title", result.get("name", "No Title")), + "snippet": result.get("description", result.get("content", result.get("snippet", "No Snippet"))), + "url": result.get("url", result.get("link", "#")), + }) + + if error_msg: + logger.warning(f"Unexpected YaCy response format: {type(search_data)}") + # Direct list format + for result in search_data[:max_results]: + formatted_results.append({ + "title": result.get("title", result.get("name", "No Title")), + "snippet": result.get("description", result.get("content", result.get("snippet", "No Snippet"))), + "url": result.get("url", result.get("link", "#")), + }) + if error_msg: return {"error": error_msg} diff --git a/maestro_backend/ai_researcher/dynamic_config.py b/maestro_backend/ai_researcher/dynamic_config.py index 14187df..14a3622 100644 --- a/maestro_backend/ai_researcher/dynamic_config.py +++ b/maestro_backend/ai_researcher/dynamic_config.py @@ -1,4 +1,5 @@ import os +#from pathlib import Path from typing import Dict, Any, Optional from ai_researcher.user_context import get_user_settings @@ -222,6 +223,18 @@ def get_searxng_categories(mission_id: Optional[str] = None) -> str: return os.getenv("SEARXNG_CATEGORIES", "general") +def get_yacy_base_url(mission_id: Optional[str] = None) -> Optional[str]: + """Get the YACY base URL from user settings or environment.""" + # Check user settings first + user_settings = get_user_settings() + if user_settings: + search_settings = user_settings.get("search", {}) + if search_settings and search_settings.get("yacy_base_url"): + return search_settings["yacy_base_url"] + + # Fallback to environment variable + return os.getenv("YACY_BASE_URL") + def get_search_depth(mission_id: Optional[str] = None) -> str: """Get the search depth (standard/advanced) from user settings or environment.""" # Check user settings first diff --git a/maestro_backend/api/schemas.py b/maestro_backend/api/schemas.py index 759ca94..b9071f7 100644 --- a/maestro_backend/api/schemas.py +++ b/maestro_backend/api/schemas.py @@ -86,6 +86,7 @@ class SearchSettings(BaseModel): jina_read_full_content: Optional[bool] = None jina_fetch_favicons: Optional[bool] = None jina_bypass_cache: Optional[bool] = None + yacy_base_url: Optional[str] = None class WebFetchSettings(BaseModel): provider: str = "original" # "original", "jina", or "original_with_jina_fallback" diff --git a/maestro_frontend/src/features/auth/components/SearchSettingsTab.tsx b/maestro_frontend/src/features/auth/components/SearchSettingsTab.tsx index c7f4660..f1d06af 100644 --- a/maestro_frontend/src/features/auth/components/SearchSettingsTab.tsx +++ b/maestro_frontend/src/features/auth/components/SearchSettingsTab.tsx @@ -28,51 +28,51 @@ const SEARXNG_CATEGORIES = [ export const SearchSettingsTab: React.FC = () => { const { draftSettings, setDraftSettings } = useSettingsStore() - const handleProviderChange = (provider: 'tavily' | 'linkup' | 'searxng' | 'jina') => { + const handleProviderChange = (provider: 'tavily' | 'linkup' | 'searxng' | 'jina' | 'yacy') => { if (!draftSettings) return - + const newSearch = { ...draftSettings.search, provider } - + setDraftSettings({ search: newSearch }) } const handleApiKeyChange = (field: string, value: string | boolean | number) => { if (!draftSettings) return - + const newSearch = { ...draftSettings.search, [field]: value } - + setDraftSettings({ search: newSearch }) } const handleCategoriesChange = (categoryValue: string, checked: boolean) => { if (!draftSettings) return - + const currentCategories = draftSettings.search.searxng_categories || 'general' const categoriesArray = currentCategories.split(',').map(c => c.trim()).filter(c => c) - + let newCategoriesArray if (checked) { newCategoriesArray = [...categoriesArray.filter(c => c !== categoryValue), categoryValue] } else { newCategoriesArray = categoriesArray.filter(c => c !== categoryValue) } - + // Ensure at least one category is selected if (newCategoriesArray.length === 0) { newCategoriesArray = ['general'] } - + const newSearch = { ...draftSettings.search, searxng_categories: newCategoriesArray.join(',') } - + setDraftSettings({ search: newSearch }) } @@ -120,6 +120,7 @@ export const SearchSettingsTab: React.FC = () => { LinkUp SearXNG Jina + YaCy @@ -142,9 +143,9 @@ export const SearchSettingsTab: React.FC = () => {

Get your API key from{' '} - @@ -172,9 +173,9 @@ export const SearchSettingsTab: React.FC = () => {

Get your API key from{' '} - @@ -202,9 +203,9 @@ export const SearchSettingsTab: React.FC = () => {

Get your API key from{' '} - @@ -283,9 +284,9 @@ export const SearchSettingsTab: React.FC = () => {

Enter the URL of your SearXNG instance. You can use a public instance or{' '} - @@ -296,6 +297,28 @@ export const SearchSettingsTab: React.FC = () => {

)} + + {draftSettings.search.provider === 'yacy' && ( +
+

+ Custom YaCy instance search interface. +

+
+ + handleApiKeyChange('yacy_base_url', e.target.value)} + placeholder="https://your-yacy-instance.com" + className="h-8 text-sm" + /> +
+

+ Enter the base URL of your YaCy instance. YaCy should have json results enabled. +

+
+ )} @@ -338,7 +361,7 @@ export const SearchSettingsTab: React.FC = () => {

- {draftSettings.search.provider === 'tavily' + {draftSettings.search.provider === 'tavily' ? 'Advanced search provides more comprehensive results but costs 2x API credits.' : 'Deep search uses an agentic workflow for more comprehensive results but takes longer.'}

@@ -477,7 +500,7 @@ export const SearchSettingsTab: React.FC = () => { characters

- Maximum length for search queries (100-400 characters). Queries exceeding this limit will be intelligently refined to preserve search intent. + Maximum length for search queries (100-400 characters). Queries exceeding this limit will be intelligently refined to preserve search intent. Default is 350 to ensure compatibility with most search providers. Tavily has a hard limit of 400 characters.

diff --git a/maestro_frontend/src/features/auth/components/SettingsStore.ts b/maestro_frontend/src/features/auth/components/SettingsStore.ts index 0af3f47..fb58321 100644 --- a/maestro_frontend/src/features/auth/components/SettingsStore.ts +++ b/maestro_frontend/src/features/auth/components/SettingsStore.ts @@ -32,7 +32,7 @@ interface AISettings { } interface SearchSettings { - provider: 'tavily' | 'linkup' | 'searxng' | 'jina' + provider: 'tavily' | 'linkup' | 'searxng' | 'jina' | 'yacy' tavily_api_key: string | null linkup_api_key: string | null jina_api_key: string | null @@ -45,6 +45,7 @@ interface SearchSettings { jina_bypass_cache?: boolean // Query refinement settings max_query_length?: number + yacy_base_url: string | null } export interface ResearchParameters { @@ -140,29 +141,29 @@ interface SettingsState { // Validation function for AI settings const validateAISettings = (settings: UserSettings): string | null => { const { ai_endpoints } = settings - + if (!ai_endpoints.advanced_mode) { // Simple mode validation - check enabled provider const enabledProvider = Object.entries(ai_endpoints.providers).find( ([_, config]) => config.enabled ) - + if (!enabledProvider) { return 'Please select an AI provider' } - + const [providerName, providerConfig] = enabledProvider - + // Check if API key is provided for providers that require it if ((providerName === 'openrouter' || providerName === 'openai') && !providerConfig.api_key) { return `Please provide an API key for ${providerName === 'openrouter' ? 'OpenRouter' : 'OpenAI'}` } - + // Check if base URL is provided for custom provider if (providerName === 'custom' && !providerConfig.base_url) { return 'Please provide a base URL for the custom provider' } - + // Check if all models have names selected const modelTypes = ['fast', 'mid', 'intelligent', 'verifier'] as const for (const modelType of modelTypes) { @@ -176,27 +177,27 @@ const validateAISettings = (settings: UserSettings): string | null => { const modelTypes = ['fast', 'mid', 'intelligent', 'verifier'] as const for (const modelType of modelTypes) { const model = ai_endpoints.advanced_models[modelType] - + if (!model.provider) { return `Please select a provider for ${modelType} model` } - + if (!model.model_name) { return `Please select a model for ${modelType} configuration` } - + // Check API key for providers that require it if ((model.provider === 'openrouter' || model.provider === 'openai') && !model.api_key) { return `Please provide an API key for ${modelType} model (${model.provider})` } - + // Check base URL for custom provider if (model.provider === 'custom' && !model.base_url) { return `Please provide a base URL for ${modelType} model (custom provider)` } } } - + return null } @@ -258,7 +259,8 @@ const defaultSettings: UserSettings = { jina_api_key: null, searxng_base_url: null, searxng_categories: null, - search_depth: 'standard' + search_depth: 'standard', + yacy_base_url: null, }, web_fetch: { provider: 'original', @@ -306,7 +308,7 @@ export const useSettingsStore = create()((set, get) => ({ settingsApi.getSettings(), settingsApi.getProfile() ]) - + // Merge with default settings to ensure all fields are present const mergedSettings = { ...defaultSettings, @@ -336,18 +338,18 @@ export const useSettingsStore = create()((set, get) => ({ ...userSettings?.appearance } } - - set({ - settings: mergedSettings, - draftSettings: mergedSettings, + + set({ + settings: mergedSettings, + draftSettings: mergedSettings, profile: userProfile, draftProfile: userProfile, - isLoading: false + isLoading: false }) } catch (error) { console.error('Failed to load settings:', error) - set({ - error: 'Failed to load settings', + set({ + error: 'Failed to load settings', isLoading: false, settings: defaultSettings, draftSettings: defaultSettings, @@ -360,16 +362,16 @@ export const useSettingsStore = create()((set, get) => ({ setDraftSettings: (newDraftSettings) => { set(state => { if (!state.draftSettings) return state; - + // If the newDraftSettings is a complete settings object (has all required properties), // use it directly instead of merging - if (newDraftSettings.ai_endpoints && newDraftSettings.search && + if (newDraftSettings.ai_endpoints && newDraftSettings.search && newDraftSettings.research_parameters && newDraftSettings.appearance) { return { draftSettings: newDraftSettings as UserSettings }; } - + // Otherwise, do the deep merge for partial updates return { draftSettings: { @@ -411,7 +413,7 @@ export const useSettingsStore = create()((set, get) => ({ setProfileField: (field, value) => { set(state => { if (!state.draftProfile) return state; - + return { draftProfile: { ...state.draftProfile, @@ -422,9 +424,9 @@ export const useSettingsStore = create()((set, get) => ({ }, discardDraftChanges: () => { - set(state => ({ + set(state => ({ draftSettings: state.settings, - draftProfile: state.profile + draftProfile: state.profile })) }, @@ -434,37 +436,37 @@ export const useSettingsStore = create()((set, get) => ({ try { set({ isLoading: true, error: null }) - + // Validate AI settings before saving const validationError = validateAISettings(draftSettings) if (validationError) { - set({ - error: validationError, - isLoading: false + set({ + error: validationError, + isLoading: false }) throw new Error(validationError) } - + const [savedSettings, savedProfile] = await Promise.all([ settingsApi.updateSettings(draftSettings), settingsApi.updateProfile(draftProfile) ]) - - set({ - settings: savedSettings, - draftSettings: savedSettings, + + set({ + settings: savedSettings, + draftSettings: savedSettings, profile: savedProfile, draftProfile: savedProfile, - isLoading: false + isLoading: false }) } catch (error: any) { console.error('Failed to update settings:', error) - const errorMessage = error.response?.data?.detail || - error.message || + const errorMessage = error.response?.data?.detail || + error.message || 'Failed to update settings' - set({ - error: errorMessage, - isLoading: false + set({ + error: errorMessage, + isLoading: false }) throw error } @@ -473,20 +475,20 @@ export const useSettingsStore = create()((set, get) => ({ testConnection: async (provider, apiKey, baseUrl) => { try { set({ isTestingConnection: true, connectionTestResult: null, error: null }) - + const result = await settingsApi.testConnection(provider, apiKey, baseUrl) - - set({ - isTestingConnection: false, - connectionTestResult: result + + set({ + isTestingConnection: false, + connectionTestResult: result }) } catch (error: any) { console.error('Connection test failed:', error) - set({ - isTestingConnection: false, - connectionTestResult: { - success: false, - message: error.response?.data?.detail || 'Connection test failed' + set({ + isTestingConnection: false, + connectionTestResult: { + success: false, + message: error.response?.data?.detail || 'Connection test failed' } }) } @@ -498,7 +500,7 @@ export const useSettingsStore = create()((set, get) => ({ const { draftSettings } = get() let apiKey: string | undefined let baseUrl: string | undefined - + if (draftSettings && provider) { // In simple mode, get from the enabled provider if (!draftSettings.ai_endpoints.advanced_mode) { @@ -519,9 +521,9 @@ export const useSettingsStore = create()((set, get) => ({ } } } - + const modelsResult = await settingsApi.getAvailableModels(provider, apiKey, baseUrl) - set(state => ({ + set(state => ({ modelsByProvider: { ...state.modelsByProvider, [provider || 'default']: modelsResult.models || [] @@ -531,10 +533,10 @@ export const useSettingsStore = create()((set, get) => ({ })) } catch (error: any) { console.error('Failed to fetch models:', error) - const errorMessage = error.response?.data?.detail || - error.message || + const errorMessage = error.response?.data?.detail || + error.message || 'Failed to fetch models. Please check your API key and base URL.' - set({ + set({ modelsFetchError: errorMessage, isLoading: false }) diff --git a/maestro_frontend/src/features/auth/components/settingsApi.ts b/maestro_frontend/src/features/auth/components/settingsApi.ts index 45dd269..3873689 100644 --- a/maestro_frontend/src/features/auth/components/settingsApi.ts +++ b/maestro_frontend/src/features/auth/components/settingsApi.ts @@ -31,7 +31,7 @@ export interface AISettings { } export interface SearchSettings { - provider: 'tavily' | 'linkup' | 'searxng' | 'jina' + provider: 'tavily' | 'linkup' | 'searxng' | 'jina' | 'yacy' tavily_api_key: string | null linkup_api_key: string | null jina_api_key: string | null @@ -42,6 +42,7 @@ export interface SearchSettings { jina_read_full_content?: boolean jina_fetch_favicons?: boolean jina_bypass_cache?: boolean + yacy_base_url: string | null } export interface ResearchParameters { @@ -146,16 +147,16 @@ export const settingsApi = { // Get available models getAvailableModels: async ( - provider?: string, - apiKey?: string, + provider?: string, + apiKey?: string, baseUrl?: string ): Promise => { const params = new URLSearchParams() if (provider) params.append('provider', provider) if (apiKey) params.append('api_key', apiKey) if (baseUrl) params.append('base_url', baseUrl) - - const url = params.toString() + + const url = params.toString() ? `${API_CONFIG.ENDPOINTS.SETTINGS.GET_MODELS}?${params.toString()}` : API_CONFIG.ENDPOINTS.SETTINGS.GET_MODELS const response = await apiClient.get(url)