diff --git a/.env.patentsberta.example b/.env.patentsberta.example new file mode 100644 index 0000000000..091a056e9a --- /dev/null +++ b/.env.patentsberta.example @@ -0,0 +1,43 @@ +# PatentsBERTa Embedding Configuration Example +# Copy this file to .env and update with your values + +# Enable PatentsBERTa embeddings +OPENAI_HOST=patentsberta + +# PatentsBERTa service configuration +PATENTSBERTA_ENDPOINT=https://patentsberta-embeddings.YOUR_REGION.azurecontainerapps.io +PATENTSBERTA_API_KEY=your-secure-api-key-here # Required for API authentication + +# Embedding configuration for PatentsBERTa (768 dimensions) +AZURE_OPENAI_EMB_DIMENSIONS=768 +AZURE_OPENAI_EMB_MODEL_NAME=PatentSBERTa + +# Search index configuration +AZURE_SEARCH_FIELD_NAME_EMBEDDING=embedding_patentsberta + +# Keep existing Azure configuration +AZURE_SUBSCRIPTION_ID=your-subscription-id +AZURE_RESOURCE_GROUP=rg-ai-master-engineer +AZURE_LOCATION=eastus + +# Azure Search Service +AZURE_SEARCH_SERVICE=your-search-service +AZURE_SEARCH_INDEX=your-search-index +AZURE_SEARCH_KEY=your-search-key + +# Azure Storage +AZURE_STORAGE_ACCOUNT=your-storage-account +AZURE_STORAGE_CONTAINER=content +AZURE_STORAGE_KEY=your-storage-key + +# Azure OpenAI (still needed for chat completions) +AZURE_OPENAI_SERVICE=your-openai-service +AZURE_OPENAI_CHATGPT_DEPLOYMENT=your-chat-deployment +AZURE_OPENAI_CHATGPT_MODEL=gpt-4 +AZURE_OPENAI_API_VERSION=2024-06-01 + +# Optional: Document Intelligence for advanced parsing +AZURE_DOCUMENTINTELLIGENCE_SERVICE=your-doc-intelligence-service + +# Optional: Vision services for multimodal +AZURE_VISION_ENDPOINT=https://your-vision-service.cognitiveservices.azure.com/ diff --git a/.gitignore b/.gitignore index 05bbf3b060..2b5c9875e7 100644 --- a/.gitignore +++ b/.gitignore @@ -109,8 +109,13 @@ celerybeat.pid # SageMath parsed files *.sage.py -# Environments +# Environments and secrets .env +.env.* +*.env +*.key +*.secret +*.pem .venv .evalenv env/ @@ -119,6 +124,14 @@ ENV/ env.bak/ venv.bak/ +# API keys and credentials +*_key +*_secret +*_token +credentials.json +secrets.json +agents/verify_credentials.py + # Spyder project settings .spyderproject .spyproject @@ -149,5 +162,37 @@ node_modules static/ data/**/*.md5 - +/data/* +!/data/.gitkeep +!/data/README.md + +# Large files and data +*.pdf +!tests/test-data/*.pdf +*.zip +*.tar.gz +*.7z +*.rar + +# Cache and temporary files +.cache/ +*.cache +*.tmp +*.temp +*.swp +*.swo +*~ + +# OS files .DS_Store +Thumbs.db +desktop.ini + +# IDE files +.vscode/ +.idea/ +*.sublime-* + +tests/test_application.py +tests/test_ollama_ocr.py +tests/test_response_accuracy.py diff --git a/SECURITY.md b/SECURITY.md deleted file mode 100644 index 388e9ad471..0000000000 --- a/SECURITY.md +++ /dev/null @@ -1,41 +0,0 @@ - - -## Security - -Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). - -If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](), please report it to us as described below. - -## Reporting Security Issues - -**Please do not report security vulnerabilities through public GitHub issues.** - -Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report). - -If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/msrc/pgp-key-msrc). - -You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). - -Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: - -- Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) -- Full paths of source file(s) related to the manifestation of the issue -- The location of the affected source code (tag/branch/commit or direct URL) -- Any special configuration required to reproduce the issue -- Step-by-step instructions to reproduce the issue -- Proof-of-concept or exploit code (if possible) -- Impact of the issue, including how an attacker might exploit the issue - -This information will help us triage your report more quickly. - -If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs. - -## Preferred Languages - -We prefer all communications to be in English. - -## Policy - -Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/msrc/cvd). - - diff --git a/agents/.deployment b/agents/.deployment new file mode 100644 index 0000000000..f2d2a53e39 --- /dev/null +++ b/agents/.deployment @@ -0,0 +1,7 @@ +[config] +SCM_DO_BUILD_DURING_DEPLOYMENT=true + + + + + diff --git a/agents/README.md b/agents/README.md new file mode 100644 index 0000000000..3865a8d964 --- /dev/null +++ b/agents/README.md @@ -0,0 +1,187 @@ +# Microsoft 365 RAG Agent + +This directory contains the Microsoft 365 Agents SDK client that replaces the web frontend. The agent provides AI-powered document search and chat capabilities across Microsoft 365 channels including Teams, Copilot, and web chat by calling the existing backend API. + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Microsoft 365 Channels │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ Teams │ │ Copilot │ │ Web Chat │ │ +│ └─────────────┘ └─────────────┘ └─────────────┘ │ +└─────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ Microsoft 365 Agents SDK │ +│ ┌─────────────────────────────────────────────────────┐ │ +│ │ Agent Application │ │ +│ │ • Message Handlers │ │ +│ │ • Channel Adapters │ │ +│ │ • Response Formatting │ │ +│ └─────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ Existing Backend │ +│ ┌─────────────────────────────────────────────────────┐ │ +│ │ Quart API Server │ │ +│ │ • /chat endpoint │ │ +│ │ • /ask endpoint │ │ +│ │ • RAG Approaches │ │ +│ │ • Azure Services │ │ +│ └─────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ +``` + +## Project Structure + +``` +agents/ +├── main.py # Main entry point +├── agent_app.py # Core agent application +├── config/ +│ └── agent_config.py # Configuration management +├── services/ +│ ├── rag_service.py # Backend API client +│ └── auth_service.py # Authentication service +├── handlers/ +│ ├── message_handler.py # General message handler +│ └── teams_handler.py # Teams-specific handler +├── adapters/ +│ └── response_adapter.py # Channel-specific response formatting +└── requirements.txt # Python dependencies +``` + +## Features + +- **Multi-Channel Support**: Works with Teams, Copilot, and web chat +- **Backend Integration**: Calls existing RAG backend API +- **Authentication**: Microsoft 365 authentication and authorization +- **Rich Responses**: Adaptive cards, citations, and interactive elements +- **Conversation State**: Maintains context across conversations +- **Error Handling**: Robust error handling and logging +- **No Duplication**: Reuses existing backend logic and services + +## Setup + +### 1. Install Dependencies + +```bash +cd agents +pip install -r requirements.txt +``` + +### 2. Configure Environment + +```bash +cp .env.example .env +# Edit .env with your configuration values +``` + +### 3. Required Configuration + +- **Bot Framework**: App ID and password from Azure Bot Service +- **Microsoft 365**: Tenant ID, client ID, and client secret +- **Backend API**: URL of the existing RAG backend (e.g., http://localhost:50505) + +### 4. Run the Agent + +```bash +python main.py +``` + +## Configuration + +### Environment Variables + +| Variable | Description | Required | +|----------|-------------|----------| +| `MICROSOFT_APP_ID` | Bot Framework app ID | Yes | +| `MICROSOFT_APP_PASSWORD` | Bot Framework app password | Yes | +| `AZURE_TENANT_ID` | Microsoft 365 tenant ID | Yes | +| `AZURE_CLIENT_ID` | Microsoft 365 client ID | Yes | +| `AZURE_CLIENT_SECRET` | Microsoft 365 client secret | Yes | +| `BACKEND_URL` | URL of the existing RAG backend | Yes | + +### Agent Settings + +| Variable | Description | Default | +|----------|-------------|---------| +| `AGENT_NAME` | Display name for the agent | "RAG Assistant" | +| `AGENT_DESCRIPTION` | Agent description | "AI-powered document search and chat assistant" | +| `MAX_CONVERSATION_TURNS` | Maximum conversation turns | 20 | +| `ENABLE_TYPING_INDICATOR` | Enable typing indicators | true | +| `ENABLE_TEAMS` | Enable Teams channel | true | +| `ENABLE_COPILOT` | Enable Copilot channel | true | +| `ENABLE_WEB_CHAT` | Enable web chat channel | true | + +## API Endpoints + +### Health Check +- **GET** `/` - Basic health check +- **GET** `/api/health` - Detailed health check + +### Bot Framework +- **POST** `/api/messages` - Main Bot Framework endpoint + +### Configuration +- **GET** `/api/config` - Get agent configuration (non-sensitive) + +## Development + +### Running Locally + +1. Set up your environment variables +2. Run the agent: `python main.py` +3. Use Bot Framework Emulator to test locally + +### Testing with Teams + +1. Deploy to Azure +2. Register with Azure Bot Service +3. Configure Teams channel +4. Test in Teams + +## Integration with Backend + +The agent integrates with the existing RAG backend by: + +1. **API Calls**: Calls existing `/chat` and `/chat/stream` endpoints +2. **No Duplication**: Reuses all existing RAG logic and services +3. **Authentication**: Passes through user context to backend +4. **Response Formatting**: Adapts backend responses for Microsoft 365 channels + +## Next Steps + +1. **Phase 2**: Test backend integration and response formatting +2. **Phase 3**: Add Teams-specific features (adaptive cards, file handling) +3. **Phase 4**: Implement Copilot integration +4. **Phase 5**: Add advanced features and monitoring + +## Troubleshooting + +### Common Issues + +1. **Authentication Errors**: Check Microsoft 365 app registration +2. **Bot Framework Errors**: Verify app ID and password +3. **Azure Service Errors**: Check service endpoints and keys +4. **Channel Errors**: Verify channel configuration + +### Logs + +The agent logs to stdout with structured logging. Check logs for: +- Authentication issues +- Service connection problems +- Message processing errors +- Channel-specific issues + +## Support + +For issues and questions: +1. Check the logs for error details +2. Verify configuration values +3. Test with Bot Framework Emulator +4. Check Azure service status \ No newline at end of file diff --git a/agents/adapters/__init__.py b/agents/adapters/__init__.py new file mode 100644 index 0000000000..34d3c5fd0c --- /dev/null +++ b/agents/adapters/__init__.py @@ -0,0 +1,6 @@ +# Adapters package + + + + + diff --git a/agents/adapters/copilot_response_adapter.py b/agents/adapters/copilot_response_adapter.py new file mode 100644 index 0000000000..687a5d7c89 --- /dev/null +++ b/agents/adapters/copilot_response_adapter.py @@ -0,0 +1,193 @@ +""" +Copilot-specific response adapter for formatting responses for Microsoft Copilot. +This adapter formats RAG responses specifically for Copilot UI consumption. +""" + +import logging +from typing import Dict, Any, List, Optional + +from services.rag_service import RAGResponse +from models.citation import Citation, CitationSource, resolve_citation_conflicts + + +logger = logging.getLogger(__name__) + + +class CopilotResponseAdapter: + """ + Response adapter for Microsoft Copilot plugin. + Formats responses to be consumed by Copilot UI. + """ + + def __init__(self): + logger.info("CopilotResponseAdapter initialized") + + def format_rag_response( + self, + rag_response: RAGResponse, + include_metadata: bool = True + ) -> Dict[str, Any]: + """ + Format a RAG response for Copilot. + + Args: + rag_response: RAG response from backend + include_metadata: Whether to include token usage and model info + + Returns: + Formatted response dictionary for Copilot + """ + try: + # Start with answer + response = { + "answer": rag_response.answer + } + + # Format citations + citations = self._format_citations(rag_response) + if citations: + response["citations"] = citations + + # Add sources (for Copilot's source tracking) + sources = self._format_sources(rag_response) + if sources: + response["sources"] = sources + + # Add thoughts if available + if rag_response.thoughts: + response["thoughts"] = [ + { + "title": thought.get("title", ""), + "description": thought.get("description", "") + } + for thought in rag_response.thoughts + ] + + # Add metadata if requested + if include_metadata: + metadata = {} + if rag_response.token_usage: + metadata["token_usage"] = rag_response.token_usage + if rag_response.model_info: + metadata["model_info"] = rag_response.model_info + if metadata: + response["metadata"] = metadata + + return response + + except Exception as e: + logger.error(f"Error formatting Copilot response: {e}", exc_info=True) + return { + "answer": rag_response.answer, + "citations": [], + "error": str(e) + } + + def _format_citations( + self, + rag_response: RAGResponse + ) -> List[Dict[str, Any]]: + """ + Format citations for Copilot. + Uses unified citations if available, falls back to legacy format. + """ + citations = [] + + # Prefer unified citations + if rag_response.unified_citations: + # Resolve conflicts (prefer corpus) + resolved = resolve_citation_conflicts( + rag_response.unified_citations, + prefer_corpus=True + ) + + for citation in resolved: + citations.append({ + "title": citation.title, + "url": citation.url, + "snippet": citation.snippet[:300] if citation.snippet else "", + "source": citation.source.value, + "provider": citation.provider.value, + "confidence": citation.confidence + }) + else: + # Fallback to legacy citations + for citation_str in rag_response.citations: + if citation_str: + citations.append({ + "title": citation_str[:100], + "url": "", + "snippet": citation_str, + "source": "unknown", + "provider": "unknown" + }) + + return citations + + def _format_sources( + self, + rag_response: RAGResponse + ) -> List[Dict[str, Any]]: + """ + Format sources for Copilot's source tracking. + """ + sources = [] + + # Add sources from rag_response.sources + for source in rag_response.sources: + if isinstance(source, dict): + sources.append({ + "title": source.get("title", source.get("sourcefile", "Document")), + "url": source.get("url", source.get("sourcepage", "")), + "type": "document" + }) + + return sources + + def format_search_results( + self, + results: List[Dict[str, Any]], + query: str + ) -> Dict[str, Any]: + """ + Format search results for Copilot search endpoint. + """ + formatted_results = [] + + for result in results: + formatted_results.append({ + "title": result.get("title", "Document"), + "url": result.get("url", ""), + "snippet": result.get("snippet", result.get("content", ""))[:300], + "source": result.get("source", "corpus") + }) + + return { + "results": formatted_results, + "totalCount": len(formatted_results), + "query": query + } + + def format_error_response( + self, + error_message: str, + error_code: Optional[str] = None + ) -> Dict[str, Any]: + """ + Format error response for Copilot. + """ + response = { + "answer": f"I encountered an error: {error_message}", + "citations": [], + "error": error_message + } + + if error_code: + response["error_code"] = error_code + + return response + + + + + diff --git a/agents/adapters/response_adapter.py b/agents/adapters/response_adapter.py new file mode 100644 index 0000000000..cdfd42a616 --- /dev/null +++ b/agents/adapters/response_adapter.py @@ -0,0 +1,301 @@ +""" +Response Adapter for Microsoft 365 Agent. +This adapter formats RAG responses for different Microsoft 365 channels. +""" + +import logging +from typing import Dict, Any, List, Optional +from dataclasses import dataclass + +from services.rag_service import RAGResponse + + +logger = logging.getLogger(__name__) + + +@dataclass +class FormattedResponse: + """Formatted response for a specific channel.""" + text: str + attachments: Optional[List[Dict[str, Any]]] = None + suggested_actions: Optional[List[Dict[str, Any]]] = None + channel_specific: Optional[Dict[str, Any]] = None + + +class ResponseAdapter: + """ + Adapter that formats RAG responses for different Microsoft 365 channels. + This ensures consistent user experience across Teams, Copilot, and other channels. + """ + + def __init__(self): + self.max_response_length = 4000 # Maximum length for most channels + self.max_sources = 5 # Maximum number of sources to show + self.max_citations = 3 # Maximum number of citations to show + + async def format_response( + self, + rag_response: RAGResponse, + channel_id: str + ) -> Dict[str, Any]: + """ + Format a RAG response for the specified channel. + """ + try: + if channel_id == "msteams": + return await self._format_for_teams(rag_response) + elif channel_id == "webchat": + return await self._format_for_web_chat(rag_response) + elif channel_id == "email": + return await self._format_for_email(rag_response) + else: + return await self._format_for_default(rag_response) + + except Exception as e: + logger.error(f"Error formatting response for channel {channel_id}: {e}") + return await self._format_for_default(rag_response) + + async def _format_for_teams(self, rag_response: RAGResponse) -> Dict[str, Any]: + """ + Format response specifically for Microsoft Teams. + Teams supports rich formatting, adaptive cards, and interactive elements. + """ + try: + # Base response text + response_text = rag_response.answer + + # Add sources if available + if rag_response.sources: + sources_text = "\n\n**📚 Sources:**\n" + for i, source in enumerate(rag_response.sources[:self.max_sources], 1): + source_title = source.get("title", "Unknown Source") + source_url = source.get("url", "") + if source_url: + sources_text += f"{i}. [{source_title}]({source_url})\n" + else: + sources_text += f"{i}. {source_title}\n" + response_text += sources_text + + # Add citations if available + if rag_response.citations: + citations_text = "\n\n**🔗 Citations:**\n" + for i, citation in enumerate(rag_response.citations[:self.max_citations], 1): + citations_text += f"{i}. {citation}\n" + response_text += citations_text + + # Add thoughts if available (for debugging/transparency) + if rag_response.thoughts: + thoughts_text = "\n\n**💭 Process:**\n" + for thought in rag_response.thoughts[:2]: # Limit to 2 thoughts + thoughts_text += f"• {thought.get('title', 'Step')}: {thought.get('description', '')}\n" + response_text += thoughts_text + + # Create suggested actions for Teams + suggested_actions = [ + { + "type": "imBack", + "title": "Ask Follow-up", + "value": "Can you provide more details about this?" + }, + { + "type": "imBack", + "title": "Search Related", + "value": "Find more information about this topic" + }, + { + "type": "imBack", + "title": "Summarize", + "value": "Can you summarize the key points?" + } + ] + + return { + "text": response_text, + "suggested_actions": suggested_actions, + "channel_specific": { + "teams": { + "supports_adaptive_cards": True, + "supports_mentions": True, + "supports_file_uploads": True + } + } + } + + except Exception as e: + logger.error(f"Error formatting for Teams: {e}") + return await self._format_for_default(rag_response) + + async def _format_for_web_chat(self, rag_response: RAGResponse) -> Dict[str, Any]: + """ + Format response for web chat interface. + Web chat supports rich HTML formatting and interactive elements. + """ + try: + # Base response text with HTML formatting + response_text = rag_response.answer + + # Add sources with HTML links + if rag_response.sources: + sources_html = "\n\n📚 Sources:
" + for i, source in enumerate(rag_response.sources[:self.max_sources], 1): + source_title = source.get("title", "Unknown Source") + source_url = source.get("url", "") + if source_url: + sources_html += f"{i}. {source_title}
" + else: + sources_html += f"{i}. {source_title}
" + response_text += sources_html + + # Add citations + if rag_response.citations: + citations_html = "\n\n🔗 Citations:
" + for i, citation in enumerate(rag_response.citations[:self.max_citations], 1): + citations_html += f"{i}. {citation}
" + response_text += citations_html + + # Create quick reply buttons + suggested_actions = [ + { + "type": "postBack", + "title": "Ask Follow-up", + "value": "Can you provide more details about this?" + }, + { + "type": "postBack", + "title": "Search Related", + "value": "Find more information about this topic" + }, + { + "type": "postBack", + "title": "New Question", + "value": "I have a different question" + } + ] + + return { + "text": response_text, + "suggested_actions": suggested_actions, + "channel_specific": { + "web_chat": { + "supports_html": True, + "supports_quick_replies": True, + "supports_typing_indicator": True + } + } + } + + except Exception as e: + logger.error(f"Error formatting for web chat: {e}") + return await self._format_for_default(rag_response) + + async def _format_for_email(self, rag_response: RAGResponse) -> Dict[str, Any]: + """ + Format response for email channels. + Email has limited formatting options and should be concise. + """ + try: + # Base response text (plain text) + response_text = rag_response.answer + + # Add sources (plain text) + if rag_response.sources: + sources_text = "\n\nSources:\n" + for i, source in enumerate(rag_response.sources[:self.max_sources], 1): + source_title = source.get("title", "Unknown Source") + source_url = source.get("url", "") + if source_url: + sources_text += f"{i}. {source_title} - {source_url}\n" + else: + sources_text += f"{i}. {source_title}\n" + response_text += sources_text + + # Add citations (plain text) + if rag_response.citations: + citations_text = "\n\nCitations:\n" + for i, citation in enumerate(rag_response.citations[:self.max_citations], 1): + citations_text += f"{i}. {citation}\n" + response_text += citations_text + + # Truncate if too long for email + if len(response_text) > 2000: + response_text = response_text[:2000] + "\n\n[Response truncated for email]" + + return { + "text": response_text, + "channel_specific": { + "email": { + "supports_html": False, + "max_length": 2000, + "plain_text_only": True + } + } + } + + except Exception as e: + logger.error(f"Error formatting for email: {e}") + return await self._format_for_default(rag_response) + + async def _format_for_default(self, rag_response: RAGResponse) -> Dict[str, Any]: + """ + Format response for default/unknown channels. + This provides a basic, universally compatible format. + """ + try: + # Base response text + response_text = rag_response.answer + + # Add sources (simple format) + if rag_response.sources: + sources_text = "\n\nSources:\n" + for i, source in enumerate(rag_response.sources[:self.max_sources], 1): + source_title = source.get("title", "Unknown Source") + sources_text += f"{i}. {source_title}\n" + response_text += sources_text + + # Add citations (simple format) + if rag_response.citations: + citations_text = "\n\nCitations:\n" + for i, citation in enumerate(rag_response.citations[:self.max_citations], 1): + citations_text += f"{i}. {citation}\n" + response_text += citations_text + + return { + "text": response_text, + "channel_specific": { + "default": { + "supports_html": False, + "supports_attachments": False, + "plain_text_only": True + } + } + } + + except Exception as e: + logger.error(f"Error formatting for default: {e}") + return { + "text": rag_response.answer or "I'm sorry, I couldn't generate a response.", + "channel_specific": {} + } + + def _truncate_text(self, text: str, max_length: int) -> str: + """Truncate text to maximum length while preserving word boundaries.""" + if len(text) <= max_length: + return text + + truncated = text[:max_length] + last_space = truncated.rfind(' ') + + if last_space > max_length * 0.8: # If we can find a good break point + return truncated[:last_space] + "..." + else: + return truncated + "..." + + def _format_source_url(self, source: Dict[str, Any]) -> str: + """Format a source URL for display.""" + url = source.get("url", "") + title = source.get("title", "Unknown Source") + + if url: + return f"[{title}]({url})" + else: + return title \ No newline at end of file diff --git a/agents/adapters/teams_response_adapter.py b/agents/adapters/teams_response_adapter.py new file mode 100644 index 0000000000..adfb016d01 --- /dev/null +++ b/agents/adapters/teams_response_adapter.py @@ -0,0 +1,431 @@ +""" +Teams-specific response adapter for formatting responses for Microsoft Teams. +This adapter handles Teams-specific UI components and formatting. +""" + +import logging +from typing import Dict, Any, List, Optional + +from botbuilder.core import TurnContext, MessageFactory +from botbuilder.schema import Activity, ActivityTypes, Attachment, CardAction, ActionTypes, TextFormatTypes + +from services.rag_service import RAGResponse +from components.teams_components import TeamsComponents, TeamsCardConfig +from constants.teams_text import TeamsTextConstants +from models.citation import Citation, CitationSource, CitationProvider, resolve_citation_conflicts + + +logger = logging.getLogger(__name__) + + +class TeamsResponseAdapter: + """ + Teams-specific response adapter for formatting RAG responses. + """ + + def __init__(self, config: Optional[TeamsCardConfig] = None): + self.config = config or TeamsCardConfig() + self.teams_components = TeamsComponents() + logger.info("TeamsResponseAdapter initialized") + + def format_rag_response( + self, + turn_context: TurnContext, + rag_response: RAGResponse, + conversation_data: Optional[Dict[str, Any]] = None + ) -> Activity: + """ + Format a RAG response for Teams with adaptive cards. + """ + try: + # Create adaptive card response + card_json = self._create_rag_response_card(rag_response, conversation_data) + attachment = self.teams_components.create_attachment_from_card(card_json) + + # Create activity with attachment + activity = MessageFactory.attachment(attachment) + activity.text = rag_response.answer # Fallback text + + # Add suggested actions + activity.suggested_actions = self._create_suggested_actions() + + return activity + + except Exception as e: + logger.error(f"Error formatting Teams RAG response: {e}") + return MessageFactory.text(rag_response.answer) + + def format_text_response( + self, + turn_context: TurnContext, + text: str, + include_suggestions: bool = True + ) -> Activity: + """ + Format a simple text response for Teams. + """ + activity = MessageFactory.text(text) + + if include_suggestions: + activity.suggested_actions = self._create_suggested_actions() + + return activity + + def format_welcome_response(self, turn_context: TurnContext) -> Activity: + """ + Format a welcome response for Teams. + """ + try: + card_json = self.teams_components.create_welcome_card() + attachment = self.teams_components.create_attachment_from_card(card_json) + + activity = MessageFactory.attachment(attachment) + activity.text = f"Welcome to {TeamsTextConstants.get_bot_name()}! {TeamsTextConstants.get_bot_description()}" + + return activity + + except Exception as e: + logger.error(f"Error formatting welcome response: {e}") + return MessageFactory.text(TeamsTextConstants.format_welcome_fallback()) + + def format_help_response(self, turn_context: TurnContext) -> Activity: + """ + Format a help response for Teams. + """ + try: + card_json = self.teams_components.create_help_card() + attachment = self.teams_components.create_attachment_from_card(card_json) + + activity = MessageFactory.attachment(attachment) + activity.text = TeamsTextConstants.format_help_main_text() + + return activity + + except Exception as e: + logger.error(f"Error formatting help response: {e}") + return MessageFactory.text(TeamsTextConstants.format_help_fallback()) + + def format_error_response( + self, + turn_context: TurnContext, + error_message: str + ) -> Activity: + """ + Format an error response for Teams. + """ + try: + card_json = self.teams_components.create_error_card(error_message) + attachment = self.teams_components.create_attachment_from_card(card_json) + + activity = MessageFactory.attachment(attachment) + activity.text = f"Error: {error_message}" + + return activity + + except Exception as e: + logger.error(f"Error formatting error response: {e}") + return MessageFactory.text(f"Error: {error_message}") + + def format_loading_response(self, turn_context: TurnContext) -> Activity: + """ + Format a loading response for Teams. + """ + try: + card_json = self.teams_components.create_loading_card() + attachment = self.teams_components.create_attachment_from_card(card_json) + + activity = MessageFactory.attachment(attachment) + activity.text = "Processing your request..." + + return activity + + except Exception as e: + logger.error(f"Error formatting loading response: {e}") + return MessageFactory.text("Processing your request...") + + def format_file_upload_response( + self, + turn_context: TurnContext, + file_name: str, + file_type: str + ) -> Activity: + """ + Format a file upload response for Teams. + """ + try: + card_json = self.teams_components.create_file_upload_card(file_name, file_type) + attachment = self.teams_components.create_attachment_from_card(card_json) + + activity = MessageFactory.attachment(attachment) + activity.text = f"I've received your file: {file_name}" + + return activity + + except Exception as e: + logger.error(f"Error formatting file upload response: {e}") + return MessageFactory.text(f"I've received your file: {file_name}") + + def format_quick_actions_response(self, turn_context: TurnContext) -> Activity: + """ + Format a quick actions response for Teams. + """ + try: + card_json = self.teams_components.create_quick_actions_card() + attachment = self.teams_components.create_attachment_from_card(card_json) + + activity = MessageFactory.attachment(attachment) + activity.text = "Choose a quick action to get started:" + + return activity + + except Exception as e: + logger.error(f"Error formatting quick actions response: {e}") + return MessageFactory.text( + "Choose a quick action to get started:\n\n" + "• Search Documents\n" + "• Get Summary\n" + "• Ask Question\n" + "• Upload File" + ) + + def _create_rag_response_card( + self, + rag_response: RAGResponse, + conversation_data: Optional[Dict[str, Any]] = None + ) -> Dict[str, Any]: + """ + Create an adaptive card for RAG response. + """ + card_json = { + "type": "AdaptiveCard", + "version": "1.4", + "body": [ + { + "type": "Container", + "style": "emphasis", + "items": [ + { + "type": "TextBlock", + "text": "🤖 RAG Assistant", + "weight": "Bolder", + "size": "Medium", + "color": "Accent" + } + ] + }, + { + "type": "TextBlock", + "text": rag_response.answer, + "wrap": True, + "size": "Medium", + "spacing": "Medium" + } + ], + "actions": [ + { + "type": "Action.Submit", + "title": "💬 Ask Follow-up", + "data": { + "action": "follow_up", + "conversation_id": conversation_data.get("conversation_id", "") if isinstance(conversation_data, dict) else "" + }, + "style": "positive" + }, + { + "type": "Action.Submit", + "title": "🔍 Search Related", + "data": { + "action": "search_related", + "conversation_id": conversation_data.get("conversation_id", "") if isinstance(conversation_data, dict) else "" + }, + "style": "default" + }, + { + "type": "Action.Submit", + "title": "📋 Summarize", + "data": { + "action": "summarize", + "conversation_id": conversation_data.get("conversation_id", "") if isinstance(conversation_data, dict) else "" + }, + "style": "default" + } + ] + } + + # Add sources section if available and enabled + if self.config.show_sources and rag_response.sources: + sources_container = { + "type": "Container", + "style": "default", + "items": [ + { + "type": "TextBlock", + "text": "📚 Sources", + "weight": "Bolder", + "size": "Small", + "color": "Accent", + "spacing": "Medium" + } + ] + } + + for i, source in enumerate(rag_response.sources[:self.config.max_sources], 1): + if isinstance(source, dict): + source_text = source.get('title', 'Unknown Source') + source_url = source.get('url', '') + + if source_url: + sources_container["items"].append({ + "type": "TextBlock", + "text": f"{i}. [{source_text}]({source_url})", + "wrap": True, + "size": "Small", + "spacing": "Small" + }) + else: + sources_container["items"].append({ + "type": "TextBlock", + "text": f"{i}. {source_text}", + "wrap": True, + "size": "Small", + "spacing": "Small" + }) + else: + # Handle string sources + sources_container["items"].append({ + "type": "TextBlock", + "text": f"{i}. {source}", + "wrap": True, + "size": "Small", + "spacing": "Small" + }) + + card_json["body"].append(sources_container) + + # Add unified citations section if available and enabled + # Prefer unified_citations over legacy citations format + citations_to_display = [] + + if rag_response.unified_citations: + # Use unified citations format + # Resolve conflicts (prefer corpus over web) + resolved_citations = resolve_citation_conflicts( + rag_response.unified_citations, + prefer_corpus=True + ) + citations_to_display = resolved_citations[:self.config.max_citations] + elif self.config.show_citations and rag_response.citations: + # Fallback to legacy citations format + citations_to_display = rag_response.citations[:self.config.max_citations] + + if citations_to_display: + citations_container = { + "type": "Container", + "style": "default", + "items": [ + { + "type": "TextBlock", + "text": "🔗 Citations", + "weight": "Bolder", + "size": "Small", + "color": "Accent", + "spacing": "Medium" + } + ] + } + + for i, citation in enumerate(citations_to_display, 1): + if isinstance(citation, Citation): + # Unified citation format + citation_text = citation.title + if citation.url: + citation_text = f"[{citation.title}]({citation.url})" + + # Add source indicator + source_indicator = "📄" if citation.source == CitationSource.CORPUS else "🌐" + + citation_block = { + "type": "TextBlock", + "text": f"{i}. {source_indicator} {citation_text}", + "wrap": True, + "size": "Small", + "spacing": "Small" + } + + # Add snippet if available + if citation.snippet: + citation_block["text"] += f"\n {citation.snippet[:150]}..." if len(citation.snippet) > 150 else f"\n {citation.snippet}" + + citations_container["items"].append(citation_block) + else: + # Legacy string format + citations_container["items"].append({ + "type": "TextBlock", + "text": f"{i}. {citation}", + "wrap": True, + "size": "Small", + "spacing": "Small" + }) + + card_json["body"].append(citations_container) + + # Add thoughts section if available and enabled + if self.config.show_thoughts and rag_response.thoughts: + thoughts_container = { + "type": "Container", + "style": "default", + "items": [ + { + "type": "TextBlock", + "text": "💭 Process", + "weight": "Bolder", + "size": "Small", + "color": "Accent", + "spacing": "Medium" + } + ] + } + + for thought in rag_response.thoughts[:self.config.max_thoughts]: + thoughts_container["items"].append({ + "type": "TextBlock", + "text": f"• {thought.get('title', 'Step')}: {thought.get('description', '')}", + "wrap": True, + "size": "Small", + "spacing": "Small" + }) + + card_json["body"].append(thoughts_container) + + # Add token usage if available and enabled + if self.config.show_usage and rag_response.token_usage: + usage_container = { + "type": "Container", + "style": "default", + "items": [ + { + "type": "TextBlock", + "text": "📊 Usage", + "weight": "Bolder", + "size": "Small", + "color": "Accent", + "spacing": "Medium" + }, + { + "type": "TextBlock", + "text": f"Tokens: {rag_response.token_usage.get('total_tokens', 'N/A')} (Prompt: {rag_response.token_usage.get('prompt_tokens', 'N/A')}, Completion: {rag_response.token_usage.get('completion_tokens', 'N/A')})", + "wrap": True, + "size": "Small", + "spacing": "Small" + } + ] + } + card_json["body"].append(usage_container) + + return card_json + + def _create_suggested_actions(self) -> List[CardAction]: + """ + Create suggested actions for Teams. + """ + return self.teams_components.get_default_suggested_actions() \ No newline at end of file diff --git a/agents/agent_app.py b/agents/agent_app.py new file mode 100644 index 0000000000..899afdb993 --- /dev/null +++ b/agents/agent_app.py @@ -0,0 +1,285 @@ +""" +Main Microsoft 365 Agent Application. +This module contains the core agent application that integrates with Microsoft 365 Agents SDK. +""" + +import asyncio +import logging +from typing import Dict, Any, Optional +from dataclasses import dataclass + +from botbuilder.core import ( + ActivityHandler, + TurnContext, + MessageFactory, + ConversationState, + UserState, + MemoryStorage, +) +from botbuilder.schema import ( + Activity, + ActivityTypes, + ChannelAccount, + ConversationReference, + ResourceResponse, +) +from botbuilder.core import BotFrameworkAdapter, BotFrameworkAdapterSettings + +from config.agent_config import AgentConfig +from services.rag_service import RAGService +from services.auth_service import AuthService, UserClaims +from handlers.message_handler import MessageHandler +from handlers.teams_handler import TeamsHandler +from adapters.response_adapter import ResponseAdapter + + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +@dataclass +class ConversationData: + """Data stored for each conversation.""" + conversation_id: str + user_id: str + channel_id: str + message_count: int = 0 + last_activity: Optional[str] = None + + +class RAGAgent(ActivityHandler): + """ + Main RAG Agent that handles Microsoft 365 activities. + Integrates with existing RAG services while providing Microsoft 365 channel support. + """ + + def __init__( + self, + config: AgentConfig, + rag_service: RAGService, + auth_service: AuthService, + conversation_state: ConversationState, + user_state: UserState, + ): + super().__init__() + self.config = config + self.rag_service = rag_service + self.auth_service = auth_service + self.conversation_state = conversation_state + self.user_state = user_state + + # Initialize handlers + self.message_handler = MessageHandler(rag_service, auth_service) + self.teams_handler = TeamsHandler(rag_service, auth_service) + self.response_adapter = ResponseAdapter() + + # Accessor for conversation data + self.conversation_data_accessor = self.conversation_state.create_property("ConversationData") + self.user_data_accessor = self.user_state.create_property("UserData") + + async def on_message_activity(self, turn_context: TurnContext) -> None: + """Handle incoming message activities.""" + try: + # Get conversation and user data + conversation_data = await self.conversation_data_accessor.get(turn_context, ConversationData) + user_data = await self.user_data_accessor.get(turn_context, dict) + + # Initialize conversation data if needed + if not conversation_data: + conversation_data = ConversationData( + conversation_id=turn_context.activity.conversation.id, + user_id=turn_context.activity.from_property.id, + channel_id=turn_context.activity.channel_id, + ) + + # Update conversation data + conversation_data.message_count += 1 + conversation_data.last_activity = turn_context.activity.text + + # Get enhanced user authentication claims + # For test channels (webchat, emulator), skip Graph API calls to avoid tenant issues + is_test_channel = turn_context.activity.channel_id in ["emulator", "directline", "webchat"] + + if is_test_channel: + # For test channels: Use basic claims without Graph API calls + logger.info(f"Test channel detected ({turn_context.activity.channel_id}) - skipping Graph API calls") + basic_claims = await self.auth_service.get_user_claims(turn_context) + user_claims = type('UserClaims', (), { + 'user_id': basic_claims.get('oid', ''), + 'user_name': basic_claims.get('name', 'Unknown User'), + 'email': basic_claims.get('email', ''), + 'tenant_id': basic_claims.get('tenant_id', ''), + 'groups': [], + 'roles': [], + 'is_authenticated': True, # Allow for test channels + 'additional_claims': basic_claims + })() + else: + # For production channels: Use enhanced claims with Graph API + try: + user_claims = await self.auth_service.get_enhanced_user_claims(turn_context) + except Exception as e: + logger.warning(f"Error getting enhanced claims: {e}. Using basic claims.") + # Fallback for emulator/testing scenarios + basic_claims = await self.auth_service.get_user_claims(turn_context) + user_claims = type('UserClaims', (), { + 'user_id': basic_claims.get('oid', ''), + 'user_name': basic_claims.get('name', 'Unknown User'), + 'email': basic_claims.get('email', ''), + 'tenant_id': basic_claims.get('tenant_id', ''), + 'groups': [], + 'roles': [], + 'is_authenticated': True, # Allow for emulator/testing + 'additional_claims': basic_claims + })() + + auth_claims = { + "oid": user_claims.user_id, + "name": user_claims.user_name, + "email": user_claims.email, + "tenant_id": user_claims.tenant_id, + "groups": user_claims.groups, + "roles": user_claims.roles, + "is_authenticated": user_claims.is_authenticated, + "additional_claims": user_claims.additional_claims + } + + # For emulator/testing: Skip strict auth checks + # For production Teams: Enforce authentication + is_emulator = turn_context.activity.channel_id in ["emulator", "directline", "webchat"] + + if not is_emulator: + # Check user permissions for production channels + if not user_claims.is_authenticated: + await turn_context.send_activity( + MessageFactory.text("I'm sorry, I need to verify your identity before I can help you. Please ensure you're properly authenticated.") + ) + return + + # Check if user has basic read permission + has_read_permission = await self.auth_service.check_user_permission(user_claims, "read_documents") + if not has_read_permission: + await turn_context.send_activity( + MessageFactory.text("I'm sorry, you don't have permission to access the document search functionality. Please contact your administrator.") + ) + return + + # Process the message based on channel + if turn_context.activity.channel_id == "msteams": + response = await self.teams_handler.handle_message( + turn_context, conversation_data, user_data, auth_claims + ) + else: + response = await self.message_handler.handle_message( + turn_context, conversation_data, user_data, auth_claims + ) + + # Send response + if response: + await turn_context.send_activity(response) + + # Save state + await self.conversation_state.save_changes(turn_context) + await self.user_state.save_changes(turn_context) + + except Exception as e: + logger.error(f"Error handling message activity: {e}") + await turn_context.send_activity( + MessageFactory.text("I'm sorry, I encountered an error processing your request. Please try again.") + ) + + async def on_members_added_activity( + self, members_added: list[ChannelAccount], turn_context: TurnContext + ) -> None: + """Handle when members are added to the conversation.""" + for member in members_added: + if member.id != turn_context.activity.recipient.id: + welcome_message = ( + f"Welcome {member.name}! I'm your AI-powered document search assistant. " + f"I can help you find information from your documents and answer questions. " + f"Just ask me anything!" + ) + await turn_context.send_activity(MessageFactory.text(welcome_message)) + + async def on_typing_activity(self, turn_context: TurnContext) -> None: + """Handle typing indicators.""" + if self.config.enable_typing_indicator: + # Echo typing indicator back + typing_activity = Activity( + type=ActivityTypes.typing, + channel_id=turn_context.activity.channel_id, + conversation=turn_context.activity.conversation, + recipient=turn_context.activity.from_property, + ) + await turn_context.send_activity(typing_activity) + + async def on_end_of_conversation_activity(self, turn_context: TurnContext) -> None: + """Handle end of conversation.""" + logger.info(f"Conversation ended: {turn_context.activity.conversation.id}") + await self.conversation_state.clear(turn_context) + await self.user_state.clear(turn_context) + + +class AgentApplication: + """ + Main application class that sets up and runs the Microsoft 365 Agent. + """ + + def __init__(self, config: AgentConfig): + self.config = config + self.config.validate() + + # Initialize services + self.rag_service = RAGService(config) + self.auth_service = AuthService(config) + + # Initialize state management + memory_storage = MemoryStorage() + self.conversation_state = ConversationState(memory_storage) + self.user_state = UserState(memory_storage) + + # Initialize the agent + self.agent = RAGAgent( + config=config, + rag_service=self.rag_service, + auth_service=self.auth_service, + conversation_state=self.conversation_state, + user_state=self.user_state, + ) + + # Initialize adapter + self.adapter = self._create_adapter() + + def _create_adapter(self) -> BotFrameworkAdapter: + """Create the Bot Framework adapter.""" + # For emulator/testing: If app_id/app_password are empty, adapter will skip auth validation + # For production: Provide actual credentials + app_id = self.config.app_id if self.config.app_id else None + app_password = self.config.app_password if self.config.app_password else None + + settings = BotFrameworkAdapterSettings( + app_id=app_id, + app_password=app_password, + ) + + adapter = BotFrameworkAdapter(settings) + + # Add error handler + async def on_error(context: TurnContext, error: Exception) -> None: + logger.error(f"Error occurred: {error}") + await context.send_activity( + MessageFactory.text("I'm sorry, I encountered an error. Please try again.") + ) + + adapter.on_turn_error = on_error + + return adapter + + async def process_activity(self, activity: Activity) -> ResourceResponse: + """Process an incoming activity.""" + return await self.adapter.process_activity(activity, "", self.agent.on_turn) + + def get_adapter(self) -> BotFrameworkAdapter: + """Get the Bot Framework adapter.""" + return self.adapter \ No newline at end of file diff --git a/agents/components/teams_components.py b/agents/components/teams_components.py new file mode 100644 index 0000000000..955d0a1394 --- /dev/null +++ b/agents/components/teams_components.py @@ -0,0 +1,462 @@ +""" +Teams-specific UI components and utilities. +This module contains reusable Teams UI components for the agent. +""" + +import logging +from typing import Dict, Any, List, Optional +from dataclasses import dataclass + +from botbuilder.schema import Attachment, CardAction, ActionTypes +from services.rag_service import RAGResponse +from constants.teams_text import TeamsTextConstants + + +logger = logging.getLogger(__name__) + + +@dataclass +class TeamsCardConfig: + """Configuration for Teams adaptive cards.""" + show_sources: bool = True + show_citations: bool = True + show_thoughts: bool = False + show_usage: bool = False + max_sources: int = 3 + max_citations: int = 3 + max_thoughts: int = 2 + include_actions: bool = True + + +class TeamsComponents: + """ + Teams-specific UI components for the agent. + """ + + @staticmethod + def create_welcome_card() -> Dict[str, Any]: + """Create a welcome card for new users.""" + return { + "type": "AdaptiveCard", + "version": "1.4", + "body": [ + { + "type": "Container", + "style": "emphasis", + "items": [ + { + "type": "TextBlock", + "text": TeamsTextConstants.format_welcome_title(), + "weight": "Bolder", + "size": "Large", + "color": "Accent" + } + ] + }, + { + "type": "TextBlock", + "text": TeamsTextConstants.format_welcome_description(), + "wrap": True, + "size": "Medium", + "spacing": "Medium" + }, + { + "type": "Container", + "style": "default", + "items": [ + { + "type": "TextBlock", + "text": TeamsTextConstants.CAPABILITIES_TITLE, + "weight": "Bolder", + "size": "Medium", + "color": "Accent", + "spacing": "Medium" + } + ] + [ + { + "type": "TextBlock", + "text": capability, + "wrap": True, + "size": "Small", + "spacing": "Small" + } + for capability in TeamsTextConstants.CAPABILITIES + ] + }, + { + "type": "Container", + "style": "default", + "items": [ + { + "type": "TextBlock", + "text": TeamsTextConstants.USAGE_TITLE, + "weight": "Bolder", + "size": "Medium", + "color": "Accent", + "spacing": "Medium" + } + ] + [ + { + "type": "TextBlock", + "text": instruction, + "wrap": True, + "size": "Small", + "spacing": "Small" + } + for instruction in TeamsTextConstants.format_usage_instructions() + ] + } + ], + "actions": [ + { + "type": "Action.Submit", + "title": TeamsTextConstants.ACTION_GET_STARTED, + "data": { + "action": "get_started" + }, + "style": "positive" + }, + { + "type": "Action.Submit", + "title": TeamsTextConstants.ACTION_HELP, + "data": { + "action": "help" + } + } + ] + } + + @staticmethod + def create_help_card() -> Dict[str, Any]: + """Create a help card with usage instructions.""" + return { + "type": "AdaptiveCard", + "version": "1.4", + "body": [ + { + "type": "Container", + "style": "emphasis", + "items": [ + { + "type": "TextBlock", + "text": TeamsTextConstants.format_help_title(), + "weight": "Bolder", + "size": "Large", + "color": "Accent" + } + ] + }, + { + "type": "Container", + "style": "default", + "items": [ + { + "type": "TextBlock", + "text": TeamsTextConstants.STRUCTURAL_ANALYSIS_TITLE, + "weight": "Bolder", + "size": "Medium", + "color": "Accent", + "spacing": "Medium" + } + ] + [ + { + "type": "TextBlock", + "text": item, + "wrap": True, + "size": "Small", + "spacing": "Small" + } + for item in TeamsTextConstants.STRUCTURAL_ANALYSIS_ITEMS + ] + }, + { + "type": "Container", + "style": "default", + "items": [ + { + "type": "TextBlock", + "text": TeamsTextConstants.TECHNICAL_CHAT_TITLE, + "weight": "Bolder", + "size": "Medium", + "color": "Accent", + "spacing": "Medium" + } + ] + [ + { + "type": "TextBlock", + "text": item, + "wrap": True, + "size": "Small", + "spacing": "Small" + } + for item in TeamsTextConstants.TECHNICAL_CHAT_ITEMS + ] + }, + { + "type": "Container", + "style": "default", + "items": [ + { + "type": "TextBlock", + "text": TeamsTextConstants.EXAMPLE_QUESTIONS_TITLE, + "weight": "Bolder", + "size": "Medium", + "color": "Accent", + "spacing": "Medium" + } + ] + [ + { + "type": "TextBlock", + "text": question, + "wrap": True, + "size": "Small", + "spacing": "Small" + } + for question in TeamsTextConstants.EXAMPLE_QUESTIONS + ] + } + ], + "actions": [ + { + "type": "Action.Submit", + "title": TeamsTextConstants.ACTION_TRY_NOW, + "data": { + "action": "try_example" + }, + "style": "positive" + }, + { + "type": "Action.Submit", + "title": TeamsTextConstants.ACTION_UPLOAD_DRAWING, + "data": { + "action": "upload_document" + } + } + ] + } + + @staticmethod + def create_error_card(error_message: str) -> Dict[str, Any]: + """Create an error card for displaying errors.""" + return { + "type": "AdaptiveCard", + "version": "1.4", + "body": [ + { + "type": "Container", + "style": "attention", + "items": [ + { + "type": "TextBlock", + "text": "⚠️ Error", + "weight": "Bolder", + "size": "Medium", + "color": "Attention" + } + ] + }, + { + "type": "TextBlock", + "text": error_message, + "wrap": True, + "size": "Medium", + "spacing": "Medium" + } + ], + "actions": [ + { + "type": "Action.Submit", + "title": "🔄 Try Again", + "data": { + "action": "retry" + }, + "style": "positive" + }, + { + "type": "Action.Submit", + "title": TeamsTextConstants.ACTION_HELP, + "data": { + "action": "help" + } + } + ] + } + + @staticmethod + def create_loading_card() -> Dict[str, Any]: + """Create a loading card while processing requests.""" + return { + "type": "AdaptiveCard", + "version": "1.4", + "body": [ + { + "type": "Container", + "style": "default", + "items": [ + { + "type": "TextBlock", + "text": TeamsTextConstants.LOADING_TITLE, + "weight": "Bolder", + "size": "Medium", + "color": "Accent" + }, + { + "type": "TextBlock", + "text": TeamsTextConstants.LOADING_MESSAGE, + "wrap": True, + "size": "Small", + "spacing": "Medium" + } + ] + } + ] + } + + @staticmethod + def create_file_upload_card(file_name: str, file_type: str) -> Dict[str, Any]: + """Create a card for file upload confirmation.""" + return { + "type": "AdaptiveCard", + "version": "1.4", + "body": [ + { + "type": "Container", + "style": "emphasis", + "items": [ + { + "type": "TextBlock", + "text": TeamsTextConstants.FILE_UPLOAD_TITLE, + "weight": "Bolder", + "size": "Medium", + "color": "Accent" + } + ] + }, + { + "type": "TextBlock", + "text": TeamsTextConstants.format_file_upload_message(file_name), + "wrap": True, + "size": "Medium", + "spacing": "Medium" + }, + { + "type": "TextBlock", + "text": TeamsTextConstants.format_file_upload_type(file_type), + "wrap": True, + "size": "Small", + "spacing": "Small" + }, + { + "type": "TextBlock", + "text": TeamsTextConstants.FILE_UPLOAD_HELP, + "wrap": True, + "size": "Medium", + "spacing": "Medium" + } + ], + "actions": [ + { + "type": "Action.Submit", + "title": "🔍 Search Document", + "data": { + "action": "search_document", + "file_name": file_name + }, + "style": "positive" + }, + { + "type": "Action.Submit", + "title": "📋 Summarize Document", + "data": { + "action": "summarize_document", + "file_name": file_name + } + } + ] + } + + @staticmethod + def create_quick_actions_card() -> Dict[str, Any]: + """Create a card with quick action buttons.""" + return { + "type": "AdaptiveCard", + "version": "1.4", + "body": [ + { + "type": "Container", + "style": "emphasis", + "items": [ + { + "type": "TextBlock", + "text": TeamsTextConstants.QUICK_ACTIONS_TITLE, + "weight": "Bolder", + "size": "Medium", + "color": "Accent" + } + ] + }, + { + "type": "TextBlock", + "text": TeamsTextConstants.QUICK_ACTIONS_MESSAGE, + "wrap": True, + "size": "Medium", + "spacing": "Medium" + } + ], + "actions": [ + { + "type": "Action.Submit", + "title": "🔍 Search Documents", + "data": { + "action": "quick_search" + }, + "style": "positive" + }, + { + "type": "Action.Submit", + "title": "📋 Get Summary", + "data": { + "action": "quick_summary" + } + }, + { + "type": "Action.Submit", + "title": "❓ Ask Question", + "data": { + "action": "quick_question" + } + }, + { + "type": "Action.Submit", + "title": "📚 Upload File", + "data": { + "action": "quick_upload" + } + } + ] + } + + @staticmethod + def create_attachment_from_card(card_json: Dict[str, Any]) -> Attachment: + """Create an attachment from a card JSON.""" + return Attachment( + content_type="application/vnd.microsoft.card.adaptive", + content=card_json + ) + + @staticmethod + def create_suggested_actions(actions: List[str]) -> List[CardAction]: + """Create suggested actions for Teams.""" + return [ + CardAction( + type=ActionTypes.im_back, + title=action, + value=action + ) + for action in actions + ] + + @staticmethod + def get_default_suggested_actions() -> List[CardAction]: + """Get default suggested actions for Teams.""" + return TeamsComponents.create_suggested_actions(TeamsTextConstants.SUGGESTED_ACTIONS) \ No newline at end of file diff --git a/agents/config/agent_config.py b/agents/config/agent_config.py new file mode 100644 index 0000000000..c73f4d823f --- /dev/null +++ b/agents/config/agent_config.py @@ -0,0 +1,102 @@ +""" +Configuration for Microsoft 365 Agents SDK integration. +This module handles configuration for the agent application. +""" + +import os +from typing import Optional +from dataclasses import dataclass + + +@dataclass +class AgentConfig: + """Configuration for the Microsoft 365 Agent.""" + + # Bot Framework Configuration + app_id: str + app_password: str + + # Microsoft 365 Configuration + tenant_id: str + client_id: str + client_secret: str + + # Backend API Configuration + backend_url: str + + # Azure Services (reuse from existing app) + azure_openai_endpoint: str + azure_openai_api_key: str + azure_openai_deployment: str + azure_search_endpoint: str + azure_search_key: str + azure_search_index: str + + # Agent Settings + agent_name: str = "RAG Assistant" + agent_description: str = "AI-powered document search and chat assistant" + max_conversation_turns: int = 20 + enable_typing_indicator: bool = True + + # Channel Settings + enable_teams: bool = True + enable_copilot: bool = True + enable_web_chat: bool = True + + @classmethod + def from_environment(cls) -> "AgentConfig": + """Create configuration from environment variables.""" + return cls( + # Bot Framework + app_id=os.getenv("MICROSOFT_APP_ID", ""), + app_password=os.getenv("MICROSOFT_APP_PASSWORD", ""), + + # Microsoft 365 + tenant_id=os.getenv("AZURE_TENANT_ID", ""), + client_id=os.getenv("AZURE_CLIENT_ID", ""), + client_secret=os.getenv("AZURE_CLIENT_SECRET", ""), + + # Backend API + backend_url=os.getenv("BACKEND_URL", "http://localhost:50505"), + + # Azure Services + azure_openai_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT", ""), + azure_openai_api_key=os.getenv("AZURE_OPENAI_API_KEY", ""), + azure_openai_deployment=os.getenv("AZURE_OPENAI_CHATGPT_DEPLOYMENT", ""), + azure_search_endpoint=os.getenv("AZURE_SEARCH_ENDPOINT", ""), + azure_search_key=os.getenv("AZURE_SEARCH_KEY", ""), + azure_search_index=os.getenv("AZURE_SEARCH_INDEX", ""), + + # Agent Settings + agent_name=os.getenv("AGENT_NAME", "RAG Assistant"), + agent_description=os.getenv("AGENT_DESCRIPTION", "AI-powered document search and chat assistant"), + max_conversation_turns=int(os.getenv("MAX_CONVERSATION_TURNS", "20")), + enable_typing_indicator=os.getenv("ENABLE_TYPING_INDICATOR", "true").lower() == "true", + + # Channel Settings + enable_teams=os.getenv("ENABLE_TEAMS", "true").lower() == "true", + enable_copilot=os.getenv("ENABLE_COPILOT", "true").lower() == "true", + enable_web_chat=os.getenv("ENABLE_WEB_CHAT", "true").lower() == "true", + ) + + def validate(self) -> None: + """Validate configuration.""" + # For production: require all fields + # For emulator/testing: app_id/app_password can be empty (adapter will skip auth) + required_fields = [ + "tenant_id", "client_id", "client_secret", "backend_url" + ] + + missing_fields = [] + for field in required_fields: + if not getattr(self, field): + missing_fields.append(field) + + # Warn if app_id/password missing (needed for production, optional for emulator) + if not self.app_id or not self.app_password: + import logging + logger = logging.getLogger(__name__) + logger.warning("MICROSOFT_APP_ID or MICROSOFT_APP_PASSWORD not set - emulator/testing mode (auth will be skipped)") + + if missing_fields: + raise ValueError(f"Missing required configuration fields: {', '.join(missing_fields)}") \ No newline at end of file diff --git a/agents/constants/teams_text.py b/agents/constants/teams_text.py new file mode 100644 index 0000000000..6df5545b16 --- /dev/null +++ b/agents/constants/teams_text.py @@ -0,0 +1,197 @@ +""" +Constants for Teams UI text content. +This module contains all text strings used in Teams UI components and responses. +""" + +import os +from typing import Dict, List + + +class TeamsTextConstants: + """Constants for Teams UI text content.""" + + # Bot configuration + DEFAULT_BOT_NAME = "Structural Engineering Assistant" + DEFAULT_BOT_DESCRIPTION = "AI-powered structural engineering document search and analysis assistant" + + # Welcome messages + WELCOME_TITLE = "🏗️ Welcome to {bot_name}" + WELCOME_DESCRIPTION = "{bot_description}. I can help you analyze structural engineering documents, answer technical questions, and provide insights from your project files." + WELCOME_FALLBACK = "Welcome to {bot_name}! {bot_description}. I can help you analyze structural engineering documents, answer technical questions, and provide insights from your project files." + + # Help messages + HELP_TITLE = "❓ {bot_name} Help" + HELP_MAIN_TEXT = "Here's how to use {bot_name}:" + HELP_FALLBACK = "Here's how to use {bot_name}:\n\n• Mention me with @{bot_name}\n• Upload structural drawings, specs, or reports\n• Ask technical questions about your projects\n• Use the buttons in my responses for quick actions" + + # Capabilities + CAPABILITIES_TITLE = "🔧 What I can do:" + CAPABILITIES = [ + "• Analyze structural drawings and specifications", + "• Answer questions about building codes and standards", + "• Review calculations and design reports", + "• Provide technical insights and recommendations", + "• Help with material specifications and load calculations" + ] + + # Usage instructions + USAGE_TITLE = "💡 How to use:" + USAGE_INSTRUCTIONS = [ + "• Mention me with @{bot_name}", + "• Upload structural drawings, specs, or reports", + "• Ask technical questions about your projects" + ] + + # Help sections + STRUCTURAL_ANALYSIS_TITLE = "📐 Structural Analysis" + STRUCTURAL_ANALYSIS_ITEMS = [ + "• Analyze structural drawings and specifications", + "• Review load calculations and design reports", + "• Check compliance with building codes" + ] + + TECHNICAL_CHAT_TITLE = "💬 Technical Chat" + TECHNICAL_CHAT_ITEMS = [ + "• Ask questions about structural engineering concepts", + "• Get explanations of design principles", + "• Request material and code recommendations" + ] + + EXAMPLE_QUESTIONS_TITLE = "🔍 Example Questions" + EXAMPLE_QUESTIONS = [ + "• 'What are the load requirements for this beam design?'", + "• 'Can you review this foundation calculation?'", + "• 'What building code applies to this steel structure?'" + ] + + # Suggested actions + SUGGESTED_ACTIONS = [ + "🔍 Analyze Drawing", + "📐 Review Calculation", + "❓ Ask Technical Question", + "📋 Upload Specification", + "❓ Help" + ] + + # Action button labels + ACTION_GET_STARTED = "🚀 Get Started" + ACTION_HELP = "❓ Help" + ACTION_TRY_NOW = "🚀 Try It Now" + ACTION_UPLOAD_DRAWING = "📐 Upload Drawing" + ACTION_ASK_FOLLOW_UP = "💬 Ask Follow-up" + ACTION_SEARCH_RELATED = "🔍 Search Related" + ACTION_SUMMARIZE = "📋 Summarize" + + # Error messages + ERROR_PROCESSING_REQUEST = "I'm sorry, I encountered an error processing your request. Please try again." + ERROR_ADAPTIVE_CARD_ACTION = "I encountered an error processing your action. Please try asking me a question directly." + ERROR_WELCOME_FORMATTING = "Error formatting welcome response" + ERROR_HELP_FORMATTING = "Error formatting help response" + + # Follow-up action responses + FOLLOW_UP_RESPONSE = """I'd be happy to provide more details! What specific aspect would you like me to elaborate on? You can ask me to: + +• Explain any part in more detail +• Provide examples +• Compare different options +• Answer related questions + +Just type your question and I'll help you out!""" + + SEARCH_RELATED_RESPONSE = """I can help you find more information about this topic! Try asking me: + +• 'What are the requirements for...?' +• 'How do I apply for...?' +• 'What are the steps to...?' +• 'Tell me more about...' + +Or just describe what you're looking for and I'll search through the documents for you!""" + + SUMMARIZE_RESPONSE = """I can help you summarize information! You can ask me to: + +• 'Summarize the key points' +• 'Give me a brief overview' +• 'What are the main takeaways?' +• 'Create a bullet point summary' + +Just let me know what you'd like me to summarize!""" + + # File upload messages + FILE_UPLOAD_TITLE = "📎 File Uploaded" + FILE_UPLOAD_MESSAGE = "I've received your file: **{file_name}**" + FILE_UPLOAD_TYPE = "File type: {file_type}" + FILE_UPLOAD_HELP = "I can help you search through this document and answer questions about its content. What would you like to know?" + + # Loading messages + LOADING_TITLE = "🔄 Processing your request..." + LOADING_MESSAGE = "Please wait while I search through your documents and generate a response." + + # Quick actions + QUICK_ACTIONS_TITLE = "⚡ Quick Actions" + QUICK_ACTIONS_MESSAGE = "Choose a quick action to get started:" + + # Mention reminder + MENTION_REMINDER = """👋 Hi! I'm your AI assistant. To ask me a question, please mention me using @{bot_name} or type your question directly.""" + + @classmethod + def get_bot_name(cls) -> str: + """Get bot name from environment or default.""" + return os.getenv("AGENT_NAME", cls.DEFAULT_BOT_NAME) + + @classmethod + def get_bot_description(cls) -> str: + """Get bot description from environment or default.""" + return os.getenv("AGENT_DESCRIPTION", cls.DEFAULT_BOT_DESCRIPTION) + + @classmethod + def format_welcome_title(cls) -> str: + """Format welcome title with bot name.""" + return cls.WELCOME_TITLE.format(bot_name=cls.get_bot_name()) + + @classmethod + def format_welcome_description(cls) -> str: + """Format welcome description with bot description.""" + return cls.WELCOME_DESCRIPTION.format(bot_description=cls.get_bot_description()) + + @classmethod + def format_welcome_fallback(cls) -> str: + """Format welcome fallback message.""" + return cls.WELCOME_FALLBACK.format( + bot_name=cls.get_bot_name(), + bot_description=cls.get_bot_description() + ) + + @classmethod + def format_help_title(cls) -> str: + """Format help title with bot name.""" + return cls.HELP_TITLE.format(bot_name=cls.get_bot_name()) + + @classmethod + def format_help_main_text(cls) -> str: + """Format help main text with bot name.""" + return cls.HELP_MAIN_TEXT.format(bot_name=cls.get_bot_name()) + + @classmethod + def format_help_fallback(cls) -> str: + """Format help fallback message.""" + return cls.HELP_FALLBACK.format(bot_name=cls.get_bot_name()) + + @classmethod + def format_usage_instructions(cls) -> List[str]: + """Format usage instructions with bot name.""" + return [instruction.format(bot_name=cls.get_bot_name()) for instruction in cls.USAGE_INSTRUCTIONS] + + @classmethod + def format_mention_reminder(cls) -> str: + """Format mention reminder with bot name.""" + return cls.MENTION_REMINDER.format(bot_name=cls.get_bot_name()) + + @classmethod + def format_file_upload_message(cls, file_name: str) -> str: + """Format file upload message with file name.""" + return cls.FILE_UPLOAD_MESSAGE.format(file_name=file_name) + + @classmethod + def format_file_upload_type(cls, file_type: str) -> str: + """Format file upload type with file type.""" + return cls.FILE_UPLOAD_TYPE.format(file_type=file_type) \ No newline at end of file diff --git a/agents/copilot-plugin-manifest.json b/agents/copilot-plugin-manifest.json new file mode 100644 index 0000000000..d32d06c5ee --- /dev/null +++ b/agents/copilot-plugin-manifest.json @@ -0,0 +1,133 @@ +{ + "$schema": "https://schemas.microsoft.com/copilot/plugin/v1.0/plugin.schema.json", + "version": "1.0.0", + "name": "ai-master-engineer-rag", + "displayName": "AI Master Engineer RAG", + "description": "Search and query organizational documents using RAG (Retrieval-Augmented Generation). Get accurate, cited answers from your knowledge base.", + "publisher": "Actual Reality", + "iconUrl": "https://ai-master-engineer-agents-a7gra3a8g7emfbf0.canadacentral-01.azurewebsites.net/icons/copilot-icon.png", + "capabilities": [ + "search", + "answer", + "citation" + ], + "authentication": { + "type": "microsoftEntraID", + "clientId": "74d6d3c9-fe0f-40d2-80d1-9b636235e6a7", + "authorizationUrl": "https://login.microsoftonline.com/b998106b-07ef-4d03-bca9-128c816631e6/oauth2/v2.0/authorize", + "tokenUrl": "https://login.microsoftonline.com/b998106b-07ef-4d03-bca9-128c816631e6/oauth2/v2.0/token", + "scopes": [ + "User.Read", + "offline_access" + ] + }, + "endpoints": { + "search": { + "url": "https://ai-master-engineer-agents-a7gra3a8g7emfbf0.canadacentral-01.azurewebsites.net/api/copilot/search", + "method": "POST", + "description": "Search documents in the knowledge base" + }, + "query": { + "url": "https://ai-master-engineer-agents-a7gra3a8g7emfbf0.canadacentral-01.azurewebsites.net/api/copilot/query", + "method": "POST", + "description": "Query documents and get AI-generated answers with citations" + }, + "health": { + "url": "https://ai-master-engineer-agents-a7gra3a8g7emfbf0.canadacentral-01.azurewebsites.net/api/copilot/health", + "method": "GET", + "description": "Health check endpoint" + } + }, + "parameters": { + "search": { + "query": { + "type": "string", + "required": true, + "description": "Search query string" + }, + "maxResults": { + "type": "number", + "required": false, + "default": 5, + "description": "Maximum number of results to return" + } + }, + "query": { + "message": { + "type": "string", + "required": true, + "description": "User question or query" + }, + "conversationHistory": { + "type": "array", + "required": false, + "description": "Previous conversation messages for context" + } + } + }, + "responses": { + "search": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "url": { + "type": "string" + }, + "snippet": { + "type": "string" + }, + "source": { + "type": "string", + "enum": ["corpus", "web"] + } + } + } + }, + "totalCount": { + "type": "number" + } + } + }, + "query": { + "type": "object", + "properties": { + "answer": { + "type": "string", + "description": "AI-generated answer" + }, + "citations": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "url": { + "type": "string" + }, + "snippet": { + "type": "string" + }, + "source": { + "type": "string" + } + } + } + }, + "thoughts": { + "type": "array", + "description": "Processing steps (optional)" + } + } + } + } +} + diff --git a/agents/copilot-plugin-manifest.template.json b/agents/copilot-plugin-manifest.template.json new file mode 100644 index 0000000000..2656dd2b87 --- /dev/null +++ b/agents/copilot-plugin-manifest.template.json @@ -0,0 +1,137 @@ +{ + "$schema": "https://schemas.microsoft.com/copilot/plugin/v1.0/plugin.schema.json", + "version": "1.0.0", + "name": "ai-master-engineer-rag", + "displayName": "AI Master Engineer RAG", + "description": "Search and query organizational documents using RAG (Retrieval-Augmented Generation). Get accurate, cited answers from your knowledge base.", + "publisher": "Actual Reality", + "iconUrl": "{{PRODUCTION_URL}}/icons/copilot-icon.png", + "capabilities": [ + "search", + "answer", + "citation" + ], + "authentication": { + "type": "microsoftEntraID", + "clientId": "74d6d3c9-fe0f-40d2-80d1-9b636235e6a7", + "authorizationUrl": "https://login.microsoftonline.com/b998106b-07ef-4d03-bca9-128c816631e6/oauth2/v2.0/authorize", + "tokenUrl": "https://login.microsoftonline.com/b998106b-07ef-4d03-bca9-128c816631e6/oauth2/v2.0/token", + "scopes": [ + "User.Read", + "offline_access" + ] + }, + "endpoints": { + "search": { + "url": "{{PRODUCTION_URL}}/api/copilot/search", + "method": "POST", + "description": "Search documents in the knowledge base" + }, + "query": { + "url": "{{PRODUCTION_URL}}/api/copilot/query", + "method": "POST", + "description": "Query documents and get AI-generated answers with citations" + }, + "health": { + "url": "{{PRODUCTION_URL}}/api/copilot/health", + "method": "GET", + "description": "Health check endpoint" + } + }, + "parameters": { + "search": { + "query": { + "type": "string", + "required": true, + "description": "Search query string" + }, + "maxResults": { + "type": "number", + "required": false, + "default": 5, + "description": "Maximum number of results to return" + } + }, + "query": { + "message": { + "type": "string", + "required": true, + "description": "User question or query" + }, + "conversationHistory": { + "type": "array", + "required": false, + "description": "Previous conversation messages for context" + } + } + }, + "responses": { + "search": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "url": { + "type": "string" + }, + "snippet": { + "type": "string" + }, + "source": { + "type": "string", + "enum": ["corpus", "web"] + } + } + } + }, + "totalCount": { + "type": "number" + } + } + }, + "query": { + "type": "object", + "properties": { + "answer": { + "type": "string", + "description": "AI-generated answer" + }, + "citations": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "url": { + "type": "string" + }, + "snippet": { + "type": "string" + }, + "source": { + "type": "string" + } + } + } + }, + "thoughts": { + "type": "array", + "description": "Processing steps (optional)" + } + } + } + } +} + + + + + diff --git a/agents/deploy-to-azure.ps1 b/agents/deploy-to-azure.ps1 new file mode 100644 index 0000000000..99e099bb0b --- /dev/null +++ b/agents/deploy-to-azure.ps1 @@ -0,0 +1,135 @@ +# PowerShell script to help deploy Agents service to Azure +# Run this from the agents/ directory + +Write-Host "===========================================" +Write-Host "Azure Deployment Helper for Agents Service" +Write-Host "===========================================" +Write-Host "" + +# Check if Azure CLI is installed +try { + $azVersion = az version + Write-Host "[OK] Azure CLI is installed" -ForegroundColor Green +} catch { + Write-Host "[ERROR] Azure CLI is not installed" -ForegroundColor Red + Write-Host "Install from: https://aka.ms/installazurecliwindows" -ForegroundColor Yellow + exit 1 +} + +# Check if logged in +try { + $account = az account show 2>$null + if ($account) { + Write-Host "[OK] Logged into Azure" -ForegroundColor Green + } else { + Write-Host "[INFO] Not logged in. Logging in..." -ForegroundColor Yellow + az login + } +} catch { + Write-Host "[INFO] Logging into Azure..." -ForegroundColor Yellow + az login +} + +Write-Host "" +Write-Host "Configuration:" -ForegroundColor Cyan + +# Get app name +$appName = Read-Host "Enter your App Service name (e.g., ai-master-engineer-agents)" +if ([string]::IsNullOrWhiteSpace($appName)) { + Write-Host "[ERROR] App name is required" -ForegroundColor Red + exit 1 +} + +# Get resource group +$resourceGroup = Read-Host "Enter resource group name (or 'new' to create one)" +if ($resourceGroup -eq "new") { + $resourceGroup = "$appName-rg" + Write-Host "[INFO] Will create resource group: $resourceGroup" -ForegroundColor Yellow +} + +# Get region +$region = Read-Host "Enter region (e.g., eastus, westus2) [default: eastus]" +if ([string]::IsNullOrWhiteSpace($region)) { + $region = "eastus" +} + +Write-Host "" +Write-Host "Creating App Service..." -ForegroundColor Cyan + +# Create resource group if needed +if ($resourceGroup -eq "$appName-rg") { + Write-Host "Creating resource group: $resourceGroup" -ForegroundColor Yellow + az group create --name $resourceGroup --location $region +} + +# Create App Service +Write-Host "Creating App Service: $appName" -ForegroundColor Yellow +az webapp create ` + --resource-group $resourceGroup ` + --name $appName ` + --runtime "PYTHON:3.11" ` + --plan "$appName-plan" ` + --location $region + +if ($LASTEXITCODE -ne 0) { + Write-Host "[ERROR] Failed to create App Service" -ForegroundColor Red + exit 1 +} + +Write-Host "[OK] App Service created" -ForegroundColor Green + +# Get the URL +$appUrl = "https://$appName.azurewebsites.net" +Write-Host "" +Write-Host "Your App Service URL: $appUrl" -ForegroundColor Green + +Write-Host "" +Write-Host "Next Steps:" -ForegroundColor Cyan +Write-Host "1. Configure App Settings (environment variables)" -ForegroundColor Yellow +Write-Host "2. Deploy code (GitHub, VS Code, or Azure CLI)" -ForegroundColor Yellow +Write-Host "3. Update copilot-plugin-manifest.json with: $appUrl" -ForegroundColor Yellow + +Write-Host "" +Write-Host "Configure App Settings now? (y/n)" -ForegroundColor Cyan +$configure = Read-Host +if ($configure -eq "y" -or $configure -eq "Y") { + Write-Host "" + Write-Host "Open Azure Portal to configure:" -ForegroundColor Yellow + Write-Host "https://portal.azure.com/#@/resource/subscriptions/$(az account show --query id -o tsv)/resourceGroups/$resourceGroup/providers/Microsoft.Web/sites/$appName/configuration" -ForegroundColor Cyan + + Write-Host "" + Write-Host "Add these settings (from agents/.env):" -ForegroundColor Yellow + Write-Host "- MICROSOFT_APP_ID" -ForegroundColor White + Write-Host "- MICROSOFT_APP_PASSWORD" -ForegroundColor White + Write-Host "- AZURE_TENANT_ID" -ForegroundColor White + Write-Host "- AZURE_CLIENT_ID" -ForegroundColor White + Write-Host "- AZURE_CLIENT_SECRET" -ForegroundColor White + Write-Host "- BACKEND_URL" -ForegroundColor White +} + +Write-Host "" +Write-Host "Deploy code now? (y/n)" -ForegroundColor Cyan +$deploy = Read-Host +if ($deploy -eq "y" -or $deploy -eq "Y") { + Write-Host "" + Write-Host "Deploying via Azure CLI..." -ForegroundColor Yellow + az webapp up --name $appName --resource-group $resourceGroup + + if ($LASTEXITCODE -eq 0) { + Write-Host "[OK] Deployment complete!" -ForegroundColor Green + Write-Host "" + Write-Host "Test your deployment:" -ForegroundColor Cyan + Write-Host "curl $appUrl/api/copilot/health" -ForegroundColor White + } +} + +Write-Host "" +Write-Host "===========================================" +Write-Host "Don't forget to update copilot-plugin-manifest.json" -ForegroundColor Yellow +Write-Host "Replace 'your-domain.com' with: $appUrl" -ForegroundColor Yellow +Write-Host "===========================================" + + + + + diff --git a/agents/handlers/copilot_handler.py b/agents/handlers/copilot_handler.py new file mode 100644 index 0000000000..da1658d471 --- /dev/null +++ b/agents/handlers/copilot_handler.py @@ -0,0 +1,269 @@ +""" +Copilot-specific handler for Microsoft 365 Copilot integration. +This handler processes Copilot plugin requests and formats responses for Copilot. +""" + +import logging +from typing import Dict, Any, List, Optional +from dataclasses import dataclass + +from botbuilder.core import TurnContext +from botbuilder.schema import Activity, ActivityTypes + +from services.rag_service import RAGService, RAGRequest, RAGResponse +from services.auth_service import AuthService +from models.citation import Citation, CitationSource, resolve_citation_conflicts + + +logger = logging.getLogger(__name__) + + +@dataclass +class CopilotRequest: + """Request from Copilot plugin.""" + query: str + conversation_history: Optional[List[Dict[str, str]]] = None + max_results: Optional[int] = 5 + context: Optional[Dict[str, Any]] = None + + +@dataclass +class CopilotResponse: + """Response formatted for Copilot.""" + answer: str + citations: List[Dict[str, Any]] + sources: List[Dict[str, Any]] + metadata: Optional[Dict[str, Any]] = None + + +class CopilotHandler: + """ + Handler for Microsoft Copilot plugin requests. + Formats responses specifically for Copilot consumption. + """ + + def __init__(self, rag_service: RAGService, auth_service: AuthService): + self.rag_service = rag_service + self.auth_service = auth_service + logger.info("CopilotHandler initialized") + + async def handle_search_request( + self, + request: CopilotRequest, + auth_claims: Dict[str, Any] + ) -> Dict[str, Any]: + """ + Handle a search request from Copilot. + Returns document search results. + """ + try: + # Create RAG request for search + rag_request = RAGRequest( + message=request.query, + conversation_history=request.conversation_history or [], + user_id=auth_claims.get("oid", ""), + channel_id="copilot", + context={ + "auth_claims": auth_claims, + "copilot_request": True, + "max_results": request.max_results + } + ) + + # Process with RAG service (this will search backend) + rag_response = await self.rag_service.process_query(rag_request) + + # Format for Copilot + results = [] + + # Convert sources to Copilot format + for source in rag_response.sources[:request.max_results or 5]: + if isinstance(source, dict): + results.append({ + "title": source.get("title", source.get("sourcefile", "Document")), + "url": source.get("url", source.get("sourcepage", "")), + "snippet": source.get("content", source.get("snippet", "")), + "source": "corpus" + }) + + # Add citations if available + if rag_response.unified_citations: + for citation in rag_response.unified_citations[:request.max_results or 5]: + if citation.source == CitationSource.CORPUS: + results.append({ + "title": citation.title, + "url": citation.url, + "snippet": citation.snippet, + "source": "corpus" + }) + + return { + "results": results, + "totalCount": len(results), + "query": request.query + } + + except Exception as e: + logger.error(f"Error handling Copilot search request: {e}", exc_info=True) + return { + "results": [], + "totalCount": 0, + "error": str(e) + } + + async def handle_query_request( + self, + request: CopilotRequest, + auth_claims: Dict[str, Any] + ) -> Dict[str, Any]: + """ + Handle a query request from Copilot. + Returns AI-generated answer with citations. + """ + try: + # Create RAG request + rag_request = RAGRequest( + message=request.query, + conversation_history=request.conversation_history or [], + user_id=auth_claims.get("oid", ""), + channel_id="copilot", + context={ + "auth_claims": auth_claims, + "copilot_request": True + } + ) + + # Process with RAG service + rag_response = await self.rag_service.process_query(rag_request) + + # Format citations for Copilot + citations = [] + + # Use unified citations if available + if rag_response.unified_citations: + resolved_citations = resolve_citation_conflicts( + rag_response.unified_citations, + prefer_corpus=True + ) + + for citation in resolved_citations: + citations.append({ + "title": citation.title, + "url": citation.url, + "snippet": citation.snippet[:200] if citation.snippet else "", + "source": citation.source.value, + "provider": citation.provider.value + }) + else: + # Fallback to legacy citations + for citation_str in rag_response.citations: + if citation_str: + citations.append({ + "title": citation_str[:100], + "url": "", + "snippet": citation_str, + "source": "unknown" + }) + + # Format response for Copilot + response = { + "answer": rag_response.answer, + "citations": citations + } + + # Add thoughts if available and requested + if rag_response.thoughts: + response["thoughts"] = [ + { + "title": thought.get("title", ""), + "description": thought.get("description", "") + } + for thought in rag_response.thoughts + ] + + # Add metadata + if rag_response.token_usage: + response["metadata"] = { + "token_usage": rag_response.token_usage, + "model_info": rag_response.model_info + } + + return response + + except Exception as e: + logger.error(f"Error handling Copilot query request: {e}", exc_info=True) + return { + "answer": f"I encountered an error processing your request: {str(e)}", + "citations": [], + "error": str(e) + } + + async def handle_activity( + self, + turn_context: TurnContext, + auth_claims: Dict[str, Any] + ) -> Optional[Activity]: + """ + Handle incoming Copilot activity. + Routes to appropriate handler based on activity type. + """ + try: + # Extract request from activity + activity = turn_context.activity + + if activity.type != ActivityTypes.message: + return None + + # Check if this is a Copilot request + channel_data = activity.channel_data or {} + if channel_data.get("channelId") != "copilot": + return None + + # Parse request from activity value or text + request_data = {} + if activity.value: + request_data = activity.value + elif activity.text: + # Simple text query + request_data = { + "query": activity.text, + "type": "query" + } + else: + return None + + request_type = request_data.get("type", "query") + + # Create Copilot request + copilot_request = CopilotRequest( + query=request_data.get("query", request_data.get("message", "")), + conversation_history=request_data.get("conversationHistory", []), + max_results=request_data.get("maxResults", 5), + context=request_data.get("context", {}) + ) + + # Route to appropriate handler + if request_type == "search": + result = await self.handle_search_request(copilot_request, auth_claims) + else: + # Default to query + result = await self.handle_query_request(copilot_request, auth_claims) + + # Return formatted response + response_activity = Activity( + type=ActivityTypes.message, + channel_id="copilot", + text=str(result.get("answer", result)) + ) + response_activity.value = result + + return response_activity + + except Exception as e: + logger.error(f"Error handling Copilot activity: {e}", exc_info=True) + return None + + + + + diff --git a/agents/handlers/message_handler.py b/agents/handlers/message_handler.py new file mode 100644 index 0000000000..0275944d6e --- /dev/null +++ b/agents/handlers/message_handler.py @@ -0,0 +1,216 @@ +""" +Message Handler for Microsoft 365 Agent. +This handler processes incoming messages and generates responses. +""" + +import logging +from typing import Dict, Any, List, Optional +from dataclasses import dataclass + +from botbuilder.core import TurnContext, MessageFactory +from botbuilder.schema import Activity + +from services.rag_service import RAGService, RAGRequest, RAGResponse +from services.auth_service import AuthService +from adapters.response_adapter import ResponseAdapter + + +logger = logging.getLogger(__name__) + + +@dataclass +class ConversationData: + """Data stored for each conversation.""" + conversation_id: str + user_id: str + channel_id: str + message_count: int = 0 + last_activity: Optional[str] = None + + +class MessageHandler: + """ + Handler for processing incoming messages and generating responses. + This handler works with the RAG service to provide intelligent responses. + """ + + def __init__(self, rag_service: RAGService, auth_service: AuthService): + self.rag_service = rag_service + self.auth_service = auth_service + self.response_adapter = ResponseAdapter() + + async def handle_message( + self, + turn_context: TurnContext, + conversation_data: ConversationData, + user_data: Dict[str, Any], + auth_claims: Dict[str, Any] + ) -> Optional[Activity]: + """ + Handle an incoming message and generate a response. + """ + try: + # Extract message text + message_text = turn_context.activity.text or "" + + if not message_text.strip(): + return MessageFactory.text("I didn't receive any message. Please try again.") + + # Get conversation history from user data + conversation_history = user_data.get("conversation_history", []) + + # Create RAG request + rag_request = RAGRequest( + message=message_text, + conversation_history=conversation_history, + user_id=conversation_data.user_id, + channel_id=conversation_data.channel_id, + context={ + "auth_claims": auth_claims, + "conversation_id": conversation_data.conversation_id, + "message_count": conversation_data.message_count + } + ) + + # Process the request with RAG service + rag_response = await self.rag_service.process_query(rag_request) + + # Update conversation history + conversation_history.append({ + "role": "user", + "content": message_text + }) + conversation_history.append({ + "role": "assistant", + "content": rag_response.answer + }) + + # Keep only the last 10 exchanges to manage context length + if len(conversation_history) > 20: # 10 user + 10 assistant messages + conversation_history = conversation_history[-20:] + + user_data["conversation_history"] = conversation_history + + # Format response for the channel + response_activity = await self._format_response( + turn_context, rag_response, conversation_data + ) + + return response_activity + + except Exception as e: + logger.error(f"Error handling message: {e}") + return MessageFactory.text( + "I'm sorry, I encountered an error processing your request. Please try again." + ) + + async def _format_response( + self, + turn_context: TurnContext, + rag_response: RAGResponse, + conversation_data: ConversationData + ) -> Activity: + """ + Format the RAG response for the specific channel. + """ + try: + # Use the response adapter to format the response + formatted_response = await self.response_adapter.format_response( + rag_response, turn_context.activity.channel_id + ) + + # Create the activity + activity = MessageFactory.text(formatted_response["text"]) + + # Add additional properties if available + if "attachments" in formatted_response: + activity.attachments = formatted_response["attachments"] + + if "suggested_actions" in formatted_response: + activity.suggested_actions = formatted_response["suggested_actions"] + + return activity + + except Exception as e: + logger.error(f"Error formatting response: {e}") + return MessageFactory.text(rag_response.answer) + + async def handle_typing_indicator( + self, + turn_context: TurnContext, + conversation_data: ConversationData + ) -> None: + """ + Handle typing indicators. + """ + try: + # Send typing indicator back + typing_activity = Activity( + type="typing", + channel_id=turn_context.activity.channel_id, + conversation=turn_context.activity.conversation, + recipient=turn_context.activity.from_property, + ) + await turn_context.send_activity(typing_activity) + + except Exception as e: + logger.error(f"Error handling typing indicator: {e}") + + async def handle_help_request( + self, + turn_context: TurnContext, + conversation_data: ConversationData + ) -> Activity: + """ + Handle help requests. + """ + help_text = """ +🤖 **RAG Assistant Help** + +I'm your AI-powered document search and chat assistant. Here's what I can do: + +**📚 Document Search** +- Ask questions about your documents +- Search for specific information +- Get summaries and insights + +**💬 Chat Features** +- Have conversations about your documents +- Ask follow-up questions +- Get detailed explanations + +**🔍 How to Use** +- Just type your question naturally +- I'll search through your documents and provide answers +- I can cite sources and provide context + +**❓ Examples** +- "What are the main benefits mentioned in the policy document?" +- "Can you summarize the key points from the meeting notes?" +- "Find information about the new procedures" + +Type your question to get started! + """ + + return MessageFactory.text(help_text) + + async def handle_unknown_command( + self, + turn_context: TurnContext, + conversation_data: ConversationData + ) -> Activity: + """ + Handle unknown commands or unclear messages. + """ + unknown_text = """ +I'm not sure what you're looking for. Here are some things I can help with: + +• Ask questions about your documents +• Search for specific information +• Get summaries and insights +• Have conversations about your content + +Try asking me a specific question, or type "help" for more information. + """ + + return MessageFactory.text(unknown_text) \ No newline at end of file diff --git a/agents/handlers/teams_handler.py b/agents/handlers/teams_handler.py new file mode 100644 index 0000000000..6ca1aa4127 --- /dev/null +++ b/agents/handlers/teams_handler.py @@ -0,0 +1,553 @@ +""" +Teams-specific handler for Microsoft 365 Agent. +This handler provides Teams-specific functionality and message formatting. +""" + +import logging +from typing import Dict, Any, List, Optional +from dataclasses import dataclass + +from botbuilder.core import TurnContext, MessageFactory +from botbuilder.schema import Activity, Attachment, CardAction, ActionTypes +# from botbuilder.adapters.teams import TeamsActivityHandler, TeamsInfo + +from services.rag_service import RAGService, RAGRequest, RAGResponse +from services.auth_service import AuthService +from adapters.response_adapter import ResponseAdapter +from adapters.teams_response_adapter import TeamsResponseAdapter +from components.teams_components import TeamsComponents, TeamsCardConfig +from constants.teams_text import TeamsTextConstants + + +logger = logging.getLogger(__name__) + + +@dataclass +class ConversationData: + """Data stored for each conversation.""" + conversation_id: str + user_id: str + channel_id: str + message_count: int = 0 + last_activity: Optional[str] = None + + +class TeamsHandler: + """ + Teams-specific handler that extends the base message handler + with Teams-specific functionality like adaptive cards, mentions, and file handling. + """ + + def __init__(self, rag_service: RAGService, auth_service: AuthService): + super().__init__() + self.rag_service = rag_service + self.auth_service = auth_service + self.response_adapter = ResponseAdapter() + self.teams_response_adapter = TeamsResponseAdapter() + self.teams_components = TeamsComponents() + + async def handle_message( + self, + turn_context: TurnContext, + conversation_data: ConversationData, + user_data: Dict[str, Any], + auth_claims: Dict[str, Any] + ) -> Optional[Activity]: + """ + Handle an incoming Teams message and generate a response. + """ + try: + # Check if this is an adaptive card action + if turn_context.activity.value: + return await self._handle_adaptive_card_action( + turn_context, conversation_data, user_data, auth_claims + ) + + # Check if the bot was mentioned + if await self._is_bot_mentioned(turn_context): + # Remove the mention from the message + message_text = await self._remove_mention(turn_context) + else: + message_text = turn_context.activity.text or "" + + if not message_text.strip(): + return await self._create_mention_reminder(turn_context) + + # Get conversation history from user data + conversation_history = user_data.get("conversation_history", []) + + # Create RAG request + rag_request = RAGRequest( + message=message_text, + conversation_history=conversation_history, + user_id=conversation_data.user_id, + channel_id=conversation_data.channel_id, + context={ + "auth_claims": auth_claims, + "conversation_id": conversation_data.conversation_id, + "message_count": conversation_data.message_count, + "teams_context": await self._get_teams_context(turn_context) + } + ) + + # Process the request with RAG service + rag_response = await self.rag_service.process_query(rag_request) + + # Update conversation history + conversation_history.append({ + "role": "user", + "content": message_text + }) + conversation_history.append({ + "role": "assistant", + "content": rag_response.answer + }) + + # Keep only the last 10 exchanges to manage context length + if len(conversation_history) > 20: # 10 user + 10 assistant messages + conversation_history = conversation_history[-20:] + + user_data["conversation_history"] = conversation_history + + # Format response for Teams + response_activity = await self._format_teams_response( + turn_context, rag_response, conversation_data + ) + + return response_activity + + except Exception as e: + logger.error(f"Error handling Teams message: {e}") + return MessageFactory.text( + "I'm sorry, I encountered an error processing your request. Please try again." + ) + + async def _handle_adaptive_card_action( + self, + turn_context: TurnContext, + conversation_data: ConversationData, + user_data: Dict[str, Any], + auth_claims: Dict[str, Any] + ) -> Optional[Activity]: + """ + Handle adaptive card button actions. + """ + try: + action_data = turn_context.activity.value + action_type = action_data.get("action", "") + + if action_type == "follow_up": + return await self._handle_follow_up_action( + turn_context, conversation_data, user_data, auth_claims + ) + elif action_type == "search_related": + return await self._handle_search_related_action( + turn_context, conversation_data, user_data, auth_claims + ) + elif action_type == "summarize": + return await self._handle_summarize_action( + turn_context, conversation_data, user_data, auth_claims + ) + else: + return MessageFactory.text( + f"I received an action '{action_type}' but I'm not sure how to handle it. Please try asking me a question directly." + ) + + except Exception as e: + logger.error(f"Error handling adaptive card action: {e}") + return MessageFactory.text( + "I encountered an error processing your action. Please try asking me a question directly." + ) + + async def _handle_follow_up_action( + self, + turn_context: TurnContext, + conversation_data: ConversationData, + user_data: Dict[str, Any], + auth_claims: Dict[str, Any] + ) -> Activity: + """Handle follow-up action.""" + return MessageFactory.text(TeamsTextConstants.FOLLOW_UP_RESPONSE) + + async def _handle_search_related_action( + self, + turn_context: TurnContext, + conversation_data: ConversationData, + user_data: Dict[str, Any], + auth_claims: Dict[str, Any] + ) -> Activity: + """Handle search related action.""" + return MessageFactory.text(TeamsTextConstants.SEARCH_RELATED_RESPONSE) + + async def _handle_summarize_action( + self, + turn_context: TurnContext, + conversation_data: ConversationData, + user_data: Dict[str, Any], + auth_claims: Dict[str, Any] + ) -> Activity: + """Handle summarize action.""" + return MessageFactory.text(TeamsTextConstants.SUMMARIZE_RESPONSE) + + async def _is_bot_mentioned(self, turn_context: TurnContext) -> bool: + """Check if the bot was mentioned in the message.""" + try: + # Check if the bot is mentioned in the activity + if hasattr(turn_context.activity, 'entities') and turn_context.activity.entities: + for entity in turn_context.activity.entities: + if entity.type == "mention" and entity.mentioned: + return True + return False + except Exception as e: + logger.warning(f"Error checking bot mention: {e}") + return False + + async def _remove_mention(self, turn_context: TurnContext) -> str: + """Remove bot mention from the message text.""" + try: + message_text = turn_context.activity.text or "" + + # Simple mention removal - in production, you'd want more sophisticated parsing + if "" in message_text and "" in message_text: + # Remove the mention tags and content + import re + message_text = re.sub(r'.*?', '', message_text).strip() + + return message_text + except Exception as e: + logger.warning(f"Error removing mention: {e}") + return turn_context.activity.text or "" + + async def _get_teams_context(self, turn_context: TurnContext) -> Dict[str, Any]: + """Get Teams-specific context information.""" + try: + context = { + "channel_id": turn_context.activity.channel_id, + "conversation_id": turn_context.activity.conversation.id, + "user_id": turn_context.activity.from_property.id, + "user_name": turn_context.activity.from_property.name, + "tenant_id": turn_context.activity.conversation.tenant_id if hasattr(turn_context.activity.conversation, 'tenant_id') else None + } + + # Try to get additional Teams context + try: + teams_info = await TeamsInfo.get_team_details(turn_context) + context["team_id"] = teams_info.id + context["team_name"] = teams_info.name + except Exception: + # Not in a team context + pass + + return context + + except Exception as e: + logger.warning(f"Error getting Teams context: {e}") + return {} + + async def _format_teams_response( + self, + turn_context: TurnContext, + rag_response: RAGResponse, + conversation_data: ConversationData + ) -> Activity: + """ + Format the RAG response specifically for Teams. + """ + try: + # Create adaptive card for rich response + if rag_response.sources or rag_response.citations: + return await self._create_adaptive_card_response( + turn_context, rag_response, conversation_data + ) + else: + # Simple text response + return MessageFactory.text(rag_response.answer) + + except Exception as e: + logger.error(f"Error formatting Teams response: {e}") + return MessageFactory.text(rag_response.answer) + + async def _create_adaptive_card_response( + self, + turn_context: TurnContext, + rag_response: RAGResponse, + conversation_data: ConversationData + ) -> Activity: + """ + Create an adaptive card response for Teams with rich formatting. + """ + try: + # Create adaptive card JSON with enhanced styling + card_json = { + "type": "AdaptiveCard", + "version": "1.4", + "body": [ + { + "type": "Container", + "style": "emphasis", + "items": [ + { + "type": "TextBlock", + "text": "🤖 RAG Assistant", + "weight": "Bolder", + "size": "Medium", + "color": "Accent" + } + ] + }, + { + "type": "TextBlock", + "text": rag_response.answer, + "wrap": True, + "size": "Medium", + "spacing": "Medium" + } + ], + "actions": [ + { + "type": "Action.Submit", + "title": "💬 Ask Follow-up", + "data": { + "action": "follow_up", + "conversation_id": conversation_data.conversation_id + }, + "style": "positive" + }, + { + "type": "Action.Submit", + "title": "🔍 Search Related", + "data": { + "action": "search_related", + "conversation_id": conversation_data.conversation_id + }, + "style": "default" + }, + { + "type": "Action.Submit", + "title": "📋 Summarize", + "data": { + "action": "summarize", + "conversation_id": conversation_data.conversation_id + }, + "style": "default" + } + ] + } + + # Add sources section if available + if rag_response.sources: + sources_container = { + "type": "Container", + "style": "default", + "items": [ + { + "type": "TextBlock", + "text": "📚 Sources", + "weight": "Bolder", + "size": "Small", + "color": "Accent", + "spacing": "Medium" + } + ] + } + + for i, source in enumerate(rag_response.sources[:3], 1): + source_text = source.get('title', 'Unknown Source') + source_url = source.get('url', '') + + if source_url: + sources_container["items"].append({ + "type": "TextBlock", + "text": f"{i}. [{source_text}]({source_url})", + "wrap": True, + "size": "Small", + "spacing": "Small" + }) + else: + sources_container["items"].append({ + "type": "TextBlock", + "text": f"{i}. {source_text}", + "wrap": True, + "size": "Small", + "spacing": "Small" + }) + + card_json["body"].append(sources_container) + + # Add citations section if available + if rag_response.citations: + citations_container = { + "type": "Container", + "style": "default", + "items": [ + { + "type": "TextBlock", + "text": "🔗 Citations", + "weight": "Bolder", + "size": "Small", + "color": "Accent", + "spacing": "Medium" + } + ] + } + + for i, citation in enumerate(rag_response.citations[:3], 1): + citations_container["items"].append({ + "type": "TextBlock", + "text": f"{i}. {citation}", + "wrap": True, + "size": "Small", + "spacing": "Small" + }) + + card_json["body"].append(citations_container) + + # Add thoughts section if available (for transparency) + if rag_response.thoughts: + thoughts_container = { + "type": "Container", + "style": "default", + "items": [ + { + "type": "TextBlock", + "text": "💭 Process", + "weight": "Bolder", + "size": "Small", + "color": "Accent", + "spacing": "Medium" + } + ] + } + + for thought in rag_response.thoughts[:2]: # Limit to 2 thoughts + thoughts_container["items"].append({ + "type": "TextBlock", + "text": f"• {thought.get('title', 'Step')}: {thought.get('description', '')}", + "wrap": True, + "size": "Small", + "spacing": "Small" + }) + + card_json["body"].append(thoughts_container) + + # Add token usage if available + if rag_response.token_usage: + usage_container = { + "type": "Container", + "style": "default", + "items": [ + { + "type": "TextBlock", + "text": "📊 Usage", + "weight": "Bolder", + "size": "Small", + "color": "Accent", + "spacing": "Medium" + }, + { + "type": "TextBlock", + "text": f"Tokens: {rag_response.token_usage.get('total_tokens', 'N/A')} (Prompt: {rag_response.token_usage.get('prompt_tokens', 'N/A')}, Completion: {rag_response.token_usage.get('completion_tokens', 'N/A')})", + "wrap": True, + "size": "Small", + "spacing": "Small" + } + ] + } + card_json["body"].append(usage_container) + + # Create attachment + attachment = Attachment( + content_type="application/vnd.microsoft.card.adaptive", + content=card_json + ) + + # Create activity with attachment + activity = MessageFactory.attachment(attachment) + activity.text = rag_response.answer # Fallback text + + return activity + + except Exception as e: + logger.error(f"Error creating adaptive card: {e}") + return MessageFactory.text(rag_response.answer) + + async def _create_mention_reminder(self, turn_context: TurnContext) -> Activity: + """Create a reminder to mention the bot.""" + reminder_text = TeamsTextConstants.format_mention_reminder() + return MessageFactory.text(reminder_text) + + async def handle_file_upload( + self, + turn_context: TurnContext, + conversation_data: ConversationData + ) -> Activity: + """ + Handle file uploads in Teams. + """ + try: + # Check if there are attachments + if turn_context.activity.attachments: + file_info = [] + for attachment in turn_context.activity.attachments: + file_info.append(f"• {attachment.name} ({attachment.content_type})") + + response_text = f""" +📎 I see you've uploaded {len(turn_context.activity.attachments)} file(s): + +{chr(10).join(file_info)} + +I can help you search through these documents once they're processed. You can ask me questions about their content, or I can provide summaries and insights. + +**What would you like to know about these files?** + """ + + return MessageFactory.text(response_text) + else: + return MessageFactory.text("I don't see any files attached. Please try uploading a file and I'll help you with it.") + + except Exception as e: + logger.error(f"Error handling file upload: {e}") + return MessageFactory.text("I encountered an error processing your file upload. Please try again.") + + async def handle_help_request( + self, + turn_context: TurnContext, + conversation_data: ConversationData + ) -> Activity: + """ + Handle help requests in Teams. + """ + help_text = """ +🤖 **RAG Assistant Help** + +I'm your AI-powered document search and chat assistant. Here's what I can do: + +**📚 Document Search** +- Ask questions about your documents +- Search for specific information +- Get summaries and insights + +**💬 Chat Features** +- Have conversations about your documents +- Ask follow-up questions +- Get detailed explanations + +**🔍 How to Use** +- Mention me with @RAG Assistant or just type your question +- I'll search through your documents and provide answers +- I can cite sources and provide context + +**❓ Examples** +- "What are the main benefits mentioned in the policy document?" +- "Can you summarize the key points from the meeting notes?" +- "Find information about the new procedures" + +**📎 File Uploads** +- Upload documents and I'll help you search through them +- Ask questions about uploaded files +- Get insights and summaries + +Type your question or mention me to get started! + """ + + return MessageFactory.text(help_text) \ No newline at end of file diff --git a/agents/main.py b/agents/main.py new file mode 100644 index 0000000000..3beb0b12e4 --- /dev/null +++ b/agents/main.py @@ -0,0 +1,495 @@ +""" +Main entry point for the Microsoft 365 Agent. +This module starts the agent application and handles incoming requests. +""" + +import asyncio +import logging +import os +import uuid +from pathlib import Path +from typing import Optional + +from dotenv import load_dotenv + +from botbuilder.core import BotFrameworkAdapter, BotFrameworkAdapterSettings +from botbuilder.schema import ( + Activity, + ActivityTypes, + ConversationReference, + ResourceResponse, +) +from quart import Quart, request, jsonify +from quart_cors import cors + +from config.agent_config import AgentConfig +from agent_app import AgentApplication + +# Load environment variables from .env file +env_path = Path(__file__).parent / ".env" +if env_path.exists(): + load_dotenv(env_path) + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + +if env_path.exists(): + logger.info(f"Loaded environment variables from {env_path}") + + +class AgentServer: + """ + Server that hosts the Microsoft 365 Agent. + This server handles incoming requests and routes them to the agent. + """ + + def __init__(self): + self.app = Quart(__name__) + # Enable CORS for Bot Framework Emulator + self.app = cors(self.app, allow_origin="*") + self.agent_app: Optional[AgentApplication] = None + self.adapter: Optional[BotFrameworkAdapter] = None + + # Add catch-all logging middleware + @self.app.before_request + async def log_request(): + # Force print to ensure it shows in console + print(f"\n{'='*60}") + print(f"[BEFORE_REQUEST] {request.method} {request.path}") + print(f"[BEFORE_REQUEST] Headers: {dict(request.headers)}") + logger.info(f"Incoming request: {request.method} {request.path}") + logger.info(f"Headers: {dict(request.headers)}") + + # Configure routes + self._setup_routes() + + def _setup_routes(self): + """Set up the Quart routes.""" + + @self.app.route("/", methods=["GET"]) + async def health_check(): + """Health check endpoint.""" + logger.info("Health check called") + return jsonify({ + "status": "healthy", + "service": "Microsoft 365 RAG Agent", + "version": "1.0.0" + }) + + @self.app.route("/api/copilot/health", methods=["GET"]) + async def copilot_health(): + """Health check endpoint for Copilot plugin.""" + return jsonify({ + "status": "healthy", + "plugin": "ai-master-engineer-rag", + "version": "1.0.0" + }) + + @self.app.route("/api/copilot/search", methods=["POST"]) + async def copilot_search(): + """Search endpoint for Copilot plugin.""" + try: + if not self.agent_app: + return jsonify({"error": "Agent not initialized"}), 500 + + body = await request.get_json() + if not body: + return jsonify({"error": "Invalid request body"}), 400 + + # Get auth claims + auth_claims = { + "oid": body.get("userId", ""), + "tenant_id": body.get("tenantId", self.agent_app.config.tenant_id) + } + + # Get Copilot handler + from handlers.copilot_handler import CopilotHandler, CopilotRequest + copilot_handler = CopilotHandler( + self.agent_app.rag_service, + self.agent_app.auth_service + ) + + # Create request + copilot_request = CopilotRequest( + query=body.get("query", ""), + conversation_history=body.get("conversationHistory"), + max_results=body.get("maxResults", 5) + ) + + # Handle request + result = await copilot_handler.handle_search_request( + copilot_request, + auth_claims + ) + + return jsonify(result) + + except Exception as e: + logger.error(f"Error in Copilot search: {e}", exc_info=True) + return jsonify({"error": str(e)}), 500 + + @self.app.route("/api/copilot/query", methods=["POST"]) + async def copilot_query(): + """Query endpoint for Copilot plugin.""" + try: + if not self.agent_app: + return jsonify({"error": "Agent not initialized"}), 500 + + body = await request.get_json() + if not body: + return jsonify({"error": "Invalid request body"}), 400 + + # Get auth claims + auth_claims = { + "oid": body.get("userId", ""), + "tenant_id": body.get("tenantId", self.agent_app.config.tenant_id) + } + + # Get Copilot handler + from handlers.copilot_handler import CopilotHandler, CopilotRequest + copilot_handler = CopilotHandler( + self.agent_app.rag_service, + self.agent_app.auth_service + ) + + # Create request + copilot_request = CopilotRequest( + query=body.get("message", body.get("query", "")), + conversation_history=body.get("conversationHistory"), + max_results=body.get("maxResults", 5) + ) + + # Handle request + result = await copilot_handler.handle_query_request( + copilot_request, + auth_claims + ) + + return jsonify(result) + + except Exception as e: + logger.error(f"Error in Copilot query: {e}", exc_info=True) + return jsonify({"error": str(e)}), 500 + + @self.app.route("/api/messages", methods=["GET", "OPTIONS"]) + async def messages_options(): + """Handle OPTIONS for CORS preflight.""" + logger.info("OPTIONS request received for /api/messages") + return "", 200 + + @self.app.route("/api/messages", methods=["POST"]) + async def messages(): + """Main endpoint for Bot Framework messages.""" + # Force print to ensure it shows in console + print(f"\n{'='*60}") + print("[MESSAGES ENDPOINT] POST /api/messages RECEIVED") + print(f"[MESSAGES ENDPOINT] Headers: {dict(request.headers)}") + logger.info("=" * 50) + logger.info("RECEIVED POST REQUEST TO /api/messages") + logger.info(f"Headers: {dict(request.headers)}") + + # Generate simple correlation id for this request + import uuid + traceparent = uuid.uuid4().hex + logger.info(f"traceparent={traceparent}") + + try: + # Get the request body FIRST - before any initialization checks + body = await request.get_json() + if not body: + logger.error("Invalid request body - body is None") + return jsonify({"error": "Invalid request body"}), 400 + + # Get channel ID from body for logging + channel_id = body.get('channelId', 'unknown') + logger.info(f"Received message on channel: {channel_id}") + if not self.agent_app: + logger.error("Agent not initialized") + return jsonify({"error": "Agent not initialized"}), 500 + + # Get the adapter from agent_app + adapter = self.agent_app.get_adapter() + if not adapter: + logger.error("Adapter not initialized") + return jsonify({"error": "Adapter not initialized"}), 500 + + logger.info(f"Received activity type: {body.get('type', 'unknown')}, channel: {body.get('channelId', 'unknown')}") + + # Get auth header (required for Bot Framework authentication) + # For emulator/testing without auth header, use empty string + auth_header = request.headers.get("Authorization", "") + + logger.info(f"=== REQUEST DETAILS ===") + logger.info(f"Channel ID from body: {channel_id}") + logger.info(f"Auth header present: {bool(auth_header)}") + if auth_header: + logger.info(f"Auth header (first 50 chars): {auth_header[:50]}...") + else: + logger.warning("No Authorization header in request") + + # Create activity from request + try: + activity = Activity().deserialize(body) + logger.info(f"Activity deserialized: type={activity.type}, channel_id={activity.channel_id}") + # Use channel_id from activity if available, fallback to body + channel_id = activity.channel_id if hasattr(activity, 'channel_id') and activity.channel_id else channel_id + except Exception as e: + logger.error(f"Failed to deserialize activity: {e}", exc_info=True) + return jsonify({"error": f"Invalid activity format: {str(e)}"}), 400 + + # Process the activity using the adapter + async def logic(context): + try: + await self.agent_app.agent.on_turn(context) + except Exception as e: + logger.error(f"Error in agent.on_turn: {e}", exc_info=True) + raise + + try: + logger.info(f"Processing activity through adapter with auth_header present: {bool(auth_header)}") + response = await adapter.process_activity( + activity, + auth_header, + logic + ) + logger.info(f"Activity processed successfully, response: {response is not None}") + + if response: + return jsonify(response.serialize()) + else: + return "", 200 + except Exception as auth_error: + # Log the full error details + error_type = type(auth_error).__name__ + error_message = str(auth_error) + logger.error(f"Adapter error type: {error_type}, message: {error_message}") + logger.error(f"Full error: {auth_error}", exc_info=True) + + # Check channel ID from the activity + channel_id_from_activity = getattr(activity, 'channel_id', 'unknown') + logger.info(f"Channel ID from activity: {channel_id_from_activity}") + + # Authentication error handling + # For local emulator only: Allow bypass for local development/testing + # For webchat and production channels: Require proper authentication + local_emulator_only = ["emulator"] + + if channel_id_from_activity in local_emulator_only: + # Only allow bypass for local Bot Framework Emulator + # This is safe because emulator runs locally and doesn't expose the service + logger.warning(f"Auth error for local emulator ({channel_id_from_activity}), attempting workaround: {auth_error}") + + try: + from botbuilder.core import TurnContext, BotAdapter + from botbuilder.schema import ConversationAccount, ChannelAccount + + # Ensure activity has required fields + if not hasattr(activity, 'conversation') or not activity.conversation: + activity.conversation = ConversationAccount(id="test-conv-id") + if not hasattr(activity, 'from_property') or not activity.from_property: + activity.from_property = ChannelAccount(id="test-user", name="Test User") + if not hasattr(activity, 'recipient') or not activity.recipient: + activity.recipient = ChannelAccount(id="bot", name="Bot") + + # Create a minimal adapter for local emulator only + class LocalEmulatorAdapter(BotAdapter): + """Adapter for local Bot Framework Emulator only - skips validation.""" + + async def send_activities(self, context, activities): + responses = [] + for activity_to_send in activities: + activity_id = getattr(activity_to_send, "id", None) or str( + uuid.uuid4() + ) + responses.append(ResourceResponse(id=activity_id)) + return responses + + async def update_activity(self, context, activity_to_update): + activity_id = getattr(activity_to_update, "id", None) or str( + uuid.uuid4() + ) + return ResourceResponse(id=activity_id) + + async def delete_activity(self, context, reference: ConversationReference): + logger.debug("Local emulator delete_activity called: %s", reference) + return None + + local_adapter = LocalEmulatorAdapter() + turn_context = TurnContext(local_adapter, activity) + + # Call the agent's on_turn directly (only for local emulator) + await self.agent_app.agent.on_turn(turn_context) + + logger.info("Successfully processed activity via local emulator workaround") + return "", 200 + + except Exception as direct_error: + logger.error(f"Local emulator processing failed: {direct_error}", exc_info=True) + return jsonify({ + "error": "Authentication failed - check bot credentials", + "channel": channel_id_from_activity, + "details": str(direct_error) + }), 401 + else: + # For webchat and production channels: Require proper authentication + # Webchat from Azure Portal should have valid bot credentials + logger.error(f"Authentication failed for channel ({channel_id_from_activity}): {auth_error}") + logger.error("This may indicate misconfigured bot credentials (MICROSOFT_APP_ID or MICROSOFT_APP_PASSWORD)") + + return jsonify({ + "error": "Authentication required", + "channel": channel_id_from_activity, + "details": "Bot Framework authentication failed. Please verify MICROSOFT_APP_ID and MICROSOFT_APP_PASSWORD are configured correctly.", + "help": "For webchat: Ensure bot credentials are set in Azure Portal Bot Channels Registration" + }), 401 + except Exception as adapter_error: + logger.error(f"Adapter.process_activity failed: {adapter_error}", exc_info=True) + # Return more detailed error for debugging + import traceback + return jsonify({ + "error": "Activity processing failed", + "details": str(adapter_error), + "type": type(adapter_error).__name__, + "traceback": traceback.format_exc() + }), 400 + + except Exception as e: + logger.error(f"Error processing message: {e}", exc_info=True) + return jsonify({"error": str(e), "type": type(e).__name__}), 500 + + @self.app.route("/api/health", methods=["GET"]) + async def health(): + """Detailed health check.""" + try: + if not self.agent_app: + return jsonify({ + "status": "unhealthy", + "error": "Agent not initialized" + }), 500 + + # Check backend connectivity and latency + import time + import aiohttp + backend_ok = False + backend_latency_ms = None + backend_status = None + backend_url = f"{self.agent_app.config.backend_url}/config" + t0 = time.time() + try: + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=3)) as s: + async with s.get(backend_url) as r: + backend_status = r.status + backend_ok = r.status == 200 + except Exception as _: + backend_ok = False + finally: + backend_latency_ms = int((time.time() - t0) * 1000) + + # Compose health payload + health_status = { + "status": "healthy" if backend_ok else "degraded", + "agent": "initialized", + "services": { + "rag_service": "initialized", + "auth_service": "initialized", + "backend": { + "ok": backend_ok, + "status_code": backend_status, + "latency_ms": backend_latency_ms, + "url": backend_url + } + } + } + + return jsonify(health_status) + + except Exception as e: + logger.error(f"Error in health check: {e}") + return jsonify({ + "status": "unhealthy", + "error": str(e) + }), 500 + + @self.app.route("/api/config", methods=["GET"]) + async def config(): + """Get agent configuration (non-sensitive parts).""" + try: + if not self.agent_app: + return jsonify({"error": "Agent not initialized"}), 500 + + config_info = { + "agent_name": self.agent_app.config.agent_name, + "agent_description": self.agent_app.config.agent_description, + "max_conversation_turns": self.agent_app.config.max_conversation_turns, + "channels": { + "teams": self.agent_app.config.enable_teams, + "copilot": self.agent_app.config.enable_copilot, + "web_chat": self.agent_app.config.enable_web_chat + } + } + + return jsonify(config_info) + + except Exception as e: + logger.error(f"Error getting config: {e}") + return jsonify({"error": "Internal server error"}), 500 + + async def initialize(self): + """Initialize the agent application.""" + try: + # Load configuration + config = AgentConfig.from_environment() + config.validate() + + # Initialize the agent application + self.agent_app = AgentApplication(config) + + # Get the adapter + self.adapter = self.agent_app.get_adapter() + + logger.info("Agent application initialized successfully") + + except Exception as e: + logger.error(f"Failed to initialize agent application: {e}") + raise + + async def run(self, host: str = "0.0.0.0", port: int = None): + """Run the agent server.""" + try: + # Initialize the agent + await self.initialize() + + # Get port from environment variable (Azure App Service uses PORT) + if port is None: + port = int(os.getenv("PORT", 8000)) + + # Start the server + logger.info(f"Starting agent server on {host}:{port}") + await self.app.run_task(host=host, port=port) + + except Exception as e: + logger.error(f"Failed to run agent server: {e}") + raise + + +async def main(): + """Main function to start the agent server.""" + try: + # Create and run the server + server = AgentServer() + await server.run() + + except KeyboardInterrupt: + logger.info("Agent server stopped by user") + except Exception as e: + logger.error(f"Agent server failed: {e}") + raise + + +if __name__ == "__main__": + # Run the agent server + asyncio.run(main()) \ No newline at end of file diff --git a/agents/models/__init__.py b/agents/models/__init__.py new file mode 100644 index 0000000000..16caa3c4e0 --- /dev/null +++ b/agents/models/__init__.py @@ -0,0 +1,17 @@ +""" +Models package for Agents service. +""" + +from .citation import Citation, CitationSource, CitationProvider, resolve_citation_conflicts + +__all__ = [ + "Citation", + "CitationSource", + "CitationProvider", + "resolve_citation_conflicts" +] + + + + + diff --git a/agents/models/citation.py b/agents/models/citation.py new file mode 100644 index 0000000000..5f26b9e5bf --- /dev/null +++ b/agents/models/citation.py @@ -0,0 +1,241 @@ +""" +Unified Citation Model for RAG Responses. + +This module defines a unified citation schema that works for both corpus sources +(Azure Cognitive Search) and web search sources (SERPR, Firecrawl, Cohere). +""" + +from dataclasses import dataclass, field +from typing import Optional, Dict, Any, List +from enum import Enum +from datetime import datetime + + +class CitationSource(str, Enum): + """Source type for citations.""" + CORPUS = "corpus" + WEB = "web" + + +class CitationProvider(str, Enum): + """Provider type for citations.""" + AZURE_SEARCH = "azure_search" + SERPR = "serpr" + FIRECRAWL = "firecrawl" + COHERE = "cohere" + UNKNOWN = "unknown" + + +@dataclass +class Citation: + """ + Unified citation model for RAG responses. + + This model works for both corpus sources (Azure Search) and web sources + (SERPR, Firecrawl, Cohere). It includes conflict resolution logic. + """ + # Required fields + source: CitationSource + provider: CitationProvider + url: str + title: str + snippet: str + + # Optional metadata + confidence: float = 1.0 # 0.0 to 1.0 + metadata: Dict[str, Any] = field(default_factory=dict) + + # Optional fields for different source types + file_path: Optional[str] = None # For corpus sources + page_number: Optional[int] = None # For corpus sources + chunk_id: Optional[str] = None # For corpus sources + domain: Optional[str] = None # For web sources + timestamp: Optional[datetime] = None # For web sources + + def canonical_url(self) -> str: + """ + Get canonical URL for deduplication. + Normalizes URLs to avoid duplicates. + """ + url = self.url.lower().strip() + # Remove common tracking parameters + if "?" in url: + base, params = url.split("?", 1) + # Keep only essential parameters + essential_params = [] + for param in params.split("&"): + key = param.split("=")[0] if "=" in param else param + # Keep essential params (customize as needed) + if key not in ["utm_source", "utm_medium", "utm_campaign", "ref", "fbclid"]: + essential_params.append(param) + if essential_params: + url = f"{base}?{'&'.join(essential_params)}" + else: + url = base + return url + + def dedup_key(self) -> str: + """ + Generate a deduplication key for conflict resolution. + Uses canonical URL + title hash. + """ + import hashlib + canonical = self.canonical_url() + title_hash = hashlib.md5(self.title.encode()).hexdigest()[:8] + return f"{canonical}:{title_hash}" + + def to_dict(self) -> Dict[str, Any]: + """Convert citation to dictionary for JSON serialization.""" + result = { + "source": self.source.value, + "provider": self.provider.value, + "url": self.url, + "title": self.title, + "snippet": self.snippet, + "confidence": self.confidence, + "metadata": self.metadata + } + + if self.file_path: + result["file_path"] = self.file_path + if self.page_number is not None: + result["page_number"] = self.page_number + if self.chunk_id: + result["chunk_id"] = self.chunk_id + if self.domain: + result["domain"] = self.domain + if self.timestamp: + result["timestamp"] = self.timestamp.isoformat() + + return result + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "Citation": + """Create Citation from dictionary.""" + timestamp = None + if data.get("timestamp"): + if isinstance(data["timestamp"], str): + timestamp = datetime.fromisoformat(data["timestamp"]) + elif isinstance(data["timestamp"], datetime): + timestamp = data["timestamp"] + + return cls( + source=CitationSource(data.get("source", "corpus")), + provider=CitationProvider(data.get("provider", "unknown")), + url=data["url"], + title=data["title"], + snippet=data["snippet"], + confidence=data.get("confidence", 1.0), + metadata=data.get("metadata", {}), + file_path=data.get("file_path"), + page_number=data.get("page_number"), + chunk_id=data.get("chunk_id"), + domain=data.get("domain"), + timestamp=timestamp + ) + + @classmethod + def from_azure_search( + cls, + doc: Dict[str, Any], + snippet: str, + confidence: float = 1.0 + ) -> "Citation": + """Create Citation from Azure Cognitive Search document.""" + return cls( + source=CitationSource.CORPUS, + provider=CitationProvider.AZURE_SEARCH, + url=doc.get("sourcepage", doc.get("sourcefile", "")), + title=doc.get("title", doc.get("sourcefile", "Document")), + snippet=snippet, + confidence=confidence, + metadata=doc, + file_path=doc.get("sourcefile"), + page_number=doc.get("page", doc.get("pagenum")), + chunk_id=doc.get("id") + ) + + @classmethod + def from_web_result( + cls, + result: Dict[str, Any], + provider: CitationProvider, + confidence: float = 1.0 + ) -> "Citation": + """Create Citation from web search result.""" + from urllib.parse import urlparse + + url = result.get("url", result.get("link", "")) + parsed = urlparse(url) + + return cls( + source=CitationSource.WEB, + provider=provider, + url=url, + title=result.get("title", result.get("name", "")), + snippet=result.get("snippet", result.get("description", "")), + confidence=confidence, + metadata=result, + domain=parsed.netloc, + timestamp=datetime.now() # Web results are current + ) + + +def resolve_citation_conflicts( + citations: List[Citation], + prefer_corpus: bool = True +) -> List[Citation]: + """ + Resolve conflicts when the same source appears via corpus and web. + + Args: + citations: List of citations (may include duplicates) + prefer_corpus: If True, prefer corpus source over web when conflicting + + Returns: + Deduplicated list of citations with conflicts resolved + """ + # Group by dedup key + citation_map: Dict[str, List[Citation]] = {} + + for citation in citations: + key = citation.dedup_key() + if key not in citation_map: + citation_map[key] = [] + citation_map[key].append(citation) + + # Resolve conflicts + resolved: List[Citation] = [] + + for key, group in citation_map.items(): + if len(group) == 1: + # No conflict + resolved.append(group[0]) + else: + # Conflict: same source via multiple providers + corpus_citations = [c for c in group if c.source == CitationSource.CORPUS] + web_citations = [c for c in group if c.source == CitationSource.WEB] + + if prefer_corpus and corpus_citations: + # Prefer corpus source + # Choose the one with highest confidence + best = max(corpus_citations, key=lambda c: c.confidence) + resolved.append(best) + elif web_citations: + # Use web source, prefer highest confidence + best = max(web_citations, key=lambda c: c.confidence) + resolved.append(best) + else: + # Fallback: use highest confidence + best = max(group, key=lambda c: c.confidence) + resolved.append(best) + + # Sort by confidence (highest first) + resolved.sort(key=lambda c: c.confidence, reverse=True) + + return resolved + + + + + diff --git a/agents/pytest.ini b/agents/pytest.ini new file mode 100644 index 0000000000..1c60bc11b5 --- /dev/null +++ b/agents/pytest.ini @@ -0,0 +1,15 @@ +[tool:pytest] +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* +addopts = + -v + --tb=short + --strict-markers + --disable-warnings + --asyncio-mode=auto +markers = + integration: marks tests as integration tests (deselect with '-m "not integration"') + unit: marks tests as unit tests + slow: marks tests as slow running \ No newline at end of file diff --git a/agents/requirements.txt b/agents/requirements.txt new file mode 100644 index 0000000000..0e7bffaf20 --- /dev/null +++ b/agents/requirements.txt @@ -0,0 +1,33 @@ +# Bot Framework dependencies +botbuilder-core>=4.15.0 +botbuilder-schema>=4.15.0 + +# Web framework +quart>=0.18.0 +quart-cors>=0.7.0 +hypercorn>=0.14.0 + +# HTTP client for calling backend +aiohttp>=3.8.0 + +# Authentication +msal>=1.24.0 +azure-identity>=1.15.0 + +# Utilities +python-dotenv>=1.0.0 +typing-extensions>=4.0.0 + +# Logging and monitoring +azure-monitor-opentelemetry>=1.6.0 + +# Bot Framework internal dependencies +botframework-connector==4.17.0 +botframework-streaming==4.17.0 +jsonpickle<1.5,>=1.2 + +# Testing dependencies +pytest>=7.0.0 +pytest-asyncio>=0.21.0 +pytest-cov>=4.0.0 +pytest-mock>=3.10.0 \ No newline at end of file diff --git a/agents/scripts/test_backend_integration.py b/agents/scripts/test_backend_integration.py new file mode 100644 index 0000000000..4f25abe228 --- /dev/null +++ b/agents/scripts/test_backend_integration.py @@ -0,0 +1,132 @@ +""" +Test script for backend integration. +This script tests the agent's ability to call the existing backend API. +""" + +import asyncio +import logging +import os +from typing import Dict, Any + +from config.agent_config import AgentConfig +from services.rag_service import RAGService, RAGRequest + + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +async def test_backend_integration(): + """Test the backend integration.""" + try: + # Load configuration + config = AgentConfig.from_environment() + + # Set dummy values for testing + config.app_id = "test-app-id" + config.app_password = "test-app-password" + config.tenant_id = "test-tenant-id" + config.client_id = "test-client-id" + config.client_secret = "test-client-secret" + + config.validate() + + logger.info(f"Testing backend integration with: {config.backend_url}") + + # Initialize RAG service + rag_service = RAGService(config) + await rag_service.initialize() + + # Test 1: Simple chat request + logger.info("Test 1: Simple chat request") + request = RAGRequest( + message="What are the main benefits mentioned in the policy document?", + conversation_history=[], + user_id="test-user-123", + channel_id="test-channel" + ) + + response = await rag_service.process_query(request) + logger.info(f"Response: {response.answer}") + logger.info(f"Sources: {len(response.sources)}") + logger.info(f"Citations: {len(response.citations)}") + logger.info(f"Thoughts: {len(response.thoughts)}") + + # Test 2: Chat with conversation history + logger.info("\nTest 2: Chat with conversation history") + request_with_history = RAGRequest( + message="Can you provide more details about the first benefit?", + conversation_history=[ + {"role": "user", "content": "What are the main benefits mentioned in the policy document?"}, + {"role": "assistant", "content": response.answer} + ], + user_id="test-user-123", + channel_id="test-channel" + ) + + response_with_history = await rag_service.process_query(request_with_history) + logger.info(f"Response with history: {response_with_history.answer}") + + # Test 3: Streaming request + logger.info("\nTest 3: Streaming request") + async for chunk in rag_service.process_query_stream(request): + logger.info(f"Stream chunk: {chunk}") + if chunk.get("type") == "error": + break + + logger.info("✅ All tests completed successfully!") + + except Exception as e: + logger.error(f"❌ Test failed: {e}") + raise + finally: + # Clean up + if 'rag_service' in locals(): + await rag_service.close() + + +async def test_backend_health(): + """Test if the backend is healthy.""" + import aiohttp + + try: + config = AgentConfig.from_environment() + + async with aiohttp.ClientSession() as session: + # Test health endpoint + async with session.get(f"{config.backend_url}/") as response: + if response.status == 200: + logger.info("✅ Backend health check passed") + return True + else: + logger.error(f"❌ Backend health check failed: {response.status}") + return False + + except Exception as e: + logger.error(f"❌ Backend health check error: {e}") + return False + + +async def main(): + """Main test function.""" + logger.info("Starting backend integration tests...") + + # Test 1: Backend health + logger.info("Step 1: Testing backend health...") + backend_healthy = await test_backend_health() + + if not backend_healthy: + logger.error("Backend is not healthy. Please start the backend first.") + logger.info("To start the backend, run: cd /workspace/app/backend && python main.py") + return + + # Test 2: Backend integration + logger.info("Step 2: Testing backend integration...") + await test_backend_integration() + + logger.info("🎉 All tests completed!") + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/agents/scripts/test_teams_integration.py b/agents/scripts/test_teams_integration.py new file mode 100644 index 0000000000..192c3063a0 --- /dev/null +++ b/agents/scripts/test_teams_integration.py @@ -0,0 +1,317 @@ +""" +Test script for Teams integration features. +This script tests the Teams-specific functionality including adaptive cards, +mentions, file handling, and response formatting. +""" + +import asyncio +import logging +import os +from dotenv import load_dotenv +from typing import Dict, Any, List, Optional + +from config.agent_config import AgentConfig +from services.rag_service import RAGService, RAGRequest, RAGResponse +from adapters.teams_response_adapter import TeamsResponseAdapter +from components.teams_components import TeamsComponents, TeamsCardConfig +from botbuilder.schema import Activity, ActivityTypes +from botbuilder.core import TurnContext + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +class MockTurnContext: + """Mock TurnContext for testing.""" + + def __init__(self, activity: Activity): + self.activity = activity + self.channel_id = "msteams" + self.conversation = activity.conversation + self.from_property = activity.from_property + self.recipient = activity.recipient + + +async def test_teams_components(): + """Test Teams components functionality.""" + logger.info("Testing Teams Components...") + + try: + # Test welcome card + welcome_card = TeamsComponents.create_welcome_card() + assert welcome_card["type"] == "AdaptiveCard" + assert "Welcome to Structural Engineering Assistant" in welcome_card["body"][0]["items"][0]["text"] + logger.info("✅ Welcome card created successfully") + + # Test help card + help_card = TeamsComponents.create_help_card() + assert help_card["type"] == "AdaptiveCard" + assert "Structural Engineering Assistant Help" in help_card["body"][0]["items"][0]["text"] + logger.info("✅ Help card created successfully") + + # Test error card + error_card = TeamsComponents.create_error_card("Test error message") + assert error_card["type"] == "AdaptiveCard" + assert "Test error message" in error_card["body"][1]["text"] + logger.info("✅ Error card created successfully") + + # Test loading card + loading_card = TeamsComponents.create_loading_card() + assert loading_card["type"] == "AdaptiveCard" + assert "Processing your request" in loading_card["body"][0]["items"][0]["text"] + logger.info("✅ Loading card created successfully") + + # Test file upload card + file_card = TeamsComponents.create_file_upload_card("test.pdf", "application/pdf") + assert file_card["type"] == "AdaptiveCard" + assert "test.pdf" in file_card["body"][1]["text"] + logger.info("✅ File upload card created successfully") + + # Test quick actions card + quick_actions_card = TeamsComponents.create_quick_actions_card() + assert quick_actions_card["type"] == "AdaptiveCard" + assert "Quick Actions" in quick_actions_card["body"][0]["items"][0]["text"] + logger.info("✅ Quick actions card created successfully") + + logger.info("🎉 All Teams components tests passed!") + + except Exception as e: + logger.error(f"❌ Teams components test failed: {e}") + raise + + +async def test_teams_response_adapter(): + """Test Teams response adapter functionality.""" + logger.info("Testing Teams Response Adapter...") + + try: + # Create mock RAG response + rag_response = RAGResponse( + answer="This is a test response with comprehensive information about the topic.", + sources=[ + {"title": "Source 1", "url": "https://example.com/source1"}, + {"title": "Source 2", "url": "https://example.com/source2"} + ], + citations=["Citation 1", "Citation 2"], + thoughts=[ + {"title": "Query Analysis", "description": "Analyzed user query"}, + {"title": "Information Retrieval", "description": "Retrieved relevant documents"} + ], + token_usage={"total_tokens": 150, "prompt_tokens": 50, "completion_tokens": 100}, + model_info={"model": "gpt-4", "temperature": "0.3"} + ) + + # Create mock activity + activity = Activity( + type=ActivityTypes.message, + text="Test message", + from_property={"id": "user1", "name": "Test User"}, + recipient={"id": "bot1", "name": "RAG Assistant"}, + conversation={"id": "conv1"} + ) + + # Create mock turn context + turn_context = MockTurnContext(activity) + + # Test response adapter + adapter = TeamsResponseAdapter() + + # Test RAG response formatting + response_activity = adapter.format_rag_response(turn_context, rag_response) + assert response_activity.attachments is not None + assert len(response_activity.attachments) > 0 + assert response_activity.attachments[0].content_type == "application/vnd.microsoft.card.adaptive" + logger.info("✅ RAG response formatting successful") + + # Test text response formatting + text_response = adapter.format_text_response(turn_context, "Simple text response") + assert text_response.text == "Simple text response" + assert text_response.suggested_actions is not None + logger.info("✅ Text response formatting successful") + + # Test welcome response + welcome_response = adapter.format_welcome_response(turn_context) + assert welcome_response.attachments is not None + assert len(welcome_response.attachments) > 0 + logger.info("✅ Welcome response formatting successful") + + # Test help response + help_response = adapter.format_help_response(turn_context) + assert help_response.attachments is not None + assert len(help_response.attachments) > 0 + logger.info("✅ Help response formatting successful") + + # Test error response + error_response = adapter.format_error_response(turn_context, "Test error") + assert error_response.attachments is not None + assert len(error_response.attachments) > 0 + logger.info("✅ Error response formatting successful") + + # Test loading response + loading_response = adapter.format_loading_response(turn_context) + assert loading_response.attachments is not None + assert len(loading_response.attachments) > 0 + logger.info("✅ Loading response formatting successful") + + # Test file upload response + file_response = adapter.format_file_upload_response(turn_context, "test.pdf", "application/pdf") + assert file_response.attachments is not None + assert len(file_response.attachments) > 0 + logger.info("✅ File upload response formatting successful") + + # Test quick actions response + quick_actions_response = adapter.format_quick_actions_response(turn_context) + assert quick_actions_response.attachments is not None + assert len(quick_actions_response.attachments) > 0 + logger.info("✅ Quick actions response formatting successful") + + logger.info("🎉 All Teams response adapter tests passed!") + + except Exception as e: + logger.error(f"❌ Teams response adapter test failed: {e}") + raise + + +async def test_adaptive_card_configuration(): + """Test adaptive card configuration options.""" + logger.info("Testing Adaptive Card Configuration...") + + try: + # Test default configuration + default_config = TeamsCardConfig() + assert default_config.show_sources == True + assert default_config.show_citations == True + assert default_config.show_thoughts == False + assert default_config.show_usage == False + assert default_config.max_sources == 3 + assert default_config.max_citations == 3 + assert default_config.max_thoughts == 2 + assert default_config.include_actions == True + logger.info("✅ Default configuration test passed") + + # Test custom configuration + custom_config = TeamsCardConfig( + show_sources=False, + show_citations=True, + show_thoughts=True, + show_usage=True, + max_sources=5, + max_citations=2, + max_thoughts=1, + include_actions=False + ) + assert custom_config.show_sources == False + assert custom_config.show_citations == True + assert custom_config.show_thoughts == True + assert custom_config.show_usage == True + assert custom_config.max_sources == 5 + assert custom_config.max_citations == 2 + assert custom_config.max_thoughts == 1 + assert custom_config.include_actions == False + logger.info("✅ Custom configuration test passed") + + logger.info("🎉 All adaptive card configuration tests passed!") + + except Exception as e: + logger.error(f"❌ Adaptive card configuration test failed: {e}") + raise + + +async def test_teams_integration(): + """Test complete Teams integration.""" + logger.info("Testing Complete Teams Integration...") + + try: + # Load configuration + config = AgentConfig.from_environment() + + # Set dummy values for testing + config.app_id = "test-app-id" + config.app_password = "test-app-password" + config.tenant_id = "test-tenant-id" + config.client_id = "test-client-id" + config.client_secret = "test-client-secret" + + config.validate() + + # Initialize RAG service + rag_service = RAGService(config) + await rag_service.initialize() + + # Test RAG service with Teams-specific request + request = RAGRequest( + message="What are the main benefits mentioned in the policy document?", + user_id="teams_user_1", + channel_id="msteams", + conversation_history=[] + ) + + # Test non-streaming response + response = await rag_service.process_query(request) + assert response.answer + assert len(response.sources) > 0 or len(response.citations) > 0 + logger.info("✅ RAG service integration test passed") + + # Test streaming response + stream_response = "" + async for chunk in rag_service.process_query_stream(request): + if chunk.get("type") == "content": + stream_response += chunk.get("content", "") + elif chunk.get("type") == "error": + raise Exception(f"Streaming error: {chunk.get('content')}") + + assert stream_response + logger.info("✅ RAG service streaming integration test passed") + + # Test Teams response formatting with real RAG response + activity = Activity( + type=ActivityTypes.message, + text="What are the main benefits?", + from_property={"id": "teams_user_1", "name": "Teams User"}, + recipient={"id": "bot1", "name": "RAG Assistant"}, + conversation={"id": "teams_conv_1"} + ) + + turn_context = MockTurnContext(activity) + adapter = TeamsResponseAdapter() + + teams_response = adapter.format_rag_response(turn_context, response) + assert teams_response.attachments is not None + assert len(teams_response.attachments) > 0 + assert teams_response.attachments[0].content_type == "application/vnd.microsoft.card.adaptive" + logger.info("✅ Teams response formatting integration test passed") + + await rag_service.close() + logger.info("🎉 All Teams integration tests passed!") + + except Exception as e: + logger.error(f"❌ Teams integration test failed: {e}") + raise + + +async def main(): + """Run all Teams integration tests.""" + logger.info("Starting Teams integration tests...") + + try: + # Test 1: Teams Components + await test_teams_components() + + # Test 2: Teams Response Adapter + await test_teams_response_adapter() + + # Test 3: Adaptive Card Configuration + await test_adaptive_card_configuration() + + # Test 4: Complete Integration + await test_teams_integration() + + logger.info("🎉 All Teams integration tests completed successfully!") + + except Exception as e: + logger.error(f"❌ Teams integration tests failed: {e}") + raise + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/agents/services/auth_service.py b/agents/services/auth_service.py new file mode 100644 index 0000000000..364e4a877e --- /dev/null +++ b/agents/services/auth_service.py @@ -0,0 +1,693 @@ +""" +Authentication Service for Microsoft 365 Agent. +This service handles authentication and authorization for the agent. +""" + +import logging +import json +from typing import Dict, Any, Optional, List +from dataclasses import dataclass +from datetime import datetime, timedelta + +from botbuilder.core import TurnContext +from azure.identity.aio import DefaultAzureCredential +from msal import ConfidentialClientApplication +import aiohttp + +from config.agent_config import AgentConfig + + +logger = logging.getLogger(__name__) + + +@dataclass +class UserClaims: + """User claims from Microsoft 365 authentication.""" + user_id: str + user_name: str + email: str + tenant_id: str + groups: List[str] + roles: List[str] + additional_claims: Dict[str, Any] + access_token: Optional[str] = None + refresh_token: Optional[str] = None + token_expires_at: Optional[datetime] = None + is_authenticated: bool = False + last_updated: Optional[datetime] = None + + +@dataclass +class GraphUserInfo: + """Microsoft Graph user information.""" + id: str + display_name: str + mail: str + user_principal_name: str + job_title: Optional[str] = None + department: Optional[str] = None + office_location: Optional[str] = None + mobile_phone: Optional[str] = None + business_phones: List[str] = None + preferred_language: Optional[str] = None + + +@dataclass +class GraphGroupInfo: + """Microsoft Graph group information.""" + id: str + display_name: str + description: Optional[str] = None + group_types: List[str] = None + security_enabled: bool = False + + +class AuthService: + """ + Authentication Service that handles Microsoft 365 authentication and authorization. + Integrates with existing authentication system while providing agent-specific functionality. + """ + + def __init__(self, config: AgentConfig): + self.config = config + self._credential = DefaultAzureCredential() + self._msal_app: Optional[ConfidentialClientApplication] = None + self._http_session: Optional[aiohttp.ClientSession] = None + self._token_cache: Dict[str, UserClaims] = {} + self._graph_base_url = "https://graph.microsoft.com/v1.0" + + async def initialize(self) -> None: + """Initialize the authentication service.""" + try: + # Initialize MSAL application for token validation + self._msal_app = ConfidentialClientApplication( + client_id=self.config.client_id, + client_credential=self.config.client_secret, + authority=f"https://login.microsoftonline.com/{self.config.tenant_id}" + ) + + # Initialize HTTP session for Microsoft Graph calls + self._http_session = aiohttp.ClientSession( + timeout=aiohttp.ClientTimeout(total=30), + headers={ + "User-Agent": "Microsoft365Agent/1.0" + } + ) + + logger.info("Auth Service initialized successfully") + + except Exception as e: + logger.error(f"Failed to initialize Auth Service: {e}") + raise + + async def get_user_claims(self, turn_context: TurnContext) -> Dict[str, Any]: + """ + Get user claims from the turn context. + This method extracts user information from the Microsoft 365 context. + """ + try: + # Extract basic user information from the turn context + user_id = turn_context.activity.from_property.id + user_name = turn_context.activity.from_property.name or "Unknown User" + + # For Teams, we can get additional user information + if turn_context.activity.channel_id == "msteams": + try: + # Get Teams user information + teams_info = await self._get_teams_user_info(turn_context) + user_name = teams_info.get("name", user_name) + email = teams_info.get("email", "") + except Exception as e: + logger.warning(f"Could not get Teams user info: {e}") + email = "" + else: + email = "" + + # Create basic claims structure + claims = { + "oid": user_id, + "name": user_name, + "email": email, + "tenant_id": self.config.tenant_id, + "groups": [], # Will be populated if needed + "roles": [], # Will be populated if needed + "channel_id": turn_context.activity.channel_id, + "conversation_id": turn_context.activity.conversation.id + } + + return claims + + except Exception as e: + logger.error(f"Error getting user claims: {e}") + # Return minimal claims on error + return { + "oid": turn_context.activity.from_property.id, + "name": "Unknown User", + "email": "", + "tenant_id": self.config.tenant_id, + "groups": [], + "roles": [], + "channel_id": turn_context.activity.channel_id, + "conversation_id": turn_context.activity.conversation.id + } + + async def _get_teams_user_info(self, turn_context: TurnContext) -> Dict[str, Any]: + """Get Teams-specific user information.""" + try: + # This would typically use TeamsInfo to get user details + # For now, we'll return basic information + return { + "name": turn_context.activity.from_property.name or "Unknown User", + "email": "", + "id": turn_context.activity.from_property.id + } + except Exception as e: + logger.warning(f"Could not get Teams user info: {e}") + return { + "name": "Unknown User", + "email": "", + "id": turn_context.activity.from_property.id + } + + async def validate_user_access(self, user_claims: Dict[str, Any], resource: str) -> bool: + """ + Validate if a user has access to a specific resource. + This method integrates with existing access control logic. + """ + try: + # For now, we'll implement basic access control + # In the next phase, we'll integrate with the existing authentication system + + # Check if user is authenticated + if not user_claims.get("oid"): + return False + + # Check if user belongs to the correct tenant + if user_claims.get("tenant_id") != self.config.tenant_id: + return False + + # Additional access control logic can be added here + # For example, checking groups, roles, or specific permissions + + return True + + except Exception as e: + logger.error(f"Error validating user access: {e}") + return False + + async def get_user_groups(self, user_claims: Dict[str, Any]) -> list[str]: + """ + Get user groups for access control. + This method can be extended to integrate with Microsoft Graph. + """ + try: + # For now, return empty list + # In the next phase, we'll integrate with Microsoft Graph to get actual groups + return user_claims.get("groups", []) + + except Exception as e: + logger.error(f"Error getting user groups: {e}") + return [] + + async def get_user_roles(self, user_claims: Dict[str, Any]) -> list[str]: + """ + Get user roles for access control. + This method can be extended to integrate with Microsoft Graph. + """ + try: + # For now, return empty list + # In the next phase, we'll integrate with Microsoft Graph to get actual roles + return user_claims.get("roles", []) + + except Exception as e: + logger.error(f"Error getting user roles: {e}") + return [] + + async def get_access_token(self, scopes: Optional[List[str]] = None) -> Optional[str]: + """ + Get an access token for Microsoft Graph API calls. + """ + try: + if scopes is None: + scopes = ["https://graph.microsoft.com/.default"] + + if not self._msal_app: + logger.error("MSAL app not initialized") + return None + + result = self._msal_app.acquire_token_for_client(scopes=scopes) + + if "access_token" in result: + return result["access_token"] + else: + logger.error(f"Failed to acquire token: {result.get('error_description')}") + return None + + except Exception as e: + logger.error(f"Error acquiring access token: {e}") + return None + + async def get_obo_token( + self, + user_token: str, + scopes: Optional[List[str]] = None + ) -> Optional[str]: + """ + Exchange a user token for an On-Behalf-Of (OBO) token. + This allows the bot to call backend APIs on behalf of the user. + + Args: + user_token: The user's access token from Teams + scopes: Optional scopes to request. Defaults to User.Read and offline_access + + Returns: + OBO token string or None if exchange fails + """ + try: + if not self._msal_app: + logger.error("MSAL app not initialized") + return None + + if scopes is None: + scopes = ["User.Read", "offline_access"] + + # Use MSAL to exchange user token for OBO token + result = self._msal_app.acquire_token_on_behalf_of( + user_assertion=user_token, + scopes=scopes + ) + + if "access_token" in result: + logger.info("Successfully acquired OBO token") + return result["access_token"] + else: + error = result.get("error", "unknown") + error_desc = result.get("error_description", "No description") + logger.error(f"Failed to acquire OBO token: {error} - {error_desc}") + return None + + except Exception as e: + logger.error(f"Error acquiring OBO token: {e}", exc_info=True) + return None + + def extract_user_token_from_activity(self, turn_context: TurnContext) -> Optional[str]: + """ + Extract user token from Teams activity. + Teams SSO tokens are typically in channel_data or via OAuthPrompt. + + Args: + turn_context: The turn context from Teams + + Returns: + User token string or None if not found + """ + try: + # Check for token in channel_data (Teams SSO) + channel_data = turn_context.activity.channel_data + if channel_data and isinstance(channel_data, dict): + # Teams may include token in channel_data + token = channel_data.get("token") or channel_data.get("ssoToken") + if token: + return token + + # Check for token in activity.value (for OAuth prompt responses) + activity_value = turn_context.activity.value + if activity_value and isinstance(activity_value, dict): + token = activity_value.get("token") or activity_value.get("accessToken") + if token: + return token + + # Check in activity properties (for future Teams SSO implementations) + if hasattr(turn_context.activity, "token_response"): + token_response = turn_context.activity.token_response + if token_response and isinstance(token_response, dict): + token = token_response.get("token") or token_response.get("access_token") + if token: + return token + + logger.debug("No user token found in activity") + return None + + except Exception as e: + logger.warning(f"Error extracting user token: {e}") + return None + + async def get_obo_user_claims(self, turn_context: TurnContext) -> Optional[Dict[str, Any]]: + """ + Get user claims using OBO token exchange. + This is the preferred method for Teams SSO. + + Args: + turn_context: The turn context from Teams + + Returns: + User claims dictionary or None if OBO exchange fails + """ + try: + # Extract user token from Teams activity + user_token = self.extract_user_token_from_activity(turn_context) + + if not user_token: + logger.debug("No user token found, cannot perform OBO exchange") + return None + + # Exchange user token for OBO token + obo_token = await self.get_obo_token(user_token) + + if not obo_token: + logger.warning("OBO token exchange failed, falling back to basic claims") + return await self.get_user_claims(turn_context) + + # Use OBO token to get user info from Graph + # For now, we'll validate the token and extract claims + # In production, you might want to decode the JWT token to get claims + try: + # Get user ID from turn context + user_id = turn_context.activity.from_property.id + + # Get enhanced claims using OBO token + access_token = await self.get_access_token() + if access_token: + graph_user = await self.get_user_from_graph(user_id, access_token) + groups = await self.get_user_groups_from_graph(user_id, access_token) + + if graph_user: + return { + "oid": user_id, + "name": graph_user.display_name, + "email": graph_user.mail or graph_user.user_principal_name, + "tenant_id": self.config.tenant_id, + "groups": [g.id for g in groups], + "roles": [], + "channel_id": turn_context.activity.channel_id, + "conversation_id": turn_context.activity.conversation.id, + "obo_token": obo_token, # Include OBO token for backend calls + "is_authenticated": True + } + except Exception as e: + logger.warning(f"Error getting user info with OBO token: {e}") + + # Fallback to basic claims with OBO token + return { + "oid": turn_context.activity.from_property.id, + "name": turn_context.activity.from_property.name or "Unknown User", + "email": "", + "tenant_id": self.config.tenant_id, + "groups": [], + "roles": [], + "channel_id": turn_context.activity.channel_id, + "conversation_id": turn_context.activity.conversation.id, + "obo_token": obo_token, + "is_authenticated": True + } + + except Exception as e: + logger.error(f"Error getting OBO user claims: {e}", exc_info=True) + return None + + async def get_user_from_graph(self, user_id: str, access_token: str) -> Optional[GraphUserInfo]: + """ + Get user information from Microsoft Graph. + """ + try: + if not self._http_session: + logger.error("HTTP session not initialized") + return None + + url = f"{self._graph_base_url}/users/{user_id}" + headers = { + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json" + } + + async with self._http_session.get(url, headers=headers) as response: + if response.status == 200: + data = await response.json() + return GraphUserInfo( + id=data.get("id", ""), + display_name=data.get("displayName", ""), + mail=data.get("mail", ""), + user_principal_name=data.get("userPrincipalName", ""), + job_title=data.get("jobTitle"), + department=data.get("department"), + office_location=data.get("officeLocation"), + mobile_phone=data.get("mobilePhone"), + business_phones=data.get("businessPhones", []), + preferred_language=data.get("preferredLanguage") + ) + else: + logger.error(f"Failed to get user from Graph: {response.status}") + return None + + except Exception as e: + logger.error(f"Error getting user from Graph: {e}") + return None + + async def get_user_groups_from_graph(self, user_id: str, access_token: str) -> List[GraphGroupInfo]: + """ + Get user groups from Microsoft Graph. + """ + try: + if not self._http_session: + logger.error("HTTP session not initialized") + return [] + + url = f"{self._graph_base_url}/users/{user_id}/memberOf" + headers = { + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json" + } + + groups = [] + async with self._http_session.get(url, headers=headers) as response: + if response.status == 200: + data = await response.json() + for item in data.get("value", []): + if item.get("@odata.type") == "#microsoft.graph.group": + groups.append(GraphGroupInfo( + id=item.get("id", ""), + display_name=item.get("displayName", ""), + description=item.get("description"), + group_types=item.get("groupTypes", []), + security_enabled=item.get("securityEnabled", False) + )) + else: + logger.error(f"Failed to get user groups from Graph: {response.status}") + + return groups + + except Exception as e: + logger.error(f"Error getting user groups from Graph: {e}") + return [] + + async def get_user_roles_from_graph(self, user_id: str, access_token: str) -> List[str]: + """ + Get user roles from Microsoft Graph (Azure AD roles). + """ + try: + if not self._http_session: + logger.error("HTTP session not initialized") + return [] + + url = f"{self._graph_base_url}/users/{user_id}/memberOf" + headers = { + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json" + } + + roles = [] + async with self._http_session.get(url, headers=headers) as response: + if response.status == 200: + data = await response.json() + for item in data.get("value", []): + if item.get("@odata.type") == "#microsoft.graph.directoryRole": + roles.append(item.get("displayName", "")) + else: + logger.error(f"Failed to get user roles from Graph: {response.status}") + + return roles + + except Exception as e: + logger.error(f"Error getting user roles from Graph: {e}") + return [] + + async def validate_token(self, token: str) -> bool: + """ + Validate a user token using Microsoft Graph. + """ + try: + if not self._http_session: + logger.error("HTTP session not initialized") + return False + + url = f"{self._graph_base_url}/me" + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json" + } + + async with self._http_session.get(url, headers=headers) as response: + return response.status == 200 + + except Exception as e: + logger.error(f"Error validating token: {e}") + return False + + async def get_enhanced_user_claims(self, turn_context: TurnContext) -> UserClaims: + """ + Get enhanced user claims with Microsoft Graph data. + """ + try: + # Get basic claims first + basic_claims = await self.get_user_claims(turn_context) + user_id = basic_claims.get("oid", "") + + # Check cache first + if user_id in self._token_cache: + cached_claims = self._token_cache[user_id] + if cached_claims.last_updated and (datetime.now() - cached_claims.last_updated).seconds < 300: # 5 minutes + return cached_claims + + # Get access token for Graph calls + access_token = await self.get_access_token() + if not access_token: + logger.warning("Could not get access token for Graph calls") + return UserClaims( + user_id=user_id, + user_name=basic_claims.get("name", "Unknown User"), + email=basic_claims.get("email", ""), + tenant_id=basic_claims.get("tenant_id", ""), + groups=[], + roles=[], + additional_claims=basic_claims, + is_authenticated=False + ) + + # Get user info from Graph + graph_user = await self.get_user_from_graph(user_id, access_token) + if graph_user: + # Get groups and roles + groups = await self.get_user_groups_from_graph(user_id, access_token) + roles = await self.get_user_roles_from_graph(user_id, access_token) + + # Create enhanced claims + enhanced_claims = UserClaims( + user_id=user_id, + user_name=graph_user.display_name, + email=graph_user.mail or graph_user.user_principal_name, + tenant_id=basic_claims.get("tenant_id", ""), + groups=[group.display_name for group in groups], + roles=roles, + additional_claims={ + **basic_claims, + "graph_user": { + "job_title": graph_user.job_title, + "department": graph_user.department, + "office_location": graph_user.office_location, + "mobile_phone": graph_user.mobile_phone, + "business_phones": graph_user.business_phones, + "preferred_language": graph_user.preferred_language + }, + "graph_groups": [ + { + "id": group.id, + "display_name": group.display_name, + "description": group.description, + "security_enabled": group.security_enabled + } + for group in groups + ] + }, + access_token=access_token, + is_authenticated=True, + last_updated=datetime.now() + ) + + # Cache the claims + self._token_cache[user_id] = enhanced_claims + + return enhanced_claims + else: + # Fallback to basic claims + return UserClaims( + user_id=user_id, + user_name=basic_claims.get("name", "Unknown User"), + email=basic_claims.get("email", ""), + tenant_id=basic_claims.get("tenant_id", ""), + groups=[], + roles=[], + additional_claims=basic_claims, + is_authenticated=False + ) + + except Exception as e: + logger.error(f"Error getting enhanced user claims: {e}") + # Return basic claims on error + basic_claims = await self.get_user_claims(turn_context) + return UserClaims( + user_id=basic_claims.get("oid", ""), + user_name=basic_claims.get("name", "Unknown User"), + email=basic_claims.get("email", ""), + tenant_id=basic_claims.get("tenant_id", ""), + groups=[], + roles=[], + additional_claims=basic_claims, + is_authenticated=False + ) + + async def check_user_permission(self, user_claims: UserClaims, permission: str) -> bool: + """ + Check if user has a specific permission based on groups and roles. + """ + try: + # Define permission mappings + permission_mappings = { + "read_documents": ["Document Readers", "All Users"], + "write_documents": ["Document Writers", "Document Administrators"], + "admin_access": ["Document Administrators", "Global Administrators"], + "structural_analysis": ["Structural Engineers", "Document Readers"], + "code_review": ["Code Reviewers", "Document Writers"], + "system_admin": ["Global Administrators", "System Administrators"] + } + + required_groups = permission_mappings.get(permission, []) + if not required_groups: + logger.warning(f"Unknown permission: {permission}") + return False + + # Check if user is in any of the required groups + user_groups = user_claims.groups + user_roles = user_claims.roles + + # Check groups + for group in user_groups: + if group in required_groups: + return True + + # Check roles + for role in user_roles: + if role in required_groups: + return True + + return False + + except Exception as e: + logger.error(f"Error checking user permission: {e}") + return False + + async def close(self) -> None: + """Close the authentication service and clean up resources.""" + try: + if self._credential: + await self._credential.close() + + if self._http_session: + await self._http_session.close() + + # Clear token cache + self._token_cache.clear() + + logger.info("Auth Service closed successfully") + + except Exception as e: + logger.error(f"Error closing Auth Service: {e}") \ No newline at end of file diff --git a/agents/services/rag_service.py b/agents/services/rag_service.py new file mode 100644 index 0000000000..3feb617ad5 --- /dev/null +++ b/agents/services/rag_service.py @@ -0,0 +1,355 @@ +""" +RAG Service for Microsoft 365 Agent. +This service calls the existing backend API instead of duplicating RAG logic. +""" + +import asyncio +import logging +from typing import Dict, Any, List, Optional, AsyncGenerator, TYPE_CHECKING +from dataclasses import dataclass, field +import aiohttp +import json + +from config.agent_config import AgentConfig + +if TYPE_CHECKING: + from models.citation import Citation + + +logger = logging.getLogger(__name__) + + +@dataclass +class RAGRequest: + """Request for RAG processing.""" + message: str + conversation_history: List[Dict[str, str]] + user_id: str + channel_id: str + context: Optional[Dict[str, Any]] = None + + +@dataclass +class RAGResponse: + """Response from RAG processing.""" + answer: str + sources: List[Dict[str, Any]] + citations: List[str] # Legacy format - string citations + thoughts: List[Dict[str, Any]] + token_usage: Optional[Dict[str, int]] = None + model_info: Optional[Dict[str, str]] = None + unified_citations: Optional[List['Citation']] = None # New unified citation format + + +class RAGService: + """ + RAG Service that calls the existing backend API. + This service acts as a bridge between the Microsoft 365 Agent and the existing backend. + """ + + def __init__(self, config: AgentConfig): + self.config = config + self._http_session: Optional[aiohttp.ClientSession] = None + self._backend_url = config.backend_url + + async def initialize(self) -> None: + """Initialize the RAG service with HTTP client.""" + try: + # Initialize HTTP session for calling backend + self._http_session = aiohttp.ClientSession( + timeout=aiohttp.ClientTimeout(total=30), + headers={ + "Content-Type": "application/json", + "User-Agent": "Microsoft365Agent/1.0" + } + ) + + logger.info("RAG Service initialized successfully") + + except Exception as e: + logger.error(f"Failed to initialize RAG Service: {e}") + raise + + async def process_query(self, request: RAGRequest) -> RAGResponse: + """ + Process a RAG query by calling the existing backend API. + """ + try: + if not self._http_session: + await self.initialize() + + # Convert conversation history to the format expected by backend + messages = self._format_messages(request.message, request.conversation_history) + + # Create context for the RAG processing + context = { + "auth_claims": {"oid": request.user_id}, + "channel_id": request.channel_id, + **(request.context or {}) + } + + # Call the existing backend /chat endpoint + response = await self._call_backend_chat(messages, context) + + return response + + except Exception as e: + logger.error(f"Error processing RAG query: {e}") + return RAGResponse( + answer="I'm sorry, I encountered an error processing your request. Please try again.", + sources=[], + citations=[], + thoughts=[{"title": "Error", "description": str(e)}] + ) + + async def process_query_stream(self, request: RAGRequest) -> AsyncGenerator[Dict[str, Any], None]: + """ + Process a RAG query with streaming response by calling backend. + """ + try: + if not self._http_session: + await self.initialize() + + # Convert conversation history to the format expected by backend + messages = self._format_messages(request.message, request.conversation_history) + + # Create context for the RAG processing + context = { + "auth_claims": {"oid": request.user_id}, + "channel_id": request.channel_id, + **(request.context or {}) + } + + # Stream the response from backend + async for chunk in self._call_backend_chat_stream(messages, context): + yield chunk + + except Exception as e: + logger.error(f"Error processing streaming RAG query: {e}") + yield { + "type": "error", + "content": "I'm sorry, I encountered an error processing your request. Please try again.", + "error": str(e) + } + + def _format_messages(self, message: str, conversation_history: List[Dict[str, str]]) -> List[Dict[str, str]]: + """Format messages for the RAG system.""" + messages = [] + + # Add conversation history + for msg in conversation_history: + messages.append({ + "role": msg.get("role", "user"), + "content": msg.get("content", "") + }) + + # Add current message + messages.append({ + "role": "user", + "content": message + }) + + return messages + + async def _call_backend_chat(self, messages: List[Dict[str, str]], context: Dict[str, Any]) -> RAGResponse: + """ + Call the existing backend /chat endpoint. + """ + try: + # Prepare the request payload + payload = { + "messages": messages, + "context": context, + "session_state": None # Will be managed by the agent + } + + # For webchat/local testing, we may not have auth tokens + # The backend will handle this via get_auth_claims_if_enabled + headers = {} + if "auth_claims" in context and context["auth_claims"].get("access_token"): + headers["Authorization"] = f"Bearer {context['auth_claims']['access_token']}" + # Propagate correlation id if present + if context.get("traceparent"): + headers["x-traceparent"] = str(context["traceparent"]) # simple correlation header + + # Make the request to the backend + async with self._http_session.post( + f"{self._backend_url}/chat", + json=payload, + headers=headers + ) as response: + if response.status == 200: + data = await response.json() + + # Force print for debugging (always visible) + print(f"\n{'='*60}") + print(f"[RAG SERVICE] Backend response received") + print(f"[RAG SERVICE] Response keys: {list(data.keys())}") + if "message" in data: + print(f"[RAG SERVICE] Message type: {type(data['message'])}") + if isinstance(data.get('message'), dict): + print(f"[RAG SERVICE] Message keys: {list(data['message'].keys())}") + print(f"[RAG SERVICE] Message content preview: {str(data['message'].get('content', ''))[:100]}") + + # Backend returns: { "message": { "content": "...", "role": "..." }, "context": { "data_points": {...}, "thoughts": [...] } } + # Extract answer from message.content + answer = "" + if "message" in data and isinstance(data["message"], dict): + answer = data["message"].get("content", "") + print(f"[RAG SERVICE] ✓ Extracted answer from message.content (length: {len(answer)})") + logger.info(f"Extracted answer from message.content: {answer[:100]}...") + elif "answer" in data: + # Fallback for different response format + answer = data.get("answer", "") + print(f"[RAG SERVICE] ✓ Extracted answer from answer field (length: {len(answer)})") + logger.info(f"Extracted answer from answer field: {answer[:100]}...") + else: + print(f"[RAG SERVICE] ✗ Could not find answer! Available keys: {list(data.keys())}") + logger.warning(f"Could not find answer in response. Available keys: {list(data.keys())}") + # Try to find any content field + if "content" in data: + answer = data.get("content", "") + print(f"[RAG SERVICE] ✓ Found answer in content field (length: {len(answer)})") + print(f"{'='*60}\n") + + # Extract data points from context + context = data.get("context", {}) + data_points = context.get("data_points", {}) + text_sources = data_points.get("text", []) + citations = data_points.get("citations", []) + thoughts = context.get("thoughts", []) + + # Convert backend citations to unified format + unified_citations = self._convert_to_unified_citations( + text_sources, + citations + ) + + # Convert backend response to RAGResponse + return RAGResponse( + answer=answer, + sources=text_sources, + citations=citations, + thoughts=thoughts, + token_usage=data.get("token_usage"), + model_info=data.get("model_info"), + unified_citations=unified_citations + ) + else: + error_text = await response.text() + logger.error(f"Backend API error {response.status}: {error_text}") + raise Exception(f"Backend API error: {response.status}") + + except Exception as e: + logger.error(f"Error calling backend chat API: {e}") + raise + + def _convert_to_unified_citations( + self, + sources: List[Dict[str, Any]], + citations: List[str] + ) -> List['Citation']: + """ + Convert backend citations to unified Citation format. + + Args: + sources: List of source documents from backend + citations: List of citation strings from backend + + Returns: + List of unified Citation objects + """ + try: + from models.citation import Citation, CitationSource, CitationProvider + + unified: List[Citation] = [] + + # Convert sources (corpus sources) + for source in sources: + if isinstance(source, dict): + try: + citation = Citation.from_azure_search( + doc=source, + snippet=source.get("content", ""), + confidence=1.0 + ) + unified.append(citation) + except Exception as e: + logger.warning(f"Error converting source to citation: {e}") + + # Convert citation strings (if any) + for citation_str in citations: + if citation_str: + # Try to parse citation string + # Format may vary, attempt to extract URL + import re + url_match = re.search(r'https?://[^\s<>"\']+', citation_str) + if url_match: + try: + citation = Citation( + source=CitationSource.WEB, + provider=CitationProvider.UNKNOWN, + url=url_match.group(0), + title=citation_str[:100], # Use citation string as title + snippet=citation_str, + confidence=0.8 # Lower confidence for string citations + ) + unified.append(citation) + except Exception as e: + logger.warning(f"Error converting citation string: {e}") + + return unified + + except ImportError: + logger.warning("Citation model not available, skipping unified citation conversion") + return [] + except Exception as e: + logger.error(f"Error converting to unified citations: {e}", exc_info=True) + return [] + + async def _call_backend_chat_stream(self, messages: List[Dict[str, str]], context: Dict[str, Any]) -> AsyncGenerator[Dict[str, Any], None]: + """ + Call the existing backend /chat/stream endpoint. + """ + try: + # Prepare the request payload + payload = { + "messages": messages, + "context": context, + "session_state": None # Will be managed by the agent + } + + # Make the streaming request to the backend + async with self._http_session.post( + f"{self._backend_url}/chat/stream", + json=payload + ) as response: + if response.status == 200: + async for line in response.content: + if line: + try: + # Parse NDJSON line + chunk_data = json.loads(line.decode('utf-8')) + yield chunk_data + except json.JSONDecodeError: + # Skip invalid JSON lines + continue + else: + error_text = await response.text() + logger.error(f"Backend streaming API error {response.status}: {error_text}") + yield { + "type": "error", + "content": f"Backend API error: {response.status}" + } + + except Exception as e: + logger.error(f"Error calling backend streaming API: {e}") + yield { + "type": "error", + "content": f"Error: {str(e)}" + } + + async def close(self) -> None: + """Close the RAG service and clean up resources.""" + if self._http_session: + await self._http_session.close() \ No newline at end of file diff --git a/agents/startup.sh b/agents/startup.sh new file mode 100644 index 0000000000..3426c6d6d3 --- /dev/null +++ b/agents/startup.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# Startup script for Azure App Service +python main.py + diff --git a/agents/tests/README.md b/agents/tests/README.md new file mode 100644 index 0000000000..b6af66ef4d --- /dev/null +++ b/agents/tests/README.md @@ -0,0 +1,183 @@ +# Microsoft 365 RAG Agent Tests + +This directory contains comprehensive pytest-based tests for the Microsoft 365 RAG Agent. + +## Test Structure + +### Test Files + +- **`test_teams_components.py`** - Tests for Teams UI components and Adaptive Cards +- **`test_teams_response_adapter.py`** - Tests for Teams response formatting and adaptation +- **`test_teams_handler.py`** - Tests for Teams message handling and adaptive card actions + +### Test Categories + +- **Unit Tests** - Test individual components in isolation with mocks +- **Integration Tests** - Test component interactions (marked with `@pytest.mark.integration`) +- **Async Tests** - Test asynchronous functionality with `@pytest.mark.asyncio` + +## Running Tests + +### Prerequisites + +Install the required testing dependencies: + +```bash +# Using system packages (Ubuntu/Debian) +sudo apt install python3-pytest python3-pytest-asyncio python3-pytest-cov python3-pytest-mock + +# Or using pip (if virtual environment is available) +pip install pytest pytest-asyncio pytest-cov pytest-mock +``` + +### Running All Tests + +```bash +# Run all tests +python3 -m pytest tests/ -v + +# Run with coverage +python3 -m pytest tests/ --cov=. --cov-report=html + +# Run specific test file +python3 -m pytest tests/test_teams_components.py -v + +# Run specific test +python3 -m pytest tests/test_teams_components.py::TestTeamsComponents::test_create_welcome_card -v +``` + +### Test Configuration + +The tests use `pytest.ini` for configuration: + +```ini +[tool:pytest] +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* +addopts = + -v + --tb=short + --strict-markers + --disable-warnings + --asyncio-mode=auto +markers = + integration: marks tests as integration tests (deselect with '-m "not integration"') + unit: marks tests as unit tests + slow: marks tests as slow running +``` + +## Test Design Principles + +### 1. **No External Dependencies** +- All tests use mocks instead of hitting real Azure services +- No network calls or external API dependencies +- Tests run fast and reliably in any environment + +### 2. **Comprehensive Coverage** +- Test all public methods and edge cases +- Test error handling and exception scenarios +- Test both success and failure paths + +### 3. **Proper Async Testing** +- Use `@pytest.mark.asyncio` for async test methods +- Properly await async methods in tests +- Mock async dependencies correctly + +### 4. **Mock Strategy** +- Mock external dependencies (RAG service, auth service, etc.) +- Use `unittest.mock.Mock` and `unittest.mock.patch` +- Create realistic mock data that matches expected interfaces + +## Example Test Structure + +```python +import pytest +from unittest.mock import Mock, patch +from components.teams_components import TeamsComponents + +class TestTeamsComponents: + """Test cases for TeamsComponents.""" + + def test_create_welcome_card(self): + """Test welcome card creation.""" + card = TeamsComponents.create_welcome_card() + + assert card["type"] == "AdaptiveCard" + assert card["version"] == "1.4" + assert len(card["body"]) > 0 + assert len(card["actions"]) > 0 + + @pytest.mark.asyncio + async def test_async_method(self, mock_dependency): + """Test async method with mocked dependency.""" + with patch.object(mock_dependency, 'method', return_value="test"): + result = await some_async_method() + assert result == "expected" +``` + +## Mock Data Patterns + +### RAG Response Mock +```python +mock_rag_response = RAGResponse( + answer="Test response", + sources=[{"title": "Source 1", "url": "https://example.com"}], + citations=["Citation 1"], + thoughts=[{"title": "Thought 1", "description": "Description 1"}], + token_usage={"total_tokens": 100}, + model_info={"model": "gpt-4"} +) +``` + +### Turn Context Mock +```python +class MockTurnContext: + def __init__(self, activity: Activity): + self.activity = activity + self.channel_id = "msteams" + self.conversation = activity.conversation + self.from_property = activity.from_property + self.recipient = Mock() + self.recipient.id = "bot1" +``` + +## Continuous Integration + +These tests are designed to run in CI/CD pipelines: + +- No external dependencies or network calls +- Fast execution (all tests complete in ~1-2 seconds) +- Reliable and deterministic results +- Proper error reporting and logging + +## Coverage Goals + +- **Unit Tests**: 100% coverage of core business logic +- **Integration Tests**: Cover all major component interactions +- **Error Handling**: Test all exception scenarios +- **Edge Cases**: Test boundary conditions and unusual inputs + +## Debugging Tests + +### Verbose Output +```bash +python3 -m pytest tests/ -v -s +``` + +### Stop on First Failure +```bash +python3 -m pytest tests/ -x +``` + +### Run Specific Test with Debug +```bash +python3 -m pytest tests/test_teams_components.py::TestTeamsComponents::test_create_welcome_card -v -s --tb=long +``` + +### Coverage Report +```bash +python3 -m pytest tests/ --cov=. --cov-report=html +# Open htmlcov/index.html in browser +``` \ No newline at end of file diff --git a/agents/tests/__init__.py b/agents/tests/__init__.py new file mode 100644 index 0000000000..88609ab965 --- /dev/null +++ b/agents/tests/__init__.py @@ -0,0 +1,3 @@ +""" +Test package for Microsoft 365 RAG Agent. +""" \ No newline at end of file diff --git a/agents/tests/test_auth_service.py b/agents/tests/test_auth_service.py new file mode 100644 index 0000000000..b217421a18 --- /dev/null +++ b/agents/tests/test_auth_service.py @@ -0,0 +1,470 @@ +""" +Pytest tests for authentication service functionality. +Tests Microsoft 365 authentication and authorization features. +""" + +import pytest +from unittest.mock import Mock, patch, AsyncMock +from datetime import datetime, timedelta +from botbuilder.schema import Activity, ActivityTypes + +from services.auth_service import AuthService, UserClaims, GraphUserInfo, GraphGroupInfo +from config.agent_config import AgentConfig + + +class MockTurnContext: + """Mock TurnContext for testing.""" + + def __init__(self, activity: Activity): + self.activity = activity + self.activity.channel_id = "msteams" # Set channel_id on activity + self.channel_id = "msteams" + self.conversation = activity.conversation + self.from_property = activity.from_property + self.recipient = activity.recipient + + +class TestAuthService: + """Test cases for AuthService.""" + + @pytest.fixture + def mock_config(self): + """Create a mock AgentConfig for testing.""" + config = Mock(spec=AgentConfig) + config.tenant_id = "test-tenant-id" + config.client_id = "test-client-id" + config.client_secret = "test-client-secret" + return config + + @pytest.fixture + def auth_service(self, mock_config): + """Create an AuthService instance for testing.""" + return AuthService(mock_config) + + @pytest.fixture + def mock_turn_context(self): + """Create a mock TurnContext for testing.""" + from_property = Mock() + from_property.id = "user1" + from_property.name = "Test User" + + conversation = Mock() + conversation.id = "conv1" + + activity = Activity( + type=ActivityTypes.message, + text="Test message", + from_property=from_property, + recipient={"id": "bot1", "name": "RAG Assistant"}, + conversation=conversation + ) + return MockTurnContext(activity) + + @pytest.fixture + def mock_graph_user(self): + """Create mock Graph user data.""" + return { + "id": "user1", + "displayName": "Test User", + "mail": "test@example.com", + "userPrincipalName": "test@example.com", + "jobTitle": "Structural Engineer", + "department": "Engineering", + "officeLocation": "Seattle", + "mobilePhone": "+1-555-0123", + "businessPhones": ["+1-555-0124"], + "preferredLanguage": "en-US" + } + + @pytest.fixture + def mock_graph_groups(self): + """Create mock Graph groups data.""" + return { + "value": [ + { + "@odata.type": "#microsoft.graph.group", + "id": "group1", + "displayName": "Structural Engineers", + "description": "Structural engineering team", + "groupTypes": [], + "securityEnabled": True + }, + { + "@odata.type": "#microsoft.graph.group", + "id": "group2", + "displayName": "Document Readers", + "description": "Users who can read documents", + "groupTypes": [], + "securityEnabled": True + } + ] + } + + @pytest.fixture + def mock_graph_roles(self): + """Create mock Graph roles data.""" + return { + "value": [ + { + "@odata.type": "#microsoft.graph.directoryRole", + "id": "role1", + "displayName": "Global Administrator" + } + ] + } + + def test_user_claims_creation(self): + """Test UserClaims dataclass creation.""" + claims = UserClaims( + user_id="user1", + user_name="Test User", + email="test@example.com", + tenant_id="tenant1", + groups=["Group1", "Group2"], + roles=["Role1"], + additional_claims={"key": "value"}, + is_authenticated=True + ) + + assert claims.user_id == "user1" + assert claims.user_name == "Test User" + assert claims.email == "test@example.com" + assert claims.tenant_id == "tenant1" + assert claims.groups == ["Group1", "Group2"] + assert claims.roles == ["Role1"] + assert claims.is_authenticated is True + + def test_graph_user_info_creation(self): + """Test GraphUserInfo dataclass creation.""" + user_info = GraphUserInfo( + id="user1", + display_name="Test User", + mail="test@example.com", + user_principal_name="test@example.com", + job_title="Engineer", + department="Engineering" + ) + + assert user_info.id == "user1" + assert user_info.display_name == "Test User" + assert user_info.mail == "test@example.com" + assert user_info.job_title == "Engineer" + assert user_info.department == "Engineering" + + def test_graph_group_info_creation(self): + """Test GraphGroupInfo dataclass creation.""" + group_info = GraphGroupInfo( + id="group1", + display_name="Test Group", + description="Test description", + group_types=["Unified"], + security_enabled=True + ) + + assert group_info.id == "group1" + assert group_info.display_name == "Test Group" + assert group_info.description == "Test description" + assert group_info.group_types == ["Unified"] + assert group_info.security_enabled is True + + @pytest.mark.skip(reason="MSAL initialization requires network calls that are difficult to mock") + @pytest.mark.asyncio + async def test_initialize(self, auth_service): + """Test authentication service initialization.""" + with patch('msal.ConfidentialClientApplication') as mock_msal: + with patch('aiohttp.ClientSession') as mock_session: + with patch('azure.identity.aio.DefaultAzureCredential') as mock_credential: + # Mock the MSAL app to avoid network calls + mock_msal_instance = Mock() + mock_msal.return_value = mock_msal_instance + + # Mock the credential + mock_credential_instance = AsyncMock() + mock_credential.return_value = mock_credential_instance + + # Mock the session + mock_session_instance = Mock() + mock_session.return_value = mock_session_instance + + await auth_service.initialize() + + mock_msal.assert_called_once() + mock_session.assert_called_once() + assert auth_service._msal_app == mock_msal_instance + + @pytest.mark.asyncio + async def test_get_user_claims(self, auth_service, mock_turn_context): + """Test getting basic user claims.""" + claims = await auth_service.get_user_claims(mock_turn_context) + + assert claims["oid"] == "user1" + assert claims["name"] == "Test User" + assert claims["tenant_id"] == "test-tenant-id" + assert claims["channel_id"] == "msteams" + assert claims["conversation_id"] == "conv1" + + @pytest.mark.asyncio + async def test_get_access_token(self, auth_service): + """Test getting access token.""" + with patch.object(auth_service, '_msal_app') as mock_msal: + mock_msal.acquire_token_for_client.return_value = { + "access_token": "test-token", + "expires_in": 3600 + } + + token = await auth_service.get_access_token() + + assert token == "test-token" + mock_msal.acquire_token_for_client.assert_called_once() + + @pytest.mark.asyncio + async def test_get_access_token_failure(self, auth_service): + """Test getting access token when it fails.""" + with patch.object(auth_service, '_msal_app') as mock_msal: + mock_msal.acquire_token_for_client.return_value = { + "error": "invalid_client", + "error_description": "Invalid client" + } + + token = await auth_service.get_access_token() + + assert token is None + + @pytest.mark.asyncio + async def test_get_user_from_graph(self, auth_service, mock_graph_user): + """Test getting user from Microsoft Graph.""" + with patch.object(auth_service, '_http_session') as mock_session: + mock_response = AsyncMock() + mock_response.status = 200 + mock_response.json.return_value = mock_graph_user + mock_session.get.return_value.__aenter__.return_value = mock_response + + user_info = await auth_service.get_user_from_graph("user1", "test-token") + + assert user_info is not None + assert user_info.id == "user1" + assert user_info.display_name == "Test User" + assert user_info.mail == "test@example.com" + assert user_info.job_title == "Structural Engineer" + assert user_info.department == "Engineering" + + @pytest.mark.asyncio + async def test_get_user_from_graph_failure(self, auth_service): + """Test getting user from Graph when it fails.""" + with patch.object(auth_service, '_http_session') as mock_session: + mock_response = AsyncMock() + mock_response.status = 404 + mock_session.get.return_value.__aenter__.return_value = mock_response + + user_info = await auth_service.get_user_from_graph("user1", "test-token") + + assert user_info is None + + @pytest.mark.asyncio + async def test_get_user_groups_from_graph(self, auth_service, mock_graph_groups): + """Test getting user groups from Microsoft Graph.""" + with patch.object(auth_service, '_http_session') as mock_session: + mock_response = AsyncMock() + mock_response.status = 200 + mock_response.json.return_value = mock_graph_groups + mock_session.get.return_value.__aenter__.return_value = mock_response + + groups = await auth_service.get_user_groups_from_graph("user1", "test-token") + + assert len(groups) == 2 + assert groups[0].id == "group1" + assert groups[0].display_name == "Structural Engineers" + assert groups[0].security_enabled is True + assert groups[1].display_name == "Document Readers" + + @pytest.mark.asyncio + async def test_get_user_roles_from_graph(self, auth_service, mock_graph_roles): + """Test getting user roles from Microsoft Graph.""" + with patch.object(auth_service, '_http_session') as mock_session: + mock_response = AsyncMock() + mock_response.status = 200 + mock_response.json.return_value = mock_graph_roles + mock_session.get.return_value.__aenter__.return_value = mock_response + + roles = await auth_service.get_user_roles_from_graph("user1", "test-token") + + assert len(roles) == 1 + assert roles[0] == "Global Administrator" + + @pytest.mark.asyncio + async def test_validate_token_success(self, auth_service): + """Test token validation when successful.""" + with patch.object(auth_service, '_http_session') as mock_session: + mock_response = AsyncMock() + mock_response.status = 200 + mock_session.get.return_value.__aenter__.return_value = mock_response + + is_valid = await auth_service.validate_token("test-token") + + assert is_valid is True + + @pytest.mark.asyncio + async def test_validate_token_failure(self, auth_service): + """Test token validation when it fails.""" + with patch.object(auth_service, '_http_session') as mock_session: + mock_response = AsyncMock() + mock_response.status = 401 + mock_session.get.return_value.__aenter__.return_value = mock_response + + is_valid = await auth_service.validate_token("invalid-token") + + assert is_valid is False + + @pytest.mark.asyncio + async def test_get_enhanced_user_claims_with_cache(self, auth_service, mock_turn_context): + """Test getting enhanced user claims with cache hit.""" + # Create cached claims + cached_claims = UserClaims( + user_id="user1", + user_name="Cached User", + email="cached@example.com", + tenant_id="test-tenant-id", + groups=["Cached Group"], + roles=["Cached Role"], + additional_claims={}, + is_authenticated=True, + last_updated=datetime.now() + ) + auth_service._token_cache["user1"] = cached_claims + + claims = await auth_service.get_enhanced_user_claims(mock_turn_context) + + assert claims.user_id == "user1" + assert claims.user_name == "Cached User" + assert claims.groups == ["Cached Group"] + + @pytest.mark.asyncio + async def test_get_enhanced_user_claims_with_graph(self, auth_service, mock_turn_context, mock_graph_user, mock_graph_groups, mock_graph_roles): + """Test getting enhanced user claims with Graph data.""" + with patch.object(auth_service, 'get_access_token', return_value="test-token"): + with patch.object(auth_service, 'get_user_from_graph') as mock_get_user: + with patch.object(auth_service, 'get_user_groups_from_graph') as mock_get_groups: + with patch.object(auth_service, 'get_user_roles_from_graph') as mock_get_roles: + # Mock Graph responses + mock_get_user.return_value = GraphUserInfo( + id="user1", + display_name="Test User", + mail="test@example.com", + user_principal_name="test@example.com", + job_title="Structural Engineer", + department="Engineering" + ) + mock_get_groups.return_value = [ + GraphGroupInfo( + id="group1", + display_name="Structural Engineers", + security_enabled=True + ) + ] + mock_get_roles.return_value = ["Global Administrator"] + + claims = await auth_service.get_enhanced_user_claims(mock_turn_context) + + assert claims.user_id == "user1" + assert claims.user_name == "Test User" + assert claims.email == "test@example.com" + assert claims.groups == ["Structural Engineers"] + assert claims.roles == ["Global Administrator"] + assert claims.is_authenticated is True + assert "graph_user" in claims.additional_claims + assert "graph_groups" in claims.additional_claims + + @pytest.mark.asyncio + async def test_check_user_permission_success(self, auth_service): + """Test checking user permission when user has permission.""" + user_claims = UserClaims( + user_id="user1", + user_name="Test User", + email="test@example.com", + tenant_id="test-tenant-id", + groups=["Structural Engineers", "Document Readers"], + roles=["Global Administrators"], + additional_claims={} + ) + + # Test structural analysis permission + has_permission = await auth_service.check_user_permission(user_claims, "structural_analysis") + assert has_permission is True + + # Test admin access permission (should work because user has Global Administrator role) + has_permission = await auth_service.check_user_permission(user_claims, "admin_access") + assert has_permission is True + + @pytest.mark.asyncio + async def test_check_user_permission_failure(self, auth_service): + """Test checking user permission when user doesn't have permission.""" + user_claims = UserClaims( + user_id="user1", + user_name="Test User", + email="test@example.com", + tenant_id="test-tenant-id", + groups=["Other Group"], + roles=["Other Role"], + additional_claims={} + ) + + # Test structural analysis permission + has_permission = await auth_service.check_user_permission(user_claims, "structural_analysis") + assert has_permission is False + + # Test admin access permission + has_permission = await auth_service.check_user_permission(user_claims, "admin_access") + assert has_permission is False + + @pytest.mark.asyncio + async def test_check_user_permission_unknown(self, auth_service): + """Test checking unknown permission.""" + user_claims = UserClaims( + user_id="user1", + user_name="Test User", + email="test@example.com", + tenant_id="test-tenant-id", + groups=["Structural Engineers"], + roles=[], + additional_claims={} + ) + + has_permission = await auth_service.check_user_permission(user_claims, "unknown_permission") + assert has_permission is False + + @pytest.mark.asyncio + async def test_close(self, auth_service): + """Test closing the authentication service.""" + # Set up mock objects + mock_credential = AsyncMock() + mock_session = AsyncMock() + auth_service._credential = mock_credential + auth_service._http_session = mock_session + + await auth_service.close() + + mock_credential.close.assert_called_once() + mock_session.close.assert_called_once() + assert len(auth_service._token_cache) == 0 + + @pytest.mark.asyncio + async def test_error_handling_in_get_user_claims(self, auth_service, mock_turn_context): + """Test error handling in get_user_claims.""" + # Mock an error in the method + with patch.object(auth_service, '_get_teams_user_info', side_effect=Exception("Test error")): + claims = await auth_service.get_user_claims(mock_turn_context) + + # Should return claims with fallback values on error + assert claims["oid"] == "user1" + assert claims["name"] == "Test User" # Uses the name from turn_context + assert claims["email"] == "" # Empty email due to error + + @pytest.mark.asyncio + async def test_error_handling_in_enhanced_claims(self, auth_service, mock_turn_context): + """Test error handling in get_enhanced_user_claims.""" + with patch.object(auth_service, 'get_access_token', side_effect=Exception("Test error")): + claims = await auth_service.get_enhanced_user_claims(mock_turn_context) + + # Should return basic claims on error + assert claims.user_id == "user1" + assert claims.is_authenticated is False \ No newline at end of file diff --git a/agents/tests/test_teams_components.py b/agents/tests/test_teams_components.py new file mode 100644 index 0000000000..00305a671e --- /dev/null +++ b/agents/tests/test_teams_components.py @@ -0,0 +1,277 @@ +""" +Pytest tests for Teams components functionality. +Tests Teams-specific UI components and utilities. +""" + +import pytest +from unittest.mock import Mock, patch +from botbuilder.schema import Activity, ActivityTypes + +from components.teams_components import TeamsComponents, TeamsCardConfig +from constants.teams_text import TeamsTextConstants + + +class TestTeamsComponents: + """Test cases for TeamsComponents.""" + + def test_create_welcome_card(self): + """Test welcome card creation.""" + card = TeamsComponents.create_welcome_card() + + assert card["type"] == "AdaptiveCard" + assert card["version"] == "1.4" + assert len(card["body"]) > 0 + assert len(card["actions"]) > 0 + + # Check title contains bot name + title_text = card["body"][0]["items"][0]["text"] + assert "Welcome to" in title_text + assert "Structural Engineering Assistant" in title_text + + # Check capabilities section exists + capabilities_section = next( + (item for item in card["body"] if item.get("items", [{}])[0].get("text") == "🔧 What I can do:"), + None + ) + assert capabilities_section is not None + + # Check usage section exists + usage_section = next( + (item for item in card["body"] if item.get("items", [{}])[0].get("text") == "💡 How to use:"), + None + ) + assert usage_section is not None + + def test_create_help_card(self): + """Test help card creation.""" + card = TeamsComponents.create_help_card() + + assert card["type"] == "AdaptiveCard" + assert card["version"] == "1.4" + assert len(card["body"]) > 0 + assert len(card["actions"]) > 0 + + # Check title contains bot name + title_text = card["body"][0]["items"][0]["text"] + assert "Help" in title_text + assert "Structural Engineering Assistant" in title_text + + # Check structural analysis section + analysis_section = next( + (item for item in card["body"] if item.get("items", [{}])[0].get("text") == "📐 Structural Analysis"), + None + ) + assert analysis_section is not None + + # Check technical chat section + chat_section = next( + (item for item in card["body"] if item.get("items", [{}])[0].get("text") == "💬 Technical Chat"), + None + ) + assert chat_section is not None + + # Check example questions section + examples_section = next( + (item for item in card["body"] if item.get("items", [{}])[0].get("text") == "🔍 Example Questions"), + None + ) + assert examples_section is not None + + def test_create_error_card(self): + """Test error card creation.""" + error_message = "Test error message" + card = TeamsComponents.create_error_card(error_message) + + assert card["type"] == "AdaptiveCard" + assert card["version"] == "1.4" + assert len(card["body"]) > 0 + assert len(card["actions"]) > 0 + + # Check error message is included + error_text = card["body"][1]["text"] + assert error_message in error_text + + # Check error title + error_title = card["body"][0]["items"][0]["text"] + assert "Error" in error_title + + def test_create_loading_card(self): + """Test loading card creation.""" + card = TeamsComponents.create_loading_card() + + assert card["type"] == "AdaptiveCard" + assert card["version"] == "1.4" + assert len(card["body"]) > 0 + + # Check loading message + loading_text = card["body"][0]["items"][0]["text"] + assert "Processing your request" in loading_text + + def test_create_file_upload_card(self): + """Test file upload card creation.""" + file_name = "test.pdf" + file_type = "application/pdf" + card = TeamsComponents.create_file_upload_card(file_name, file_type) + + assert card["type"] == "AdaptiveCard" + assert card["version"] == "1.4" + assert len(card["body"]) > 0 + assert len(card["actions"]) > 0 + + # Check file name is included + file_text = card["body"][1]["text"] + assert file_name in file_text + + # Check file type is included + type_text = card["body"][2]["text"] + assert file_type in type_text + + def test_create_quick_actions_card(self): + """Test quick actions card creation.""" + card = TeamsComponents.create_quick_actions_card() + + assert card["type"] == "AdaptiveCard" + assert card["version"] == "1.4" + assert len(card["body"]) > 0 + assert len(card["actions"]) > 0 + + # Check quick actions title + title_text = card["body"][0]["items"][0]["text"] + assert "Quick Actions" in title_text + + def test_create_attachment_from_card(self): + """Test attachment creation from card JSON.""" + card_json = { + "type": "AdaptiveCard", + "version": "1.4", + "body": [{"type": "TextBlock", "text": "Test"}] + } + + attachment = TeamsComponents.create_attachment_from_card(card_json) + + assert attachment.content_type == "application/vnd.microsoft.card.adaptive" + assert attachment.content == card_json + + def test_create_suggested_actions(self): + """Test suggested actions creation.""" + actions = ["Action 1", "Action 2", "Action 3"] + suggested_actions = TeamsComponents.create_suggested_actions(actions) + + assert len(suggested_actions) == 3 + for i, action in enumerate(suggested_actions): + assert action.title == actions[i] + assert action.value == actions[i] + + def test_get_default_suggested_actions(self): + """Test default suggested actions.""" + suggested_actions = TeamsComponents.get_default_suggested_actions() + + assert len(suggested_actions) == len(TeamsTextConstants.SUGGESTED_ACTIONS) + for i, action in enumerate(suggested_actions): + assert action.title == TeamsTextConstants.SUGGESTED_ACTIONS[i] + + +class TestTeamsCardConfig: + """Test cases for TeamsCardConfig.""" + + def test_default_configuration(self): + """Test default configuration values.""" + config = TeamsCardConfig() + + assert config.show_sources == True + assert config.show_citations == True + assert config.show_thoughts == False + assert config.show_usage == False + assert config.max_sources == 3 + assert config.max_citations == 3 + assert config.max_thoughts == 2 + assert config.include_actions == True + + def test_custom_configuration(self): + """Test custom configuration values.""" + config = TeamsCardConfig( + show_sources=False, + show_citations=True, + show_thoughts=True, + show_usage=True, + max_sources=5, + max_citations=2, + max_thoughts=1, + include_actions=False + ) + + assert config.show_sources == False + assert config.show_citations == True + assert config.show_thoughts == True + assert config.show_usage == True + assert config.max_sources == 5 + assert config.max_citations == 2 + assert config.max_thoughts == 1 + assert config.include_actions == False + + +class TestTeamsTextConstants: + """Test cases for TeamsTextConstants.""" + + def test_get_bot_name(self): + """Test bot name retrieval.""" + with patch.dict('os.environ', {'AGENT_NAME': 'Test Bot'}): + assert TeamsTextConstants.get_bot_name() == 'Test Bot' + + with patch.dict('os.environ', {}, clear=True): + assert TeamsTextConstants.get_bot_name() == TeamsTextConstants.DEFAULT_BOT_NAME + + def test_get_bot_description(self): + """Test bot description retrieval.""" + with patch.dict('os.environ', {'AGENT_DESCRIPTION': 'Test Description'}): + assert TeamsTextConstants.get_bot_description() == 'Test Description' + + with patch.dict('os.environ', {}, clear=True): + assert TeamsTextConstants.get_bot_description() == TeamsTextConstants.DEFAULT_BOT_DESCRIPTION + + def test_format_welcome_title(self): + """Test welcome title formatting.""" + with patch.dict('os.environ', {'AGENT_NAME': 'Test Bot'}): + title = TeamsTextConstants.format_welcome_title() + assert 'Welcome to Test Bot' in title + assert '🏗️' in title + + def test_format_welcome_description(self): + """Test welcome description formatting.""" + with patch.dict('os.environ', {'AGENT_DESCRIPTION': 'Test Description'}): + description = TeamsTextConstants.format_welcome_description() + assert 'Test Description' in description + + def test_format_help_title(self): + """Test help title formatting.""" + with patch.dict('os.environ', {'AGENT_NAME': 'Test Bot'}): + title = TeamsTextConstants.format_help_title() + assert 'Test Bot Help' in title + assert '❓' in title + + def test_format_usage_instructions(self): + """Test usage instructions formatting.""" + with patch.dict('os.environ', {'AGENT_NAME': 'Test Bot'}): + instructions = TeamsTextConstants.format_usage_instructions() + assert len(instructions) == len(TeamsTextConstants.USAGE_INSTRUCTIONS) + # Only the first instruction should contain the bot name + assert 'Test Bot' in instructions[0] + + def test_format_mention_reminder(self): + """Test mention reminder formatting.""" + with patch.dict('os.environ', {'AGENT_NAME': 'Test Bot'}): + reminder = TeamsTextConstants.format_mention_reminder() + assert 'Test Bot' in reminder + assert '👋' in reminder + + def test_format_file_upload_message(self): + """Test file upload message formatting.""" + message = TeamsTextConstants.format_file_upload_message("test.pdf") + assert "test.pdf" in message + assert "**test.pdf**" in message + + def test_format_file_upload_type(self): + """Test file upload type formatting.""" + file_type = TeamsTextConstants.format_file_upload_type("application/pdf") + assert "application/pdf" in file_type + assert "File type:" in file_type \ No newline at end of file diff --git a/agents/tests/test_teams_handler.py b/agents/tests/test_teams_handler.py new file mode 100644 index 0000000000..eb13e5ed05 --- /dev/null +++ b/agents/tests/test_teams_handler.py @@ -0,0 +1,328 @@ +""" +Pytest tests for Teams handler functionality. +Tests Teams-specific message handling and adaptive card actions. +""" + +import pytest +from unittest.mock import Mock, patch, AsyncMock +from botbuilder.schema import Activity, ActivityTypes + +from handlers.teams_handler import TeamsHandler +from services.rag_service import RAGService, RAGRequest, RAGResponse +from services.auth_service import AuthService +from constants.teams_text import TeamsTextConstants + + +class MockTurnContext: + """Mock TurnContext for testing.""" + + def __init__(self, activity: Activity): + self.activity = activity + self.channel_id = "msteams" + self.conversation = activity.conversation + self.from_property = activity.from_property + self.recipient = Mock() + self.recipient.id = "bot1" + + +class TestTeamsHandler: + """Test cases for TeamsHandler.""" + + @pytest.fixture + def mock_rag_service(self): + """Create a mock RAG service.""" + return Mock(spec=RAGService) + + @pytest.fixture + def mock_auth_service(self): + """Create a mock auth service.""" + return Mock(spec=AuthService) + + @pytest.fixture + def teams_handler(self, mock_rag_service, mock_auth_service): + """Create a TeamsHandler instance for testing.""" + return TeamsHandler(mock_rag_service, mock_auth_service) + + @pytest.fixture + def mock_turn_context(self): + """Create a mock TurnContext for testing.""" + activity = Activity( + type=ActivityTypes.message, + text="Test message", + from_property={"id": "user1", "name": "Test User"}, + recipient={"id": "bot1", "name": "RAG Assistant"}, + conversation={"id": "conv1"} + ) + return MockTurnContext(activity) + + @pytest.fixture + def mock_rag_response(self): + """Create a mock RAGResponse for testing.""" + return RAGResponse( + answer="This is a test response.", + sources=[{"title": "Source 1"}], + citations=["Citation 1"], + thoughts=[{"title": "Thought 1", "description": "Description 1"}], + token_usage={"total_tokens": 100}, + model_info={"model": "gpt-4"} + ) + + @pytest.mark.asyncio + async def test_handle_message_with_mention(self, teams_handler, mock_turn_context, mock_rag_response): + """Test message handling with bot mention.""" + # Mock the mention detection + with patch.object(teams_handler, '_is_bot_mentioned', return_value=True): + with patch.object(teams_handler, '_remove_mention', return_value="What are the main benefits?"): + with patch.object(teams_handler.rag_service, 'process_query', return_value=mock_rag_response): + with patch.object(teams_handler, '_create_adaptive_card_response', return_value=Mock()): + conversation_data = {"conversation_id": "conv1"} + user_data = {"conversation_history": []} + auth_claims = {"user_id": "user1"} + + response = await teams_handler.handle_message( + mock_turn_context, conversation_data, user_data, auth_claims + ) + + assert response is not None + + @pytest.mark.asyncio + async def test_handle_message_without_mention(self, teams_handler, mock_turn_context): + """Test message handling without bot mention.""" + # Mock no mention + with patch.object(teams_handler, '_is_bot_mentioned', return_value=False): + with patch.object(teams_handler, '_create_mention_reminder', return_value=Mock()): + conversation_data = {"conversation_id": "conv1"} + user_data = {"conversation_history": []} + auth_claims = {"user_id": "user1"} + + response = await teams_handler.handle_message( + mock_turn_context, conversation_data, user_data, auth_claims + ) + + assert response is not None + + @pytest.mark.asyncio + async def test_handle_message_empty_text(self, teams_handler, mock_turn_context): + """Test message handling with empty text.""" + mock_turn_context.activity.text = "" + + with patch.object(teams_handler, '_is_bot_mentioned', return_value=False): + with patch.object(teams_handler, '_create_mention_reminder', return_value=Mock()): + conversation_data = {"conversation_id": "conv1"} + user_data = {"conversation_history": []} + auth_claims = {"user_id": "user1"} + + response = await teams_handler.handle_message( + mock_turn_context, conversation_data, user_data, auth_claims + ) + + assert response is not None + + @pytest.mark.asyncio + async def test_handle_adaptive_card_action_follow_up(self, teams_handler, mock_turn_context): + """Test adaptive card action handling for follow-up.""" + mock_turn_context.activity.value = {"action": "follow_up"} + + conversation_data = {"conversation_id": "conv1"} + user_data = {"conversation_history": []} + auth_claims = {"user_id": "user1"} + + response = await teams_handler._handle_adaptive_card_action( + mock_turn_context, conversation_data, user_data, auth_claims + ) + + assert response is not None + assert TeamsTextConstants.FOLLOW_UP_RESPONSE in response.text + + @pytest.mark.asyncio + async def test_handle_adaptive_card_action_search_related(self, teams_handler, mock_turn_context): + """Test adaptive card action handling for search related.""" + mock_turn_context.activity.value = {"action": "search_related"} + + conversation_data = {"conversation_id": "conv1"} + user_data = {"conversation_history": []} + auth_claims = {"user_id": "user1"} + + response = await teams_handler._handle_adaptive_card_action( + mock_turn_context, conversation_data, user_data, auth_claims + ) + + assert response is not None + assert TeamsTextConstants.SEARCH_RELATED_RESPONSE in response.text + + @pytest.mark.asyncio + async def test_handle_adaptive_card_action_summarize(self, teams_handler, mock_turn_context): + """Test adaptive card action handling for summarize.""" + mock_turn_context.activity.value = {"action": "summarize"} + + conversation_data = {"conversation_id": "conv1"} + user_data = {"conversation_history": []} + auth_claims = {"user_id": "user1"} + + response = await teams_handler._handle_adaptive_card_action( + mock_turn_context, conversation_data, user_data, auth_claims + ) + + assert response is not None + assert TeamsTextConstants.SUMMARIZE_RESPONSE in response.text + + @pytest.mark.asyncio + async def test_handle_adaptive_card_action_unknown(self, teams_handler, mock_turn_context): + """Test adaptive card action handling for unknown action.""" + mock_turn_context.activity.value = {"action": "unknown_action"} + + conversation_data = {"conversation_id": "conv1"} + user_data = {"conversation_history": []} + auth_claims = {"user_id": "user1"} + + response = await teams_handler._handle_adaptive_card_action( + mock_turn_context, conversation_data, user_data, auth_claims + ) + + assert response is not None + assert "unknown_action" in response.text + + @pytest.mark.asyncio + async def test_handle_follow_up_action(self, teams_handler, mock_turn_context): + """Test follow-up action handling.""" + conversation_data = {"conversation_id": "conv1"} + user_data = {"conversation_history": []} + auth_claims = {"user_id": "user1"} + + response = await teams_handler._handle_follow_up_action( + mock_turn_context, conversation_data, user_data, auth_claims + ) + + assert response is not None + assert TeamsTextConstants.FOLLOW_UP_RESPONSE in response.text + + @pytest.mark.asyncio + async def test_handle_search_related_action(self, teams_handler, mock_turn_context): + """Test search related action handling.""" + conversation_data = {"conversation_id": "conv1"} + user_data = {"conversation_history": []} + auth_claims = {"user_id": "user1"} + + response = await teams_handler._handle_search_related_action( + mock_turn_context, conversation_data, user_data, auth_claims + ) + + assert response is not None + assert TeamsTextConstants.SEARCH_RELATED_RESPONSE in response.text + + @pytest.mark.asyncio + async def test_handle_summarize_action(self, teams_handler, mock_turn_context): + """Test summarize action handling.""" + conversation_data = {"conversation_id": "conv1"} + user_data = {"conversation_history": []} + auth_claims = {"user_id": "user1"} + + response = await teams_handler._handle_summarize_action( + mock_turn_context, conversation_data, user_data, auth_claims + ) + + assert response is not None + assert TeamsTextConstants.SUMMARIZE_RESPONSE in response.text + + @pytest.mark.asyncio + async def test_is_bot_mentioned_true(self, teams_handler, mock_turn_context): + """Test bot mention detection when mentioned.""" + # Mock activity with mention + mock_turn_context.activity.entities = [ + Mock(type="mention", as_dict=lambda: {"mentioned": {"id": "bot1"}}) + ] + + result = await teams_handler._is_bot_mentioned(mock_turn_context) + assert result is True + + @pytest.mark.asyncio + async def test_is_bot_mentioned_false(self, teams_handler, mock_turn_context): + """Test bot mention detection when not mentioned.""" + # Mock activity without mention + mock_turn_context.activity.entities = [] + + result = await teams_handler._is_bot_mentioned(mock_turn_context) + assert result is False + + @pytest.mark.asyncio + async def test_remove_mention(self, teams_handler, mock_turn_context): + """Test mention removal from message text.""" + # Mock activity with Teams-style mention tags + mock_turn_context.activity.text = "RAG Assistant What are the main benefits?" + + result = await teams_handler._remove_mention(mock_turn_context) + assert result == "What are the main benefits?" + + @pytest.mark.asyncio + async def test_create_mention_reminder(self, teams_handler, mock_turn_context): + """Test mention reminder creation.""" + response = await teams_handler._create_mention_reminder(mock_turn_context) + + assert response is not None + assert TeamsTextConstants.format_mention_reminder() in response.text + + @pytest.mark.asyncio + async def test_handle_file_upload(self, teams_handler, mock_turn_context): + """Test file upload handling.""" + # Mock file attachment + mock_turn_context.activity.attachments = [ + Mock(content_type="application/vnd.microsoft.teams.file.download.info") + ] + + conversation_data = {"conversation_id": "conv1"} + + response = await teams_handler.handle_file_upload( + mock_turn_context, conversation_data + ) + + assert response is not None + assert "file" in response.text.lower() + + # Note: handle_members_added and handle_members_removed methods don't exist in TeamsHandler + + @pytest.mark.asyncio + async def test_create_adaptive_card_response(self, teams_handler, mock_turn_context, mock_rag_response): + """Test adaptive card response creation.""" + conversation_data = Mock() + conversation_data.conversation_id = "conv1" + + response = await teams_handler._create_adaptive_card_response( + mock_turn_context, mock_rag_response, conversation_data + ) + + assert response is not None + assert response.attachments is not None + assert len(response.attachments) > 0 + assert response.attachments[0].content_type == "application/vnd.microsoft.card.adaptive" + + @pytest.mark.asyncio + async def test_error_handling_in_message_handling(self, teams_handler, mock_turn_context): + """Test error handling in message handling.""" + with patch.object(teams_handler, '_is_bot_mentioned', side_effect=Exception("Test error")): + conversation_data = {"conversation_id": "conv1"} + user_data = {"conversation_history": []} + auth_claims = {"user_id": "user1"} + + response = await teams_handler.handle_message( + mock_turn_context, conversation_data, user_data, auth_claims + ) + + assert response is not None + assert "error" in response.text.lower() + + @pytest.mark.asyncio + async def test_error_handling_in_adaptive_card_action(self, teams_handler, mock_turn_context): + """Test error handling in adaptive card action handling.""" + mock_turn_context.activity.value = {"action": "follow_up"} + + with patch.object(teams_handler, '_handle_follow_up_action', side_effect=Exception("Test error")): + conversation_data = {"conversation_id": "conv1"} + user_data = {"conversation_history": []} + auth_claims = {"user_id": "user1"} + + response = await teams_handler._handle_adaptive_card_action( + mock_turn_context, conversation_data, user_data, auth_claims + ) + + assert response is not None + assert "error" in response.text.lower() \ No newline at end of file diff --git a/agents/tests/test_teams_response_adapter.py b/agents/tests/test_teams_response_adapter.py new file mode 100644 index 0000000000..45bcf67649 --- /dev/null +++ b/agents/tests/test_teams_response_adapter.py @@ -0,0 +1,234 @@ +""" +Pytest tests for Teams response adapter functionality. +Tests Teams-specific response formatting and adaptation. +""" + +import pytest +from unittest.mock import Mock, patch, AsyncMock +from botbuilder.schema import Activity, ActivityTypes + +from adapters.teams_response_adapter import TeamsResponseAdapter +from components.teams_components import TeamsCardConfig +from services.rag_service import RAGResponse +from constants.teams_text import TeamsTextConstants + + +class MockTurnContext: + """Mock TurnContext for testing.""" + + def __init__(self, activity: Activity): + self.activity = activity + self.channel_id = "msteams" + self.conversation = activity.conversation + self.from_property = activity.from_property + self.recipient = activity.recipient + + +class TestTeamsResponseAdapter: + """Test cases for TeamsResponseAdapter.""" + + @pytest.fixture + def adapter(self): + """Create a TeamsResponseAdapter instance for testing.""" + return TeamsResponseAdapter() + + @pytest.fixture + def mock_turn_context(self): + """Create a mock TurnContext for testing.""" + activity = Activity( + type=ActivityTypes.message, + text="Test message", + from_property={"id": "user1", "name": "Test User"}, + recipient={"id": "bot1", "name": "RAG Assistant"}, + conversation={"id": "conv1"} + ) + return MockTurnContext(activity) + + @pytest.fixture + def mock_rag_response(self): + """Create a mock RAGResponse for testing.""" + return RAGResponse( + answer="This is a test response with comprehensive information about the topic.", + sources=[ + {"title": "Source 1", "url": "https://example.com/source1"}, + {"title": "Source 2", "url": "https://example.com/source2"} + ], + citations=["Citation 1", "Citation 2"], + thoughts=[ + {"title": "Query Analysis", "description": "Analyzed user query"}, + {"title": "Information Retrieval", "description": "Retrieved relevant documents"} + ], + token_usage={"total_tokens": 150, "prompt_tokens": 50, "completion_tokens": 100}, + model_info={"model": "gpt-4", "temperature": "0.3"} + ) + + def test_format_rag_response(self, adapter, mock_turn_context, mock_rag_response): + """Test RAG response formatting.""" + response_activity = adapter.format_rag_response(mock_turn_context, mock_rag_response) + + assert response_activity.attachments is not None + assert len(response_activity.attachments) > 0 + assert response_activity.attachments[0].content_type == "application/vnd.microsoft.card.adaptive" + assert response_activity.text == mock_rag_response.answer + + # Check adaptive card content + card_content = response_activity.attachments[0].content + assert card_content["type"] == "AdaptiveCard" + assert card_content["version"] == "1.4" + assert len(card_content["body"]) > 0 + assert len(card_content["actions"]) > 0 + + def test_format_text_response(self, adapter, mock_turn_context): + """Test text response formatting.""" + text = "Simple text response" + response_activity = adapter.format_text_response(mock_turn_context, text) + + assert response_activity.text == text + assert response_activity.suggested_actions is not None + assert len(response_activity.suggested_actions) > 0 + + def test_format_text_response_without_suggestions(self, adapter, mock_turn_context): + """Test text response formatting without suggestions.""" + text = "Simple text response" + response_activity = adapter.format_text_response(mock_turn_context, text, include_suggestions=False) + + assert response_activity.text == text + assert response_activity.suggested_actions is None + + def test_format_welcome_response(self, adapter, mock_turn_context): + """Test welcome response formatting.""" + response_activity = adapter.format_welcome_response(mock_turn_context) + + assert response_activity.attachments is not None + assert len(response_activity.attachments) > 0 + assert response_activity.attachments[0].content_type == "application/vnd.microsoft.card.adaptive" + + # Check text contains bot name + assert "Welcome to" in response_activity.text + assert "Structural Engineering Assistant" in response_activity.text + + def test_format_help_response(self, adapter, mock_turn_context): + """Test help response formatting.""" + response_activity = adapter.format_help_response(mock_turn_context) + + assert response_activity.attachments is not None + assert len(response_activity.attachments) > 0 + assert response_activity.attachments[0].content_type == "application/vnd.microsoft.card.adaptive" + + # Check text contains bot name + assert "Here's how to use" in response_activity.text + assert "Structural Engineering Assistant" in response_activity.text + + def test_format_error_response(self, adapter, mock_turn_context): + """Test error response formatting.""" + error_message = "Test error message" + response_activity = adapter.format_error_response(mock_turn_context, error_message) + + assert response_activity.attachments is not None + assert len(response_activity.attachments) > 0 + assert response_activity.attachments[0].content_type == "application/vnd.microsoft.card.adaptive" + assert error_message in response_activity.text + + def test_format_loading_response(self, adapter, mock_turn_context): + """Test loading response formatting.""" + response_activity = adapter.format_loading_response(mock_turn_context) + + assert response_activity.attachments is not None + assert len(response_activity.attachments) > 0 + assert response_activity.attachments[0].content_type == "application/vnd.microsoft.card.adaptive" + assert "Processing your request" in response_activity.text + + def test_format_file_upload_response(self, adapter, mock_turn_context): + """Test file upload response formatting.""" + file_name = "test.pdf" + file_type = "application/pdf" + response_activity = adapter.format_file_upload_response(mock_turn_context, file_name, file_type) + + assert response_activity.attachments is not None + assert len(response_activity.attachments) > 0 + assert response_activity.attachments[0].content_type == "application/vnd.microsoft.card.adaptive" + assert file_name in response_activity.text + + def test_format_quick_actions_response(self, adapter, mock_turn_context): + """Test quick actions response formatting.""" + response_activity = adapter.format_quick_actions_response(mock_turn_context) + + assert response_activity.attachments is not None + assert len(response_activity.attachments) > 0 + assert response_activity.attachments[0].content_type == "application/vnd.microsoft.card.adaptive" + assert "Choose a quick action" in response_activity.text + + def test_create_rag_response_card_with_sources(self, adapter, mock_rag_response): + """Test RAG response card creation with sources.""" + card_json = adapter._create_rag_response_card(mock_rag_response) + + assert card_json["type"] == "AdaptiveCard" + assert card_json["version"] == "1.4" + assert len(card_json["body"]) > 0 + assert len(card_json["actions"]) > 0 + + # Check sources section exists + sources_section = next( + (item for item in card_json["body"] if item.get("items", [{}])[0].get("text") == "📚 Sources"), + None + ) + assert sources_section is not None + + def test_create_rag_response_card_with_citations(self, adapter, mock_rag_response): + """Test RAG response card creation with citations.""" + card_json = adapter._create_rag_response_card(mock_rag_response) + + # Check citations section exists + citations_section = next( + (item for item in card_json["body"] if item.get("items", [{}])[0].get("text") == "🔗 Citations"), + None + ) + assert citations_section is not None + + def test_create_rag_response_card_with_thoughts_disabled(self, adapter, mock_rag_response): + """Test RAG response card creation with thoughts disabled.""" + adapter.config = TeamsCardConfig(show_thoughts=False) + card_json = adapter._create_rag_response_card(mock_rag_response) + + # Check thoughts section does not exist + thoughts_section = next( + (item for item in card_json["body"] if item.get("items", [{}])[0].get("text") == "💭 Process"), + None + ) + assert thoughts_section is None + + def test_create_rag_response_card_with_usage_disabled(self, adapter, mock_rag_response): + """Test RAG response card creation with usage disabled.""" + adapter.config = TeamsCardConfig(show_usage=False) + card_json = adapter._create_rag_response_card(mock_rag_response) + + # Check usage section does not exist + usage_section = next( + (item for item in card_json["body"] if item.get("items", [{}])[0].get("text") == "📊 Usage"), + None + ) + assert usage_section is None + + def test_create_suggested_actions(self, adapter): + """Test suggested actions creation.""" + suggested_actions = adapter._create_suggested_actions() + + assert len(suggested_actions) == len(TeamsTextConstants.SUGGESTED_ACTIONS) + for i, action in enumerate(suggested_actions): + assert action.title == TeamsTextConstants.SUGGESTED_ACTIONS[i] + + def test_error_handling_in_welcome_response(self, adapter, mock_turn_context): + """Test error handling in welcome response.""" + with patch.object(adapter.teams_components, 'create_welcome_card', side_effect=Exception("Test error")): + response_activity = adapter.format_welcome_response(mock_turn_context) + + assert response_activity.text is not None + assert "Welcome to" in response_activity.text + + def test_error_handling_in_help_response(self, adapter, mock_turn_context): + """Test error handling in help response.""" + with patch.object(adapter.teams_components, 'create_help_card', side_effect=Exception("Test error")): + response_activity = adapter.format_help_response(mock_turn_context) + + assert response_activity.text is not None + assert "Here's how to use" in response_activity.text \ No newline at end of file diff --git a/agents/update-manifest.ps1 b/agents/update-manifest.ps1 new file mode 100644 index 0000000000..088d6c37cf --- /dev/null +++ b/agents/update-manifest.ps1 @@ -0,0 +1,40 @@ +# PowerShell script to update copilot-plugin-manifest.json with production URL + +param( + [Parameter(Mandatory=$true)] + [string]$ProductionUrl +) + +$manifestPath = Join-Path $PSScriptRoot "copilot-plugin-manifest.json" + +if (-not (Test-Path $manifestPath)) { + Write-Host "[ERROR] Manifest file not found: $manifestPath" -ForegroundColor Red + exit 1 +} + +Write-Host "Updating manifest with production URL: $ProductionUrl" -ForegroundColor Cyan + +# Read manifest +$manifestContent = Get-Content $manifestPath -Raw + +# Remove trailing slash if present +$ProductionUrl = $ProductionUrl.TrimEnd('/') + +# Replace all instances of your-domain.com +$manifestContent = $manifestContent -replace 'https://your-domain\.com', $ProductionUrl + +# Write back +Set-Content -Path $manifestPath -Value $manifestContent -NoNewline + +Write-Host "[OK] Manifest updated successfully!" -ForegroundColor Green +Write-Host "" +Write-Host "Updated endpoints:" -ForegroundColor Cyan +Write-Host "- Search: $ProductionUrl/api/copilot/search" -ForegroundColor White +Write-Host "- Query: $ProductionUrl/api/copilot/query" -ForegroundColor White +Write-Host "- Health: $ProductionUrl/api/copilot/health" -ForegroundColor White +Write-Host "- Icon: $ProductionUrl/icons/copilot-icon.png" -ForegroundColor White + + + + + diff --git a/app/backend/.dockerignore b/app/backend/.dockerignore deleted file mode 100644 index 9008115fc8..0000000000 --- a/app/backend/.dockerignore +++ /dev/null @@ -1,7 +0,0 @@ -.git -__pycache__ -*.pyc -*.pyo -*.pyd -.Python -env diff --git a/app/backend/app.py b/app/backend/app.py index 62707e0cd7..645dbcae8a 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -54,6 +54,7 @@ CONFIG_AGENTIC_RETRIEVAL_ENABLED, CONFIG_ASK_APPROACH, CONFIG_AUTH_CLIENT, + CONFIG_CACHE, CONFIG_CHAT_APPROACH, CONFIG_CHAT_HISTORY_BROWSER_ENABLED, CONFIG_CHAT_HISTORY_COSMOS_ENABLED, @@ -83,6 +84,7 @@ CONFIG_USER_BLOB_MANAGER, CONFIG_USER_UPLOAD_ENABLED, CONFIG_VECTOR_SEARCH_ENABLED, + OCR_ON_INGEST, ) from core.authentication import AuthenticationHelper from core.sessionhelper import create_session_id @@ -97,11 +99,14 @@ setup_openai_client, setup_search_info, ) +from services.ocr_service import OCRService from prepdocslib.blobmanager import AdlsBlobManager, BlobManager from prepdocslib.embeddings import ImageEmbeddings from prepdocslib.filestrategy import UploadUserFileStrategy from prepdocslib.listfilestrategy import File +logger = logging.getLogger(__name__) + bp = Blueprint("routes", __name__, static_folder="static") # Fix Windows registry issue with mimetypes mimetypes.add_type("application/javascript", ".js") @@ -110,7 +115,68 @@ @bp.route("/") async def index(): - return await bp.send_static_file("index.html") + """Basic health check.""" + return jsonify({"status": "healthy", "service": "RAG Backend"}) + +@bp.route("/health", methods=["GET"]) +async def health(): + """Enhanced health check with dependency probes.""" + import aiohttp + from datetime import datetime + + health_status = { + "status": "healthy", + "timestamp": datetime.utcnow().isoformat(), + "dependencies": {} + } + + # Check Azure AI Search + search_ok = False + search_latency_ms = None + try: + search_client: SearchClient = current_app.config.get(CONFIG_SEARCH_CLIENT) + if search_client: + t0 = time.time() + # Simple ping - get index stats + await search_client.get_document_count() + search_latency_ms = int((time.time() - t0) * 1000) + search_ok = True + except Exception as e: + search_ok = False + health_status["dependencies"]["azure_search"] = {"ok": False, "error": str(e)[:100]} + + if search_ok: + health_status["dependencies"]["azure_search"] = { + "ok": True, + "latency_ms": search_latency_ms + } + + # Check Azure OpenAI + openai_ok = False + openai_latency_ms = None + try: + openai_client = current_app.config.get(CONFIG_OPENAI_CLIENT) + if openai_client: + t0 = time.time() + # Simple ping - list models (lightweight call) + await openai_client.models.list() + openai_latency_ms = int((time.time() - t0) * 1000) + openai_ok = True + except Exception as e: + openai_ok = False + health_status["dependencies"]["azure_openai"] = {"ok": False, "error": str(e)[:100]} + + if openai_ok: + health_status["dependencies"]["azure_openai"] = { + "ok": True, + "latency_ms": openai_latency_ms + } + + # Overall status + if not search_ok or not openai_ok: + health_status["status"] = "degraded" + + return jsonify(health_status) # Empty page is recommended for login redirect to work. @@ -219,6 +285,13 @@ async def chat(auth_claims: dict[str, Any]): request_json = await request.get_json() context = request_json.get("context", {}) context["auth_claims"] = auth_claims + + # Extract correlation ID from request headers + traceparent = request.headers.get("x-traceparent") or request.headers.get("traceparent") + if traceparent: + context["traceparent"] = traceparent + current_app.logger.info(f"traceparent={traceparent}") + try: approach: Approach = cast(Approach, current_app.config[CONFIG_CHAT_APPROACH]) @@ -508,13 +581,23 @@ async def setup_clients(): current_app.config[CONFIG_CREDENTIAL] = azure_credential # Set up clients for AI Search and Storage + # For local development, use AZURE_SEARCH_KEY if available (avoids needing azd) + from azure.core.credentials import AzureKeyCredential + AZURE_SEARCH_KEY = os.getenv("AZURE_SEARCH_KEY") + if AZURE_SEARCH_KEY: + current_app.logger.info("Using AZURE_SEARCH_KEY for Azure Search authentication") + search_credential = AzureKeyCredential(AZURE_SEARCH_KEY) + else: + current_app.logger.info("Using Azure credential (Managed Identity/Azure CLI) for Azure Search authentication") + search_credential = azure_credential + search_client = SearchClient( endpoint=AZURE_SEARCH_ENDPOINT, index_name=AZURE_SEARCH_INDEX, - credential=azure_credential, + credential=search_credential, ) agent_client = KnowledgeAgentRetrievalClient( - endpoint=AZURE_SEARCH_ENDPOINT, agent_name=AZURE_SEARCH_AGENT, credential=azure_credential + endpoint=AZURE_SEARCH_ENDPOINT, agent_name=AZURE_SEARCH_AGENT, credential=search_credential ) # Set up the global blob storage manager (used for global content/images, but not user uploads) @@ -620,6 +703,17 @@ async def setup_clients(): vision_endpoint=AZURE_VISION_ENDPOINT, use_multimodal=USE_MULTIMODAL, ) + + # Initialize OCR service for runtime uploads if enabled + ocr_service = None + if OCR_ON_INGEST: + ocr_candidate = OCRService() + if ocr_candidate.is_enabled(): + ocr_service = ocr_candidate + logger.info("OCR service enabled for runtime user uploads") + else: + logger.warning("OCR_ON_INGEST is enabled but no OCR provider is configured; skipping OCR for runtime uploads.") + ingester = UploadUserFileStrategy( search_info=search_info, file_processors=file_processors, @@ -627,6 +721,7 @@ async def setup_clients(): image_embeddings=image_embeddings_service, search_field_name_embedding=AZURE_SEARCH_FIELD_NAME_EMBEDDING, blob_manager=user_blob_manager, + ocr_service=ocr_service, ) current_app.config[CONFIG_INGESTER] = ingester @@ -696,6 +791,12 @@ async def setup_clients(): ) # ChatReadRetrieveReadApproach is used by /chat for multi-turn conversation + # Initialize cache (Redis or in-memory) + from services.cache import create_cache + from config import REDIS_URL + cache = await create_cache(REDIS_URL) + current_app.config[CONFIG_CACHE] = cache + current_app.config[CONFIG_CHAT_APPROACH] = ChatReadRetrieveReadApproach( search_client=search_client, search_index_name=AZURE_SEARCH_INDEX, @@ -729,6 +830,9 @@ async def close_clients(): await current_app.config[CONFIG_GLOBAL_BLOB_MANAGER].close_clients() if user_blob_manager := current_app.config.get(CONFIG_USER_BLOB_MANAGER): await user_blob_manager.close_clients() + # Close cache connection + if cache := current_app.config.get(CONFIG_CACHE): + await cache.close() def create_app(): @@ -736,23 +840,27 @@ def create_app(): app.register_blueprint(bp) app.register_blueprint(chat_history_cosmosdb_bp) - if os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING"): + app_insights_conn_str = os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING", "").strip() + if app_insights_conn_str and app_insights_conn_str.lower() not in ("", "none", "your-app-insights-connection-string"): app.logger.info("APPLICATIONINSIGHTS_CONNECTION_STRING is set, enabling Azure Monitor") - configure_azure_monitor( - instrumentation_options={ - "django": {"enabled": False}, - "psycopg2": {"enabled": False}, - "fastapi": {"enabled": False}, - } - ) - # This tracks HTTP requests made by aiohttp: - AioHttpClientInstrumentor().instrument() - # This tracks HTTP requests made by httpx: - HTTPXClientInstrumentor().instrument() - # This tracks OpenAI SDK requests: - OpenAIInstrumentor().instrument() - # This middleware tracks app route requests: - app.asgi_app = OpenTelemetryMiddleware(app.asgi_app) # type: ignore[assignment] + try: + configure_azure_monitor( + instrumentation_options={ + "django": {"enabled": False}, + "psycopg2": {"enabled": False}, + "fastapi": {"enabled": False}, + } + ) + # This tracks HTTP requests made by aiohttp: + AioHttpClientInstrumentor().instrument() + # This tracks HTTP requests made by httpx: + HTTPXClientInstrumentor().instrument() + # This tracks OpenAI SDK requests: + OpenAIInstrumentor().instrument() + # This middleware tracks app route requests: + app.asgi_app = OpenTelemetryMiddleware(app.asgi_app) # type: ignore[assignment] + except Exception as e: + app.logger.warning(f"Failed to configure Azure Monitor: {e}. Continuing without telemetry.") # Log levels should be one of https://docs.python.org/3/library/logging.html#logging-levels # Set root level to WARNING to avoid seeing overly verbose logs from SDKS diff --git a/app/backend/approaches/__init__.py b/app/backend/approaches/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/app/backend/approaches/approach.py b/app/backend/approaches/approach.py index 04a74a8818..eb1f5860a7 100644 --- a/app/backend/approaches/approach.py +++ b/app/backend/approaches/approach.py @@ -104,6 +104,7 @@ class ExtraInfo: data_points: DataPoints thoughts: list[ThoughtStep] = field(default_factory=list) followup_questions: Optional[list[Any]] = None + unified_citations: Optional[list[dict[str, Any]]] = None # Unified citation format @dataclass @@ -214,51 +215,70 @@ async def search( ) -> list[Document]: search_text = query_text if use_text_search else "" search_vectors = vectors if use_vector_search else [] - if use_semantic_ranker: - results = await self.search_client.search( - search_text=search_text, - filter=filter, - top=top, - query_caption="extractive|highlight-false" if use_semantic_captions else None, - query_rewrites="generative" if use_query_rewriting else None, - vector_queries=search_vectors, - query_type=QueryType.SEMANTIC, - query_language=self.query_language, - query_speller=self.query_speller, - semantic_configuration_name="default", - semantic_query=query_text, - ) - else: - results = await self.search_client.search( - search_text=search_text, - filter=filter, - top=top, - vector_queries=search_vectors, - ) + + try: + if use_semantic_ranker: + results = await self.search_client.search( + search_text=search_text, + filter=filter, + top=top, + query_caption="extractive|highlight-false" if use_semantic_captions else None, + query_rewrites="generative" if use_query_rewriting else None, + vector_queries=search_vectors, + query_type=QueryType.SEMANTIC, + query_language=self.query_language, + query_speller=self.query_speller, + semantic_configuration_name="default", + semantic_query=query_text, + ) + else: + results = await self.search_client.search( + search_text=search_text, + filter=filter, + top=top, + vector_queries=search_vectors, + ) + except Exception as search_error: + # Handle empty index or missing searchable fields gracefully + error_msg = str(search_error) + if "CannotSearchWithoutSearchableFields" in error_msg or "searchable string fields" in error_msg: + # Return empty results instead of crashing - allows bot to respond with a helpful message + return [] + # Re-raise other exceptions + raise documents: list[Document] = [] - async for page in results.by_page(): - async for document in page: - documents.append( - Document( - id=document.get("id"), - content=document.get("content"), - category=document.get("category"), - sourcepage=document.get("sourcepage"), - sourcefile=document.get("sourcefile"), - oids=document.get("oids"), - groups=document.get("groups"), - captions=cast(list[QueryCaptionResult], document.get("@search.captions")), - score=document.get("@search.score"), - reranker_score=document.get("@search.reranker_score"), - images=document.get("images"), + try: + async for page in results.by_page(): + async for document in page: + documents.append( + Document( + id=document.get("id"), + content=document.get("content"), + category=document.get("category"), + sourcepage=document.get("sourcepage"), + sourcefile=document.get("sourcefile"), + oids=document.get("oids"), + groups=document.get("groups"), + captions=cast(list[QueryCaptionResult], document.get("@search.captions")), + score=document.get("@search.score"), + reranker_score=document.get("@search.reranker_score"), + images=document.get("images"), + ) ) - ) - - qualified_documents = [ - doc - for doc in documents - if ( + except Exception as e: + # Handle empty index or missing searchable fields gracefully + error_msg = str(e) + if "CannotSearchWithoutSearchableFields" in error_msg or "searchable string fields" in error_msg: + # Return empty results instead of crashing - allows bot to respond with a helpful message + return [] + # Re-raise other exceptions + raise + + qualified_documents = [ + doc + for doc in documents + if ( (doc.score or 0) >= (minimum_search_score or 0) and (doc.reranker_score or 0) >= (minimum_reranker_score or 0) ) diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index bc51dc107a..ca6b4f9b0c 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -18,11 +18,17 @@ Approach, ExtraInfo, ThoughtStep, + DataPoints, ) from approaches.promptmanager import PromptManager from core.authentication import AuthenticationHelper from prepdocslib.blobmanager import AdlsBlobManager, BlobManager from prepdocslib.embeddings import ImageEmbeddings +from config import ENABLE_WEB_SEARCH, SERPER_API_KEY, WEB_CACHE_TTL_S, CONFIG_CACHE +from typing import List, Dict +from quart import current_app +import hashlib +import json class ChatReadRetrieveReadApproach(Approach): @@ -128,6 +134,16 @@ async def run_without_streaming( if overrides.get("suggest_followup_questions"): content, followup_questions = self.extract_followup_questions(content) extra_info.followup_questions = followup_questions + + # Filter citations to only include those actually used in the answer + from services.citation_filter import filter_citations_by_answer + if extra_info.data_points.citations and content: + filtered_citations = filter_citations_by_answer( + extra_info.data_points.citations, + content + ) + extra_info.data_points.citations = filtered_citations + # Assume last thought is for generating answer if self.include_token_usage and extra_info.thoughts and chat_completion_response.usage: extra_info.thoughts[-1].update_token_usage(chat_completion_response.usage) @@ -271,6 +287,252 @@ async def run_until_final_call( async def run_search_approach( self, messages: list[ChatCompletionMessageParam], overrides: dict[str, Any], auth_claims: dict[str, Any] ): + # Phase 1B scaffolding: allow a simple 'mode' switch with safe defaults + # Default to hybrid mode if web search is enabled, otherwise use rag + default_mode = "hybrid" if ENABLE_WEB_SEARCH and SERPER_API_KEY else "rag" + mode = overrides.get("mode", default_mode) # rag | web | hybrid + + # Hybrid mode: merge RAG + Web results + if mode == "hybrid": + if not ENABLE_WEB_SEARCH: + # Fallback to RAG-only if web search disabled + mode = "rag" + else: + # Run both RAG and Web in parallel, then merge + import asyncio + from services.web_search.serper_client import SerperClient + from services.web_search.normalizer import normalize_serper + + original_user_query = messages[-1]["content"] + if not isinstance(original_user_query, str): + raise ValueError("The most recent message content must be a string.") + + # Get query for web search + query_messages = self.prompt_manager.render_prompt( + self.query_rewrite_prompt, {"user_query": original_user_query, "past_messages": messages[:-1]} + ) + tools: list[ChatCompletionToolParam] = self.query_rewrite_tools + chat_completion = cast( + ChatCompletion, + await self.create_chat_completion( + self.chatgpt_deployment, + self.chatgpt_model, + messages=query_messages, + overrides=overrides, + response_token_limit=self.get_response_token_limit(self.chatgpt_model, 100), + temperature=0.0, + tools=tools, + reasoning_effort=self.get_lowest_reasoning_effort(self.chatgpt_model), + ), + ) + query_text = self.get_search_query(chat_completion, original_user_query) + + # Run RAG search (reuse existing logic but with mode=rag override) + rag_overrides = {**overrides, "mode": "rag"} + rag_info = await self.run_search_approach(messages, rag_overrides, auth_claims) + + # Run web search with caching + web_info = None + if SERPER_API_KEY: + try: + top = overrides.get("top", 3) + + # Check cache first + cache = current_app.config.get(CONFIG_CACHE) + raw_items = None + if cache: + # Create cache key from query and top parameter + cache_key_data = {"query": query_text, "top": top, "provider": "serper"} + cache_key = f"web_search:{hashlib.md5(json.dumps(cache_key_data, sort_keys=True).encode()).hexdigest()}" + cached_result = await cache.get(cache_key) + if cached_result: + raw_items = cached_result + + # If not in cache, fetch from API + if raw_items is None: + raw_items = await SerperClient(SERPER_API_KEY).search(query_text, top) + # Cache the result + if cache: + cache_key_data = {"query": query_text, "top": top, "provider": "serper"} + cache_key = f"web_search:{hashlib.md5(json.dumps(cache_key_data, sort_keys=True).encode()).hexdigest()}" + await cache.set(cache_key, raw_items, WEB_CACHE_TTL_S) + + normalized = normalize_serper(raw_items) + web_text_sources = [f"{item.get('url','')}: {item.get('snippet','')}" for item in normalized] + web_citations = [item.get("url", "") for item in normalized] + + # Build unified citations for web + from services.citation_builder import build_unified_from_text_sources + web_text_sources_for_cit = [{"url": item.get("url"), "title": item.get("title"), "content": item.get("snippet"), "sourcefile": item.get("url")} for item in normalized] + web_unified = build_unified_from_text_sources(web_text_sources_for_cit) + for cit in web_unified: + cit["source"] = "web" + cit["provider"] = "serper" + + web_info = ExtraInfo( + DataPoints(text=web_text_sources, images=[], citations=web_citations), + unified_citations=web_unified, + ) + except Exception as e: + # Web search failed, continue with RAG only + pass + + # Merge RAG + Web results with deduplication + merged_text = list(rag_info.data_points.text or []) + merged_citations = list(rag_info.data_points.citations or []) + merged_unified = list(rag_info.unified_citations or []) + seen_urls = set() + + # Add RAG URLs to seen set + for cit in rag_info.data_points.citations or []: + seen_urls.add(cit.lower()) + + # Add web results (deduplicate by URL) + if web_info: + for text_src in web_info.data_points.text or []: + # Extract URL from text source + if "http" in text_src: + url = text_src.split(":")[0] if ":" in text_src else "" + if url.lower() not in seen_urls: + merged_text.append(text_src) + seen_urls.add(url.lower()) + + for cit in web_info.data_points.citations or []: + if cit.lower() not in seen_urls: + merged_citations.append(cit) + seen_urls.add(cit.lower()) + + # Merge unified citations + merged_unified.extend(web_info.unified_citations or []) + + return ExtraInfo( + DataPoints(text=merged_text, images=rag_info.data_points.images, citations=merged_citations), + thoughts=rag_info.thoughts + [ + ThoughtStep( + title="Hybrid mode (RAG + Web)", + description=f"Merged {len(rag_info.data_points.text or [])} RAG + {len(web_info.data_points.text or []) if web_info else 0} web results", + props={"mode": "hybrid"}, + ) + ], + unified_citations=merged_unified, + ) + + if mode == "web": + if not ENABLE_WEB_SEARCH: + # Web search is disabled; return empty data points but do not crash + return ExtraInfo( + DataPoints(text=[], images=[], citations=[]), + thoughts=[ + ThoughtStep( + title="Web search disabled", + description="ENABLE_WEB_SEARCH flag is false; returning no external results.", + props={"mode": mode}, + ) + ], + ) + + # Generate a query (reuse the standard rewrite step for consistency) + original_user_query = messages[-1]["content"] + if not isinstance(original_user_query, str): + raise ValueError("The most recent message content must be a string.") + + query_messages = self.prompt_manager.render_prompt( + self.query_rewrite_prompt, {"user_query": original_user_query, "past_messages": messages[:-1]} + ) + tools: list[ChatCompletionToolParam] = self.query_rewrite_tools + chat_completion = cast( + ChatCompletion, + await self.create_chat_completion( + self.chatgpt_deployment, + self.chatgpt_model, + messages=query_messages, + overrides=overrides, + response_token_limit=self.get_response_token_limit(self.chatgpt_model, 100), + temperature=0.0, + tools=tools, + reasoning_effort=self.get_lowest_reasoning_effort(self.chatgpt_model), + ), + ) + query_text = self.get_search_query(chat_completion, original_user_query) + + # Call SERPER and normalize results + if not SERPER_API_KEY: + return ExtraInfo( + DataPoints(text=[], images=[], citations=[]), + thoughts=[ + ThoughtStep( + title="Missing SERPER_API_KEY", + description="Set SERPER_API_KEY to enable web search.", + props={"mode": mode}, + ) + ], + ) + + try: + from services.web_search.serper_client import SerperClient + from services.web_search.normalizer import normalize_serper + + top = overrides.get("top", 3) + + # Check cache first + cache = current_app.config.get(CONFIG_CACHE) + raw_items = None + if cache: + # Create cache key from query and top parameter + cache_key_data = {"query": query_text, "top": top, "provider": "serper"} + cache_key = f"web_search:{hashlib.md5(json.dumps(cache_key_data, sort_keys=True).encode()).hexdigest()}" + cached_result = await cache.get(cache_key) + if cached_result: + raw_items = cached_result + + # If not in cache, fetch from API + if raw_items is None: + raw_items: List[Dict[str, Any]] = await SerperClient(SERPER_API_KEY).search(query_text, top) + # Cache the result + if cache: + cache_key_data = {"query": query_text, "top": top, "provider": "serper"} + cache_key = f"web_search:{hashlib.md5(json.dumps(cache_key_data, sort_keys=True).encode()).hexdigest()}" + await cache.set(cache_key, raw_items, WEB_CACHE_TTL_S) + + normalized = normalize_serper(raw_items) + + # Build DataPoints from normalized results + text_sources = [f"{item.get('url','')}: {item.get('snippet','')}" for item in normalized] + citations = [item.get("url", "") for item in normalized] + + # Build unified citations from web results + from services.citation_builder import build_unified_from_text_sources + # Convert normalized web results to text_sources format for citation builder + web_text_sources = [{"url": item.get("url"), "title": item.get("title"), "content": item.get("snippet"), "sourcefile": item.get("url")} for item in normalized] + unified_citations = build_unified_from_text_sources(web_text_sources) + # Mark as web sources + for cit in unified_citations: + cit["source"] = "web" + cit["provider"] = "serper" + + return ExtraInfo( + DataPoints(text=text_sources, images=[], citations=citations), + thoughts=[ + ThoughtStep( + title="Web search (SERPER)", + description=f"Query: {query_text}", + props={"top": top, "results": len(normalized)}, + ) + ], + unified_citations=unified_citations, + ) + except Exception as e: + return ExtraInfo( + DataPoints(text=[], images=[], citations=[]), + thoughts=[ + ThoughtStep( + title="Web search error", + description=str(e), + props={"mode": mode}, + ) + ], + ) + use_text_search = overrides.get("retrieval_mode") in ["text", "hybrid", None] use_vector_search = overrides.get("retrieval_mode") in ["vectors", "hybrid", None] use_semantic_ranker = True if overrides.get("semantic_ranker") else False @@ -347,6 +609,13 @@ async def run_search_approach( download_image_sources=send_image_sources, user_oid=auth_claims.get("oid"), ) + + # Build unified citations from RAG results + from services.citation_builder import build_unified_from_text_sources + # Convert Document results to text_sources format + rag_text_sources = [{"sourcepage": doc.sourcepage, "sourcefile": doc.sourcefile, "title": doc.sourcefile or "Document", "content": doc.content or ""} for doc in results] + unified_citations = build_unified_from_text_sources(rag_text_sources) + extra_info = ExtraInfo( data_points, thoughts=[ @@ -379,6 +648,7 @@ async def run_search_approach( [result.serialize_for_results() for result in results], ), ], + unified_citations=unified_citations, ) return extra_info diff --git a/app/backend/approaches/prompts/ask_answer_question.prompty b/app/backend/approaches/prompts/ask_answer_question.prompty index 136ea1cf54..697c7617d4 100644 --- a/app/backend/approaches/prompts/ask_answer_question.prompty +++ b/app/backend/approaches/prompts/ask_answer_question.prompty @@ -4,21 +4,34 @@ description: Answer a single question (with no chat history) using solely text s model: api: chat sample: - user_query: What does a product manager do? + user_query: What are the load requirements for steel beam connections? text_sources: - - "role_library.pdf#page=29: The Manager of Product Management will collaborate with internal teams, such as engineering, sales, marketing, and finance, as well as external partners, suppliers, and customers to ensure successful product execution. Responsibilities: · Lead the product management team and provide guidance on product strategy, design, development, and launch. · Develop and implement product life-cycle management processes. · Monitor and analyze industry trends to identify opportunities for new products. · Develop product marketing plans and go-to-market strategies. · Research customer needs and develop customer-centric product roadmaps. · Collaborate with internal teams to ensure product execution and successful launch. · Develop pricing strategies and cost models. · Oversee product portfolio and performance metrics. · Manage product development budget. · Analyze product performance and customer feedback to identify areas for improvement. Qualifications: · Bachelor's degree in business, engineering, or a related field. · At least 5 years of experience in product management. · Proven track record of successful product launches." - - "role_library.pdf#page=23: Company: Contoso Electronics Location: Anywhere Job Type: Full-Time Salary: Competitive, commensurate with experience Job Summary: The Senior Manager of Product Management will be responsible for leading the product management team at Contoso Electronics. This role includes developing strategies, plans and objectives for the product management team and managing the day-to-day operations. The Senior Manager of Product Management will be responsible for the successful launch of new products and the optimization of existing products. Responsibilities: · Develop and implement product management strategies, plans and objectives to maximize team performance. · Analyze competitive landscape and market trends to develop product strategies. · Lead the product management team in the development of product plans, roadmaps and launch plans. · Monitor the performance of product management team, analyze results and implement corrective action as needed. · Manage the product lifecycle, including product development, launch, and end of life. · Ensure product features and benefits meet customer requirements. · Establish and maintain relationships with key customers, partners, and vendors." - - "role_library.pdf#page=28: · 7+ years of experience in research and development in the electronics sector. · Proven track record of successfully designing, testing, and optimizing products. · Experience leading a team of researchers and engineers. · Excellent problem-solving and analytical skills. · Ability to work in a fast-paced environment and meet tight deadlines.· Knowledge of industry trends, technologies, and regulations. · Excellent communication and presentation skills. Manager of Product Management Job Title: Manager of Product Management, Contoso Electronics Job Summary: The Manager of Product Management is responsible for overseeing the product management team, driving product development and marketing strategy for Contoso Electronics. This individual will be accountable for the successful launch of new products and the implementation of product life-cycle management processes. The Manager of Product Management will collaborate with internal teams, such as engineering, sales, marketing, and finance, as well as external partners, suppliers, and customers to ensure successful product execution." + - "structural_standards.pdf#page=15: Steel beam connections must be designed to resist the applied loads including dead loads, live loads, wind loads, and seismic forces. Connection design shall follow AISC 360 specifications. Bolted connections: Use high-strength bolts (A325 or A490) with proper edge distances and spacing. Minimum edge distance shall be 1.25 times bolt diameter. Maximum spacing between bolts shall not exceed 24 times the thickness of the thinner connected part. Welded connections: Use E70XX electrodes for A992 steel. Fillet welds shall have minimum size per AISC Table J2.4. Full penetration welds required for moment connections." + - "design_manual.pdf#page=42: Company: SteelWorks Engineering Location: Multiple Offices Job Summary: Structural Engineer responsibilities include analyzing structural systems, designing steel and concrete members, preparing construction documents, and ensuring compliance with building codes. The engineer will collaborate with architects, contractors, and other disciplines to deliver safe and economical structural solutions. Responsibilities: · Perform structural analysis using software such as SAP2000, ETABS, or STAAD.Pro · Design structural members per AISC, ACI, and IBC codes · Prepare detailed drawings and specifications · Conduct site visits and construction administration · Review shop drawings and submittals · Coordinate with MEP engineers and architects" + - "project_specs.pdf#page=8: Seismic Design Requirements: All structures in Seismic Design Category D must include special moment-resisting frames or braced frames. Base shear calculations shall use the equivalent lateral force procedure per ASCE 7. Response modification factor R shall be selected based on structural system type. Special detailing requirements apply to beam-column connections in high seismic zones. Drift limits: Story drift shall not exceed 0.020hsx for Risk Category I and II buildings." --- system: {% if override_prompt %} {{ override_prompt }} {% else %} -Assistant helps the company employees with their questions about internal documents. Be brief in your answers. -Answer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. +Assistant helps structural engineering company employees with their questions about technical documents, design standards, project specifications, and engineering practices. Be brief and technically accurate in your answers. + +You have access to two types of sources: +1. **Corpus sources** (document files like .pdf, .txt) - These are from the company's internal knowledge base. Prioritize these when they contain relevant information. +2. **Web sources** (URLs starting with http:// or https://) - These are from web search. Use these when corpus sources don't contain the answer. + +**Answering rules:** +- **CRITICAL**: Only use and cite documents that are directly relevant to answering the specific question asked. If the question asks about a specific document or topic, ONLY use information from documents related to that topic. +- If corpus sources contain the answer, use them and cite them with [filename.pdf#page=N] format. +- If corpus sources don't contain the answer but web sources do, use web sources and cite them with [URL] format. +- Only say "I don't know" if neither corpus nor web sources contain enough information to answer the question. +- **CRITICAL**: Only cite documents that you actually use in your answer. If multiple documents are retrieved but only one is relevant to the question, ONLY use and cite that one document. Do not include information or citations from irrelevant documents, even if they were retrieved. +- If a question asks about a specific document (e.g., "code review documents"), only use information from that specific document type, not other unrelated documents. +- Always cite your sources. Use square brackets to reference sources, for example [document.pdf#page=1] for corpus or [https://example.com] for web sources. +- Don't combine sources, list each source separately, for example [doc1.pdf#page=1][doc2.pdf#page=2] or [https://site1.com][https://site2.com]. + You CANNOT ask clarifying questions to the user, since the user will have no way to reply. If the question is not in English, answer in the language used in the question. -Each source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]. {% if image_sources %} Each image source has the document file name in the top left corner of the image with coordinates (10,10) pixels with format , and the image figure name is right-aligned in the top right corner of the image. @@ -27,21 +40,24 @@ Each text source starts in a new line and has the file name followed by colon an Always include the source document filename for each fact you use in the response in the format: [document_name.ext#page=N]. If you are referencing an image, add the image filename in the format: [document_name.ext#page=N(image_name.png)]. {% endif %} -Possible citations for current question: {% for citation in citations %} [{{ citation }}] {% endfor %} +Available sources for this question (you may not need all of them): {% for citation in citations %} [{{ citation }}] {% endfor %} + +**IMPORTANT**: Only cite documents that you actually use in your answer. Do not cite documents that don't contain relevant information for the question. If a document is retrieved but doesn't help answer the question, do not include it in your citations. + {{ injected_prompt }} {% endif %} user: -What is the deductible for the employee plan for a visit to Overlake in Bellevue? +What is the maximum allowable deflection for a steel beam under live load? Sources: -info1.txt: deductibles depend on whether you are in-network or out-of-network. In-network deductibles are $500 for employee and $1000 for family. Out-of-network deductibles are $1000 for employee and $2000 for family. -info2.pdf: Overlake is in-network for the employee plan. -info3.pdf: Overlake is the name of the area that includes a park and ride near Bellevue. -info4.pdf: In-network institutions include Overlake, Swedish and others in the region. +code_requirements.pdf: Maximum deflection limits for structural members under live load shall not exceed L/360 for floors supporting plaster ceilings, L/300 for floors not supporting plaster ceilings, and L/240 for roof members. +design_guide.pdf: Steel beam deflection calculations must include both immediate deflection and long-term deflection effects. +project_manual.pdf: Special deflection limits may apply to equipment-sensitive areas where L/480 or stricter limits are required. +aisc_manual.pdf: Deflection limits are specified in IBC Table 1604.3 and should be verified during design phase. assistant: -In-network deductibles are $500 for employee and $1000 for family [info1.txt] and Overlake is in-network for the employee plan [info2.pdf][info4.pdf]. +Maximum deflection limits for steel beams under live load are L/360 for floors supporting plaster ceilings, L/300 for floors not supporting plaster ceilings, and L/240 for roof members [code_requirements.pdf]. Special deflection limits may apply to equipment-sensitive areas where L/480 or stricter limits are required [project_manual.pdf]. user: {{ user_query }} diff --git a/app/backend/approaches/prompts/chat_answer_question.prompty b/app/backend/approaches/prompts/chat_answer_question.prompty index c1a7fc1709..9f65a10897 100644 --- a/app/backend/approaches/prompts/chat_answer_question.prompty +++ b/app/backend/approaches/prompts/chat_answer_question.prompty @@ -4,27 +4,40 @@ description: Answer a question (with chat history) using solely text sources. model: api: chat sample: - user_query: What does a product manager do that a CEO doesn't? + user_query: What's the difference between ASD and LRFD design methods? include_follow_up_questions: true past_messages: - role: user - content: "What does a CEO do?" + content: "What design methods are used for steel structures?" - role: assistant - content: "A CEO, or Chief Executive Officer, is responsible for providing strategic direction and oversight to a company to ensure its long-term success and profitability. They develop and implement strategies and objectives for financial success and growth, provide guidance to the executive team, manage day-to-day operations, ensure compliance with laws and regulations, develop and maintain relationships with stakeholders, monitor industry trends, and represent the company in public events 12. [role_library.pdf#page=1][role_library.pdf#page=3]" + content: "Steel structures can be designed using two primary methods: Allowable Stress Design (ASD) and Load and Resistance Factor Design (LRFD). Both methods are acceptable per AISC 360 specifications. ASD uses a factor of safety applied to allowable stresses, while LRFD uses load factors and resistance factors to account for uncertainties in loads and material properties. [design_standards.pdf#page=12][aisc_manual.pdf#page=2-3]" text_sources: - - "role_library.pdf#page=29: The Manager of Product Management will collaborate with internal teams, such as engineering, sales, marketing, and finance, as well as external partners, suppliers, and customers to ensure successful product execution. Responsibilities: · Lead the product management team and provide guidance on product strategy, design, development, and launch. · Develop and implement product life-cycle management processes. · Monitor and analyze industry trends to identify opportunities for new products. · Develop product marketing plans and go-to-market strategies. · Research customer needs and develop customer-centric product roadmaps. · Collaborate with internal teams to ensure product execution and successful launch. · Develop pricing strategies and cost models. · Oversee product portfolio and performance metrics. · Manage product development budget. · Analyze product performance and customer feedback to identify areas for improvement. Qualifications: · Bachelor's degree in business, engineering, or a related field. · At least 5 years of experience in product management. · Proven track record of successful product launches." - - "role_library.pdf#page=23: Company: Contoso Electronics Location: Anywhere Job Type: Full-Time Salary: Competitive, commensurate with experience Job Summary: The Senior Manager of Product Management will be responsible for leading the product management team at Contoso Electronics. This role includes developing strategies, plans and objectives for the product management team and managing the day-to-day operations. The Senior Manager of Product Management will be responsible for the successful launch of new products and the optimization of existing products. Responsibilities: · Develop and implement product management strategies, plans and objectives to maximize team performance. · Analyze competitive landscape and market trends to develop product strategies. · Lead the product management team in the development of product plans, roadmaps and launch plans. · Monitor the performance of product management team, analyze results and implement corrective action as needed. · Manage the product lifecycle, including product development, launch, and end of life. · Ensure product features and benefits meet customer requirements. · Establish and maintain relationships with key customers, partners, and vendors." - - "role_library.pdf#page=28: · 7+ years of experience in research and development in the electronics sector. · Proven track record of successfully designing, testing, and optimizing products. · Experience leading a team of researchers and engineers. · Excellent problem-solving and analytical skills. · Ability to work in a fast-paced environment and meet tight deadlines.· Knowledge of industry trends, technologies, and regulations. · Excellent communication and presentation skills. Manager of Product Management Job Title: Manager of Product Management, Contoso Electronics Job Summary: The Manager of Product Management is responsible for overseeing the product management team, driving product development and marketing strategy for Contoso Electronics. This individual will be accountable for the successful launch of new products and the implementation of product life-cycle management processes. The Manager of Product Management will collaborate with internal teams, such as engineering, sales, marketing, and finance, as well as external partners, suppliers, and customers to ensure successful product execution." + - "design_standards.pdf#page=15: ASD (Allowable Stress Design) uses a single factor of safety applied to the nominal strength to determine allowable stress. The basic equation is: Required Strength ≤ Allowable Strength = Nominal Strength / Ω (safety factor). ASD has been traditionally used and is familiar to many engineers. Load combinations for ASD are simpler and use service-level loads without load factors." + - "aisc_manual.pdf#page=8: LRFD (Load and Resistance Factor Design) uses separate factors for loads and resistance. The basic equation is: Required Strength ≤ Design Strength = φ × Nominal Strength. Load factors account for variability and uncertainty in loads, while resistance factors account for variability in material properties and construction. LRFD generally results in more economical designs and is the preferred method for new construction." + - "structural_code.pdf#page=22: Both ASD and LRFD methods are permitted by AISC 360. The choice between methods often depends on project requirements, engineer preference, and client specifications. Mixed use of both methods in the same project is not recommended. Connection design should use the same method as the member design for consistency." --- system: {% if override_prompt %} {{ override_prompt }} {% else %} -Assistant helps the company employees with their questions about internal documents. Be brief in your answers. -Answer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. +Assistant helps structural engineering company employees with their questions about technical documents, design standards, project specifications, and engineering practices. Be brief and technically accurate in your answers. + +You have access to two types of sources: +1. **Corpus sources** (document files like .pdf, .txt) - These are from the company's internal knowledge base. Prioritize these when they contain relevant information. +2. **Web sources** (URLs starting with http:// or https://) - These are from web search. Use these when corpus sources don't contain the answer. + +**Answering rules:** +- **CRITICAL**: Only use and cite documents that are directly relevant to answering the specific question asked. If the question asks about a specific document or topic, ONLY use information from documents related to that topic. +- If corpus sources contain the answer, use them and cite them with [filename.pdf#page=N] format. +- If corpus sources don't contain the answer but web sources do, use web sources and cite them with [URL] format. +- Only say "I don't know" if neither corpus nor web sources contain enough information to answer the question. +- **CRITICAL**: Only cite documents that you actually use in your answer. If multiple documents are retrieved but only one is relevant to the question, ONLY use and cite that one document. Do not include information or citations from irrelevant documents, even if they were retrieved. +- If a question asks about a specific document (e.g., "code review documents"), only use information from that specific document type, not other unrelated documents. +- Always cite your sources. Use square brackets to reference sources, for example [document.pdf#page=1] for corpus or [https://example.com] for web sources. +- Don't combine sources, list each source separately, for example [doc1.pdf#page=1][doc2.pdf#page=2] or [https://site1.com][https://site2.com]. + If asking a clarifying question to the user would help, ask the question. If the question is not in English, answer in the language used in the question. -Each source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]. {% if image_sources %} Each image source has the document file name in the top left corner of the image with coordinates (10,10) pixels with format , and the image figure name is right-aligned in the top right corner of the image. @@ -33,7 +46,10 @@ Each text source starts in a new line and has the file name followed by colon an Always include the source document filename for each fact you use in the response in the format: [document_name.ext#page=N]. If you are referencing an image, add the image filename in the format: [document_name.ext#page=N(image_name.png)]. {% endif %} -Possible citations for current question: {% for citation in citations %} [{{ citation }}] {% endfor %} +Available sources for this question (you may not need all of them): {% for citation in citations %} [{{ citation }}] {% endfor %} + +**IMPORTANT**: Only cite documents that you actually use in your answer. Do not cite documents that don't contain relevant information for the question. If a document is retrieved but doesn't help answer the question, do not include it in your citations. + {{ injected_prompt }} {% endif %} diff --git a/app/backend/approaches/prompts/chat_query_rewrite.prompty b/app/backend/approaches/prompts/chat_query_rewrite.prompty index 545b3f5b8c..9b60c84f99 100644 --- a/app/backend/approaches/prompts/chat_query_rewrite.prompty +++ b/app/backend/approaches/prompts/chat_query_rewrite.prompty @@ -6,12 +6,12 @@ model: parameters: tools: ${file:chat_query_rewrite_tools.json} sample: - user_query: Does it include hearing? + user_query: Does it apply to seismic zones? past_messages: - role: user - content: "What is included in my Northwind Health Plus plan that is not in standard?" + content: "What are the special detailing requirements for moment connections?" - role: assistant - content: "The Northwind Health Plus plan includes coverage for emergency services, mental health and substance abuse coverage, and out-of-network services, which are not included in the Northwind Standard plan. [Benefit_Options.pdf#page=3]" + content: "Special detailing requirements for moment connections include reduced beam sections (RBS), welded flange-bolted web connections, and extended end-plate connections. These connections must satisfy both strength and ductility requirements per AISC 341. [seismic_design.pdf#page=15]" --- system: Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base. @@ -24,16 +24,16 @@ If the question is not in English, translate the question to English before gene If you cannot generate a search query, return just the number 0. user: -How did crypto do last year? +What are the wind load requirements for high-rise buildings? assistant: -Summarize Cryptocurrency Market Dynamics from last year +Wind load requirements high-rise buildings structural design user: -What are my health plans? +What is the minimum concrete strength for footings? assistant: -Show available health plans +Minimum concrete strength footings foundation design requirements {% for message in past_messages %} {{ message["role"] }}: diff --git a/app/backend/approaches/retrievethenread.py b/app/backend/approaches/retrievethenread.py index ca92eaff64..81a6fb9c26 100644 --- a/app/backend/approaches/retrievethenread.py +++ b/app/backend/approaches/retrievethenread.py @@ -118,6 +118,17 @@ async def run( response_token_limit=self.get_response_token_limit(self.chatgpt_model, 1024), ), ) + answer_content = chat_completion.choices[0].message.content + + # Filter citations to only include those actually used in the answer + from services.citation_filter import filter_citations_by_answer + if extra_info.data_points.citations and answer_content: + filtered_citations = filter_citations_by_answer( + extra_info.data_points.citations, + answer_content + ) + extra_info.data_points.citations = filtered_citations + extra_info.thoughts.append( self.format_thought_step_for_chatcompletion( title="Prompt to generate answer", @@ -130,7 +141,7 @@ async def run( ) return { "message": { - "content": chat_completion.choices[0].message.content, + "content": answer_content, "role": chat_completion.choices[0].message.role, }, "context": { diff --git a/app/backend/chat_history/__init__.py b/app/backend/chat_history/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/app/backend/config.py b/app/backend/config.py index 947f546776..a2354f59f3 100644 --- a/app/backend/config.py +++ b/app/backend/config.py @@ -35,3 +35,29 @@ CONFIG_RAG_SEARCH_IMAGE_EMBEDDINGS = "rag_search_image_embeddings" CONFIG_RAG_SEND_TEXT_SOURCES = "rag_send_text_sources" CONFIG_RAG_SEND_IMAGE_SOURCES = "rag_send_image_sources" +CONFIG_CACHE = "cache" + +# Feature flags and provider keys (Phase 1B scaffolding) +import os + +ENABLE_WEB_SEARCH = os.getenv("ENABLE_WEB_SEARCH", "false").lower() == "true" +SERPER_API_KEY = os.getenv("SERPER_API_KEY") +WEB_CACHE_TTL_S = int(os.getenv("WEB_CACHE_TTL_S", "3600")) +REDIS_URL = os.getenv("REDIS_URL") # Optional Redis cache URL + +# OCR Configuration +OCR_PROVIDER = os.getenv("OCR_PROVIDER", "none").lower() # ollama, azure_document_intelligence, none +OCR_ON_INGEST = os.getenv("OCR_ON_INGEST", "false").lower() == "true" # Run OCR during document ingestion +OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://localhost:11434/v1") +OLLAMA_OCR_MODEL = os.getenv("OLLAMA_OCR_MODEL", "llava:7b") # Must be a vision-capable model (llava, bakllava, etc.) +OLLAMA_TIMEOUT = int(os.getenv("OLLAMA_TIMEOUT", "120")) +AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT = os.getenv("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT") +AZURE_DOCUMENT_INTELLIGENCE_KEY = os.getenv("AZURE_DOCUMENT_INTELLIGENCE_KEY") +AZURE_DOCUMENT_INTELLIGENCE_MODEL = os.getenv("AZURE_DOCUMENT_INTELLIGENCE_MODEL", "prebuilt-read") + +# NOMIC Embeddings Configuration +NOMIC_API_KEY = os.getenv("NOMIC_API_KEY") +NOMIC_ENDPOINT = os.getenv("NOMIC_ENDPOINT") # Optional custom endpoint +NOMIC_USE_SDK = os.getenv("NOMIC_USE_SDK", "false").lower() == "true" # Use Python SDK instead of API +NOMIC_INFERENCE_MODE = os.getenv("NOMIC_INFERENCE_MODE", "remote").lower() # local or remote (SDK only) +ENABLE_NOMIC_EMBEDDINGS = os.getenv("ENABLE_NOMIC_EMBEDDINGS", "false").lower() == "true" \ No newline at end of file diff --git a/app/backend/core/keyvault_config.py b/app/backend/core/keyvault_config.py new file mode 100644 index 0000000000..21777a9a72 --- /dev/null +++ b/app/backend/core/keyvault_config.py @@ -0,0 +1,115 @@ +""" +Key Vault Configuration Helper. + +Provides a centralized way to load configuration from Key Vault or environment variables. +""" + +import logging +import os +from typing import Optional, Dict, Any +from services.keyvault_secrets import KeyVaultSecretReader + +logger = logging.getLogger(__name__) + + +class KeyVaultConfigLoader: + """ + Loads application configuration from Key Vault with environment variable fallback. + + This is a convenience wrapper around KeyVaultSecretReader that provides + application-specific secret loading. + """ + + def __init__(self, key_vault_url: Optional[str] = None, credential: Optional[Any] = None): + """ + Initialize config loader. + + Args: + key_vault_url: Key Vault URL (optional, will use AZURE_KEY_VAULT_ENDPOINT env var) + credential: Azure credential (optional) + """ + self.reader = KeyVaultSecretReader( + key_vault_url=key_vault_url, + credential=credential, + enable_keyvault=True + ) + + async def load_bot_secrets(self) -> Dict[str, Optional[str]]: + """ + Load Bot Framework secrets from Key Vault. + + Returns: + Dictionary with MICROSOFT_APP_ID and MICROSOFT_APP_PASSWORD + """ + return await self.reader.get_secrets({ + "MICROSOFT_APP_ID": "MICROSOFT_APP_ID", + "MICROSOFT_APP_PASSWORD": "MICROSOFT_APP_PASSWORD" + }) + + async def load_azure_secrets(self) -> Dict[str, Optional[str]]: + """ + Load Azure service secrets from Key Vault. + + Returns: + Dictionary with Azure service keys + """ + return await self.reader.get_secrets({ + "AZURE_SEARCH_KEY": "AZURE_SEARCH_KEY", + "AZURE_OPENAI_API_KEY": "AZURE_OPENAI_API_KEY", + "AZURE_CLIENT_SECRET": "AZURE_CLIENT_SECRET", + "AZURE_DOCUMENT_INTELLIGENCE_KEY": "AZURE_DOCUMENT_INTELLIGENCE_KEY" + }) + + async def load_web_search_secrets(self) -> Dict[str, Optional[str]]: + """ + Load web search provider secrets from Key Vault. + + Returns: + Dictionary with web search API keys + """ + return await self.reader.get_secrets({ + "SERPER_API_KEY": "SERPER_API_KEY" + }) + + async def load_all_secrets(self) -> Dict[str, Optional[str]]: + """ + Load all application secrets from Key Vault. + + Returns: + Dictionary with all secrets + """ + bot_secrets = await self.load_bot_secrets() + azure_secrets = await self.load_azure_secrets() + web_secrets = await self.load_web_search_secrets() + + return {**bot_secrets, **azure_secrets, **web_secrets} + + async def close(self): + """Close the Key Vault reader.""" + await self.reader.close() + + +async def get_secret_from_keyvault_or_env( + secret_name: str, + env_var_name: Optional[str] = None, + key_vault_url: Optional[str] = None, + credential: Optional[Any] = None +) -> Optional[str]: + """ + Convenience function to get a secret from Key Vault or environment variable. + + Args: + secret_name: Name of the secret in Key Vault + env_var_name: Optional environment variable name (defaults to secret_name) + key_vault_url: Optional Key Vault URL (uses AZURE_KEY_VAULT_ENDPOINT if not provided) + credential: Optional Azure credential + + Returns: + Secret value or None if not found + """ + reader = KeyVaultSecretReader(key_vault_url=key_vault_url, credential=credential) + try: + return await reader.get_secret(secret_name, env_var_name) + finally: + await reader.close() + diff --git a/app/backend/gunicorn.conf.py b/app/backend/gunicorn.conf.py deleted file mode 100644 index 9144e3cc00..0000000000 --- a/app/backend/gunicorn.conf.py +++ /dev/null @@ -1,18 +0,0 @@ -import multiprocessing -import os - -max_requests = 1000 -max_requests_jitter = 50 -log_file = "-" -bind = "0.0.0.0" - -timeout = 230 -# https://learn.microsoft.com/troubleshoot/azure/app-service/web-apps-performance-faqs#why-does-my-request-time-out-after-230-seconds - -num_cpus = multiprocessing.cpu_count() -if os.getenv("WEBSITE_SKU") == "LinuxFree": - # Free tier reports 2 CPUs but can't handle multiple workers - workers = 1 -else: - workers = (num_cpus * 2) + 1 -worker_class = "custom_uvicorn_worker.CustomUvicornWorker" diff --git a/app/backend/load_azd_env.py b/app/backend/load_azd_env.py index 2f2db6aa8f..9a37519595 100644 --- a/app/backend/load_azd_env.py +++ b/app/backend/load_azd_env.py @@ -12,14 +12,32 @@ def load_azd_env(): """Get path to current azd env file and load file using python-dotenv""" result = subprocess.run("azd env list -o json", shell=True, capture_output=True, text=True) if result.returncode != 0: - raise Exception("Error loading azd env") + # Fallback to loading .env file if azd is not available + logger.info("azd not available, attempting to load .env file instead") + env_file_path = os.path.join(os.path.dirname(__file__), ".env") + if os.path.exists(env_file_path): + logger.info("Loading .env from %s", env_file_path) + load_dotenv(env_file_path, override=True) + return + else: + logger.warning("No .env file found at %s", env_file_path) + return env_json = json.loads(result.stdout) env_file_path = None for entry in env_json: if entry["IsDefault"]: env_file_path = entry["DotEnvPath"] if not env_file_path: - raise Exception("No default azd env file found") + # Fallback to .env file + logger.info("No default azd env found, attempting to load .env file instead") + env_file_path = os.path.join(os.path.dirname(__file__), ".env") + if os.path.exists(env_file_path): + logger.info("Loading .env from %s", env_file_path) + load_dotenv(env_file_path, override=True) + return + else: + logger.warning("No .env file found at %s", env_file_path) + return loading_mode = os.getenv("LOADING_MODE_FOR_AZD_ENV_VARS") or "override" if loading_mode == "no-override": logger.info("Loading azd env from %s, but not overriding existing environment variables", env_file_path) @@ -27,3 +45,9 @@ def load_azd_env(): else: logger.info("Loading azd env from %s, which may override existing environment variables", env_file_path) load_dotenv(env_file_path, override=True) + + # Also load from local .env file as fallback (for variables not in azd env) + local_env_path = os.path.join(os.path.dirname(__file__), ".env") + if os.path.exists(local_env_path) and local_env_path != env_file_path: + logger.info("Also loading local .env from %s (as fallback for missing variables)", local_env_path) + load_dotenv(local_env_path, override=False) # Don't override azd vars, but fill in missing ones diff --git a/app/backend/prepdocs.py b/app/backend/prepdocs.py index 7ec157c08a..886152a46f 100644 --- a/app/backend/prepdocs.py +++ b/app/backend/prepdocs.py @@ -20,6 +20,8 @@ ImageEmbeddings, OpenAIEmbeddingService, ) +from prepdocslib.patentsberta_embeddings import PatentsBertaEmbeddings +from prepdocslib.nomic_embeddings import NomicEmbeddings from prepdocslib.fileprocessor import FileProcessor from prepdocslib.filestrategy import FileStrategy from prepdocslib.htmlparser import LocalHTMLParser @@ -29,6 +31,7 @@ from prepdocslib.jsonparser import JsonParser from prepdocslib.listfilestrategy import ( ADLSGen2ListFileStrategy, + AzureBlobListFileStrategy, ListFileStrategy, LocalListFileStrategy, ) @@ -41,6 +44,8 @@ from prepdocslib.strategy import DocumentAction, SearchInfo, Strategy from prepdocslib.textparser import TextParser from prepdocslib.textsplitter import SentenceTextSplitter, SimpleTextSplitter +from services.ocr_service import OCRService +from config import OCR_ON_INGEST logger = logging.getLogger("scripts") @@ -127,6 +132,10 @@ def setup_list_file_strategy( datalake_filesystem: Union[str, None], datalake_path: Union[str, None], datalake_key: Union[str, None], + blob_storage_account: Union[str, None], + blob_storage_container: Union[str, None], + blob_path_prefix: Union[str, None], + blob_storage_key: Union[str, None], ): list_file_strategy: ListFileStrategy if datalake_storage_account: @@ -140,11 +149,24 @@ def setup_list_file_strategy( data_lake_path=datalake_path, credential=adls_gen2_creds, ) + elif blob_storage_account and blob_storage_container: + blob_creds: Union[AsyncTokenCredential, str] = azure_credential if blob_storage_key is None else blob_storage_key + logger.info( + "Using Azure Blob Storage container '%s' in account '%s'", + blob_storage_container, + blob_storage_account, + ) + list_file_strategy = AzureBlobListFileStrategy( + storage_account=blob_storage_account, + storage_container=blob_storage_container, + credential=blob_creds, + path_prefix=blob_path_prefix, + ) elif local_files: logger.info("Using local files: %s", local_files) list_file_strategy = LocalListFileStrategy(path_pattern=local_files) else: - raise ValueError("Either local_files or datalake_storage_account must be provided.") + raise ValueError("Provide either local_files, Azure Blob Storage details, or datalake_storage_account.") return list_file_strategy @@ -153,6 +175,8 @@ class OpenAIHost(str, Enum): AZURE = "azure" AZURE_CUSTOM = "azure_custom" LOCAL = "local" + PATENTSBERTA = "patentsberta" + NOMIC = "nomic" def setup_embeddings_service( @@ -169,12 +193,38 @@ def setup_embeddings_service( openai_org: Union[str, None], disable_vectors: bool = False, disable_batch_vectors: bool = False, + patentsberta_endpoint: Union[str, None] = None, + patentsberta_api_key: Union[str, None] = None, + nomic_endpoint: Union[str, None] = None, + nomic_api_key: Union[str, None] = None, + nomic_model: str = "nomic-embed-text-v1.5", + nomic_use_sdk: bool = False, ): if disable_vectors: logger.info("Not setting up embeddings service") return None - if openai_host in [OpenAIHost.AZURE, OpenAIHost.AZURE_CUSTOM]: + if openai_host == OpenAIHost.PATENTSBERTA: + if patentsberta_endpoint is None: + raise ValueError("PATENTSBERTA_ENDPOINT environment variable required for PatentsBERTa embeddings") + logger.info("Setting up PatentsBERTa embedding service") + return PatentsBertaEmbeddings( + endpoint=patentsberta_endpoint, + api_key=patentsberta_api_key, + batch_size=16, + max_retries=3 + ) + elif openai_host == OpenAIHost.NOMIC: + logger.info(f"Setting up NOMIC embedding service (model: {nomic_model})") + return NomicEmbeddings( + model=nomic_model, + api_key=nomic_api_key, + endpoint=nomic_endpoint, + use_sdk=nomic_use_sdk, + batch_size=16, + max_retries=3 + ) + elif openai_host in [OpenAIHost.AZURE, OpenAIHost.AZURE_CUSTOM]: azure_open_ai_credential: Union[AsyncTokenCredential, AzureKeyCredential] = ( azure_credential if azure_openai_key is None else AzureKeyCredential(azure_openai_key) ) @@ -245,6 +295,14 @@ def setup_openai_client( base_url=os.environ["OPENAI_BASE_URL"], api_key="no-key-required", ) + elif openai_host == OpenAIHost.PATENTSBERTA: + logger.info("OPENAI_HOST is patentsberta, using PatentsBERTa for embeddings only - no OpenAI client needed for chat") + # For PatentsBERTa, we only use the embedding service, but we still need a dummy OpenAI client + # for any potential chat completions (though they won't be used in embedding-only scenarios) + openai_client = AsyncOpenAI( + api_key="not-needed-for-patentsberta", + base_url="https://api.openai.com/v1", # Dummy URL, won't be used + ) else: logger.info( "OPENAI_HOST is not azure, setting up OpenAI client using OPENAI_API_KEY and OPENAI_ORGANIZATION environment variables" @@ -491,13 +549,18 @@ async def main(strategy: Strategy, setup_index: bool = True): ) exit(1) + storage_key_cli = clean_key_if_exists(args.storagekey) + storage_key_env = clean_key_if_exists(os.getenv("AZURE_STORAGE_KEY")) + storage_sas_env = clean_key_if_exists(os.getenv("AZURE_STORAGE_SAS_TOKEN")) + resolved_storage_key = storage_key_cli or storage_key_env or storage_sas_env + blob_manager = setup_blob_manager( azure_credential=azd_credential, storage_account=os.environ["AZURE_STORAGE_ACCOUNT"], storage_container=os.environ["AZURE_STORAGE_CONTAINER"], storage_resource_group=os.environ["AZURE_STORAGE_RESOURCE_GROUP"], subscription_id=os.environ["AZURE_SUBSCRIPTION_ID"], - storage_key=clean_key_if_exists(args.storagekey), + storage_key=resolved_storage_key, image_storage_container=os.environ.get("AZURE_IMAGESTORAGE_CONTAINER"), # Pass the image container ) list_file_strategy = setup_list_file_strategy( @@ -507,6 +570,10 @@ async def main(strategy: Strategy, setup_index: bool = True): datalake_filesystem=os.getenv("AZURE_ADLS_GEN2_FILESYSTEM"), datalake_path=os.getenv("AZURE_ADLS_GEN2_FILESYSTEM_PATH"), datalake_key=clean_key_if_exists(args.datalakekey), + blob_storage_account=os.getenv("AZURE_STORAGE_ACCOUNT"), + blob_storage_container=os.getenv("AZURE_STORAGE_CONTAINER"), + blob_path_prefix=os.getenv("AZURE_STORAGE_BLOB_PREFIX"), + blob_storage_key=resolved_storage_key, ) # https://learn.microsoft.com/azure/ai-services/openai/api-version-deprecation#latest-ga-api-release @@ -517,7 +584,7 @@ async def main(strategy: Strategy, setup_index: bool = True): openai_embeddings_service = setup_embeddings_service( azure_credential=azd_credential, openai_host=OPENAI_HOST, - emb_model_name=os.environ["AZURE_OPENAI_EMB_MODEL_NAME"], + emb_model_name=os.environ.get("AZURE_OPENAI_EMB_MODEL_NAME", "PatentSBERTa"), emb_model_dimensions=emb_model_dimensions, azure_openai_service=os.getenv("AZURE_OPENAI_SERVICE"), azure_openai_custom_url=os.getenv("AZURE_OPENAI_CUSTOM_URL"), @@ -528,6 +595,12 @@ async def main(strategy: Strategy, setup_index: bool = True): openai_org=os.getenv("OPENAI_ORGANIZATION"), disable_vectors=dont_use_vectors, disable_batch_vectors=args.disablebatchvectors, + patentsberta_endpoint=os.getenv("PATENTSBERTA_ENDPOINT"), + patentsberta_api_key=os.getenv("PATENTSBERTA_API_KEY"), + nomic_endpoint=os.getenv("NOMIC_ENDPOINT"), + nomic_api_key=os.getenv("NOMIC_API_KEY"), + nomic_model=os.getenv("NOMIC_MODEL", "nomic-embed-text-v1.5"), + nomic_use_sdk=os.getenv("NOMIC_USE_SDK", "false").lower() == "true", ) openai_client = setup_openai_client( openai_host=OPENAI_HOST, @@ -579,6 +652,14 @@ async def main(strategy: Strategy, setup_index: bool = True): use_multimodal=use_multimodal, ) + ocr_service = None + if OCR_ON_INGEST: + ocr_candidate = OCRService() + if ocr_candidate.is_enabled(): + ocr_service = ocr_candidate + else: + logger.warning("OCR_ON_INGEST is enabled but no OCR provider is configured; skipping OCR.") + ingestion_strategy = FileStrategy( search_info=search_info, list_file_strategy=list_file_strategy, @@ -594,6 +675,8 @@ async def main(strategy: Strategy, setup_index: bool = True): category=args.category, use_content_understanding=use_content_understanding, content_understanding_endpoint=os.getenv("AZURE_CONTENTUNDERSTANDING_ENDPOINT"), + ocr_service=ocr_service, + ocr_on_ingest=ocr_service is not None and OCR_ON_INGEST, ) try: diff --git a/app/backend/prepdocslib/filestrategy.py b/app/backend/prepdocslib/filestrategy.py index 2f0e6e6a52..83be610aa7 100644 --- a/app/backend/prepdocslib/filestrategy.py +++ b/app/backend/prepdocslib/filestrategy.py @@ -1,5 +1,5 @@ import logging -from typing import Optional +from typing import Optional, Any, Dict, Set from azure.core.credentials import AzureKeyCredential @@ -21,6 +21,8 @@ async def parse_file( blob_manager: Optional[BaseBlobManager] = None, image_embeddings_client: Optional[ImageEmbeddings] = None, user_oid: Optional[str] = None, + ocr_service: Optional[Any] = None, + enable_ocr: bool = False, ) -> list[Section]: key = file.file_extension().lower() processor = file_processors.get(key) @@ -46,6 +48,38 @@ async def parse_file( section.chunk.images = [ image for page in pages if page.page_num == section.chunk.page_num for image in page.images ] + if enable_ocr and ocr_service is not None: + ocr_cache: Dict[str, str] = {} + for section in sections: + if not section.chunk.images: + continue + processed_ids: Set[str] = set() + for image in section.chunk.images: + if image is None: + continue + image_id = image.figure_id or image.filename or f"{file.filename()}-{image.page_num}" + if image_id in processed_ids: + continue + processed_ids.add(image_id) + if image_id not in ocr_cache: + if not getattr(image, "bytes", None): + ocr_cache[image_id] = "" + else: + try: + ocr_result = await ocr_service.extract_text(image.bytes) + if ocr_result and getattr(ocr_result, "text", None): + extracted_text = ocr_result.text.strip() + ocr_cache[image_id] = extracted_text + image.ocr_text = extracted_text or None + else: + ocr_cache[image_id] = "" + except Exception as exc: + logger.warning("Failed to run OCR for image %s: %s", image.filename, exc) + ocr_cache[image_id] = "" + ocr_text = ocr_cache.get(image_id, "").strip() + if ocr_text: + image.ocr_text = ocr_text + section.chunk.text = f"{section.chunk.text.rstrip()}\n\n[Image OCR: {image_id}]\n{ocr_text}" return sections @@ -69,6 +103,8 @@ def __init__( category: Optional[str] = None, use_content_understanding: bool = False, content_understanding_endpoint: Optional[str] = None, + ocr_service: Optional[Any] = None, + ocr_on_ingest: bool = False, ): self.list_file_strategy = list_file_strategy self.blob_manager = blob_manager @@ -83,6 +119,8 @@ def __init__( self.category = category self.use_content_understanding = use_content_understanding self.content_understanding_endpoint = content_understanding_endpoint + self.ocr_service = ocr_service + self.ocr_on_ingest = ocr_on_ingest def setup_search_manager(self): self.search_manager = SearchManager( @@ -117,7 +155,13 @@ async def run(self): try: await self.blob_manager.upload_blob(file) sections = await parse_file( - file, self.file_processors, self.category, self.blob_manager, self.image_embeddings + file, + self.file_processors, + self.category, + self.blob_manager, + self.image_embeddings, + ocr_service=self.ocr_service if self.ocr_on_ingest else None, + enable_ocr=self.ocr_on_ingest, ) if sections: await self.search_manager.update_content(sections, url=file.url) @@ -147,6 +191,7 @@ def __init__( search_field_name_embedding: Optional[str] = None, embeddings: Optional[OpenAIEmbeddings] = None, image_embeddings: Optional[ImageEmbeddings] = None, + ocr_service: Optional[Any] = None, ): self.file_processors = file_processors self.embeddings = embeddings @@ -163,10 +208,18 @@ def __init__( search_images=False, ) self.search_field_name_embedding = search_field_name_embedding + self.ocr_service = ocr_service async def add_file(self, file: File, user_oid: str): sections = await parse_file( - file, self.file_processors, None, self.blob_manager, self.image_embeddings, user_oid=user_oid + file, + self.file_processors, + None, + self.blob_manager, + self.image_embeddings, + user_oid=user_oid, + ocr_service=self.ocr_service, + enable_ocr=self.ocr_service is not None, ) if sections: await self.search_manager.update_content(sections, url=file.url) diff --git a/app/backend/prepdocslib/listfilestrategy.py b/app/backend/prepdocslib/listfilestrategy.py index 405a623149..99a3d7d041 100644 --- a/app/backend/prepdocslib/listfilestrategy.py +++ b/app/backend/prepdocslib/listfilestrategy.py @@ -1,5 +1,6 @@ import base64 import hashlib +import inspect import logging import os import re @@ -10,9 +11,8 @@ from typing import IO, Optional, Union from azure.core.credentials_async import AsyncTokenCredential -from azure.storage.filedatalake.aio import ( - DataLakeServiceClient, -) +from azure.storage.blob.aio import BlobServiceClient +from azure.storage.filedatalake.aio import DataLakeServiceClient logger = logging.getLogger("scripts") @@ -200,3 +200,78 @@ async def list(self) -> AsyncGenerator[File, None]: os.remove(temp_file_path) except Exception as file_delete_exception: logger.error(f"\tGot an error while deleting {temp_file_path} -> {file_delete_exception}") + + +class AzureBlobListFileStrategy(ListFileStrategy): + """ + Concrete strategy for listing files that are located in a standard Azure Blob Storage container. + """ + + def __init__( + self, + storage_account: str, + storage_container: str, + credential: Union[AsyncTokenCredential, str], + path_prefix: Optional[str] = None, + ): + self.storage_account = storage_account + self.storage_container = storage_container + self.credential = credential + if path_prefix: + # Normalize prefix to avoid double slashes + self.path_prefix = path_prefix.lstrip("/") + else: + self.path_prefix = None + + @staticmethod + async def _close_client(client): + close = getattr(client, "close", None) + if close: + result = close() + if inspect.isawaitable(result): + await result + + async def list_paths(self) -> AsyncGenerator[str, None]: + service_client = BlobServiceClient( + account_url=f"https://{self.storage_account}.blob.core.windows.net", credential=self.credential + ) + container_client = service_client.get_container_client(self.storage_container) + try: + async for blob in container_client.list_blobs(name_starts_with=self.path_prefix or None): + # Skip virtual directories + if blob.name.endswith("/"): + continue + yield blob.name + finally: + await self._close_client(container_client) + await self._close_client(service_client) + + async def list(self) -> AsyncGenerator[File, None]: + service_client = BlobServiceClient( + account_url=f"https://{self.storage_account}.blob.core.windows.net", credential=self.credential + ) + container_client = service_client.get_container_client(self.storage_container) + try: + async for blob in container_client.list_blobs(name_starts_with=self.path_prefix or None): + if blob.name.endswith("/"): + continue + + blob_client = container_client.get_blob_client(blob.name) + temp_file_path = os.path.join(tempfile.gettempdir(), os.path.basename(blob.name)) + + try: + downloader = await blob_client.download_blob() + with open(temp_file_path, "wb") as temp_file: + data = await downloader.readall() + temp_file.write(data) + + yield File(content=open(temp_file_path, "rb"), url=blob_client.url) + except Exception as blob_exception: + logger.error(f"\tGot an error while reading {blob.name} -> {blob_exception} --> skipping file") + try: + os.remove(temp_file_path) + except Exception as file_delete_exception: + logger.error(f"\tGot an error while deleting {temp_file_path} -> {file_delete_exception}") + finally: + await self._close_client(container_client) + await self._close_client(service_client) diff --git a/app/backend/prepdocslib/nomic_embeddings.py b/app/backend/prepdocslib/nomic_embeddings.py new file mode 100644 index 0000000000..34a3e788f3 --- /dev/null +++ b/app/backend/prepdocslib/nomic_embeddings.py @@ -0,0 +1,260 @@ +""" +NOMIC Embeddings Service + +Supports multiple NOMIC embedding models: +- nomic-embed-text-v1.5: General text embeddings +- nomic-embed-code-v1: Code-specific embeddings +- nomic-embed-vision-v1.5: Multimodal (text + image) embeddings + +Can be used via NOMIC Python SDK (local or remote) or via API endpoint. +""" + +import aiohttp +import asyncio +import os +from typing import List, Optional, Literal +import logging + +logger = logging.getLogger("scripts") + + +class NomicEmbeddings: + """ + Class for using NOMIC embeddings. + Supports both SDK-based (Python package) and API-based access. + Follows the same interface pattern as OpenAIEmbeddings for seamless integration. + """ + + # Model dimensions (NOMIC embeddings have fixed dimensions) + MODEL_DIMENSIONS = { + "nomic-embed-text-v1.5": 768, + "nomic-embed-code-v1": 768, + "nomic-embed-vision-v1.5": 768, + } + + # Default task types for different models + DEFAULT_TASK_TYPES = { + "nomic-embed-text-v1.5": "search_document", + "nomic-embed-code-v1": "search_document", + "nomic-embed-vision-v1.5": "search_document", + } + + def __init__( + self, + model: str = "nomic-embed-text-v1.5", + api_key: Optional[str] = None, + endpoint: Optional[str] = None, + use_sdk: bool = False, + inference_mode: Literal["local", "remote"] = "remote", + task_type: Optional[str] = None, + batch_size: int = 16, + max_retries: int = 3, + ): + """ + Initialize NOMIC embeddings service. + + Args: + model: NOMIC model name (e.g., 'nomic-embed-text-v1.5', 'nomic-embed-code-v1') + api_key: NOMIC API key (required for remote mode or API endpoint) + endpoint: Optional custom API endpoint URL (if using custom deployment) + use_sdk: If True, use NOMIC Python SDK. If False, use API endpoint. + inference_mode: 'local' or 'remote' (only used if use_sdk=True) + task_type: Task type ('search_document', 'search_query', 'classification', 'clustering') + batch_size: Batch size for embedding requests + max_retries: Maximum retry attempts + """ + self.model = model + self.api_key = api_key.strip() if api_key else None + self.endpoint = endpoint.rstrip('/') if endpoint else None + self.use_sdk = use_sdk + self.inference_mode = inference_mode + self.task_type = task_type or self.DEFAULT_TASK_TYPES.get(model, "search_document") + self.batch_size = batch_size + self.max_retries = max_retries + self.embedding_dimensions = self.MODEL_DIMENSIONS.get(model, 768) + + # Validate model + if model not in self.MODEL_DIMENSIONS: + logger.warning(f"Unknown NOMIC model: {model}. Using default dimensions (768)") + + # Validate API key for remote/API mode + if not use_sdk and not self.api_key and not self.endpoint: + # Try to get from environment + self.api_key = os.getenv("NOMIC_API_KEY") + if not self.api_key: + logger.warning("NOMIC API key not provided. Some operations may fail.") + + async def create_embedding(self, text: str) -> List[float]: + """Create embedding for a single text.""" + embeddings = await self.create_embeddings([text]) + return embeddings[0] if embeddings else [] + + async def create_embeddings(self, texts: List[str]) -> List[List[float]]: + """Create embeddings for a list of texts.""" + all_embeddings = [] + + # Process in batches + for i in range(0, len(texts), self.batch_size): + batch = texts[i:i + self.batch_size] + batch_embeddings = await self._create_batch_embeddings(batch) + all_embeddings.extend(batch_embeddings) + + return all_embeddings + + async def _create_batch_embeddings(self, texts: List[str]) -> List[List[float]]: + """Create embeddings for a batch of texts with retry logic.""" + if self.use_sdk: + return await self._create_batch_embeddings_sdk(texts) + else: + return await self._create_batch_embeddings_api(texts) + + async def _create_batch_embeddings_sdk(self, texts: List[str]) -> List[List[float]]: + """Create embeddings using NOMIC Python SDK.""" + try: + # Import NOMIC SDK (might not be installed) + from nomic import embed + + # Generate embeddings + output = embed.text( + texts=texts, + model=self.model, + task_type=self.task_type, + inference_mode=self.inference_mode, + ) + + embeddings = output.get('embeddings', []) + logger.info( + f"Computed NOMIC embeddings (SDK) in batch. Batch size: {len(texts)}, Model: {self.model}" + ) + return embeddings + + except ImportError: + logger.error("NOMIC SDK not installed. Install with: pip install nomic") + raise Exception("NOMIC SDK not available. Install with: pip install nomic") + except Exception as e: + logger.error(f"NOMIC SDK embedding error: {e}") + raise + + async def _create_batch_embeddings_api(self, texts: List[str]) -> List[List[float]]: + """Create embeddings using NOMIC API endpoint.""" + # Determine API endpoint + if self.endpoint: + api_url = f"{self.endpoint}/v1/embeddings" + else: + # Default NOMIC API endpoint + api_url = "https://api-atlas.nomic.ai/v1/embeddings" + + headers = { + 'Content-Type': 'application/json', + } + + if self.api_key: + headers['Authorization'] = f'Bearer {self.api_key}' + + payload = { + 'model': self.model, + 'texts': texts, + 'task_type': self.task_type, + } + + for attempt in range(self.max_retries): + try: + async with aiohttp.ClientSession() as session: + async with session.post( + api_url, + json=payload, + headers=headers, + timeout=aiohttp.ClientTimeout(total=120) # Longer timeout for embeddings + ) as response: + if response.status == 200: + result = await response.json() + logger.info( + f"Computed NOMIC embeddings (API) in batch. Batch size: {len(texts)}, Model: {self.model}" + ) + + # Handle different response formats + if 'embeddings' in result: + return result['embeddings'] + elif 'data' in result: + # OpenAI-compatible format + return [item['embedding'] for item in result['data']] + else: + logger.error(f"Unexpected NOMIC API response format: {result.keys()}") + raise Exception("Unexpected NOMIC API response format") + else: + error_text = await response.text() + logger.error(f"NOMIC API error {response.status}: {error_text}") + if attempt == self.max_retries - 1: + raise Exception(f"NOMIC API failed after {self.max_retries} attempts: {error_text}") + + except asyncio.TimeoutError: + logger.warning(f"NOMIC API timeout on attempt {attempt + 1}") + if attempt == self.max_retries - 1: + raise Exception("NOMIC service timeout") + + except Exception as e: + logger.error(f"NOMIC embedding error on attempt {attempt + 1}: {e}") + if attempt == self.max_retries - 1: + raise + + # Wait before retry (exponential backoff) + await asyncio.sleep(2 ** attempt) + + raise Exception("NOMIC embedding generation failed") + + def get_embedding_dimensions(self) -> int: + """Return the dimension size of embeddings.""" + return self.embedding_dimensions + + @property + def open_ai_dimensions(self) -> int: + """Compatibility property for OpenAIEmbeddings interface.""" + return self.embedding_dimensions + + +# Convenience functions for creating NOMIC embeddings instances +def create_nomic_text_embeddings( + api_key: Optional[str] = None, + endpoint: Optional[str] = None, + use_sdk: bool = False, +) -> NomicEmbeddings: + """Create NOMIC text embeddings instance.""" + return NomicEmbeddings( + model="nomic-embed-text-v1.5", + api_key=api_key, + endpoint=endpoint, + use_sdk=use_sdk, + ) + + +def create_nomic_code_embeddings( + api_key: Optional[str] = None, + endpoint: Optional[str] = None, + use_sdk: bool = False, +) -> NomicEmbeddings: + """Create NOMIC code embeddings instance.""" + return NomicEmbeddings( + model="nomic-embed-code-v1", + api_key=api_key, + endpoint=endpoint, + use_sdk=use_sdk, + ) + + +def create_nomic_vision_embeddings( + api_key: Optional[str] = None, + endpoint: Optional[str] = None, + use_sdk: bool = False, +) -> NomicEmbeddings: + """Create NOMIC vision (multimodal) embeddings instance.""" + return NomicEmbeddings( + model="nomic-embed-vision-v1.5", + api_key=api_key, + endpoint=endpoint, + use_sdk=use_sdk, + ) + + + + + diff --git a/app/backend/prepdocslib/page.py b/app/backend/prepdocslib/page.py index b87a81e88f..2bfa27bd44 100644 --- a/app/backend/prepdocslib/page.py +++ b/app/backend/prepdocslib/page.py @@ -12,6 +12,7 @@ class ImageOnPage: page_num: int # 0-indexed url: Optional[str] = None embedding: Optional[list[float]] = None + ocr_text: Optional[str] = None @dataclass diff --git a/app/backend/prepdocslib/patentsberta_embeddings.py b/app/backend/prepdocslib/patentsberta_embeddings.py new file mode 100644 index 0000000000..d7aa85bb79 --- /dev/null +++ b/app/backend/prepdocslib/patentsberta_embeddings.py @@ -0,0 +1,95 @@ +import aiohttp +import asyncio +from typing import List, Optional +import logging + +logger = logging.getLogger("scripts") + +class PatentsBertaEmbeddings: + """ + Class for using PatentsBERTa embeddings from a custom FastAPI service + Follows the same interface pattern as OpenAIEmbeddings for seamless integration + """ + + def __init__( + self, + endpoint: str, + api_key: Optional[str] = None, + batch_size: int = 16, + max_retries: int = 3 + ): + self.endpoint = endpoint.rstrip('/') + # Clean up API key (remove any trailing whitespace/newlines) + self.api_key = api_key.strip() if api_key else None + self.batch_size = batch_size + self.max_retries = max_retries + self.embedding_dimensions = 768 # PatentsBERTa dimension size + + async def create_embedding(self, text: str) -> List[float]: + """Create embedding for a single text using PatentsBERTa service""" + embeddings = await self.create_embeddings([text]) + return embeddings[0] if embeddings else [] + + async def create_embeddings(self, texts: List[str]) -> List[List[float]]: + """Create embeddings for a list of texts using PatentsBERTa service""" + all_embeddings = [] + + # Process in batches to avoid overwhelming the service + for i in range(0, len(texts), self.batch_size): + batch = texts[i:i + self.batch_size] + batch_embeddings = await self._create_batch_embeddings(batch) + all_embeddings.extend(batch_embeddings) + + return all_embeddings + + async def _create_batch_embeddings(self, texts: List[str]) -> List[List[float]]: + """Create embeddings for a batch of texts with retry logic""" + headers = {'Content-Type': 'application/json'} + if self.api_key: + headers['X-API-Key'] = self.api_key + + payload = { + 'texts': texts, + 'normalize': True + } + + for attempt in range(self.max_retries): + try: + async with aiohttp.ClientSession() as session: + async with session.post( + f"{self.endpoint}/embeddings", + json=payload, + headers=headers, + timeout=aiohttp.ClientTimeout(total=60) + ) as response: + if response.status == 200: + result = await response.json() + logger.info( + "Computed PatentsBERTa embeddings in batch. Batch size: %d", + len(texts) + ) + return result['embeddings'] + else: + error_text = await response.text() + logger.error(f"PatentsBERTa API error: {response.status} - {error_text}") + if attempt == self.max_retries - 1: + raise Exception(f"PatentsBERTa API failed after {self.max_retries} attempts") + + except asyncio.TimeoutError: + logger.warning(f"PatentsBERTa timeout on attempt {attempt + 1}") + if attempt == self.max_retries - 1: + raise Exception("PatentsBERTa service timeout") + + except Exception as e: + logger.error(f"PatentsBERTa embedding error on attempt {attempt + 1}: {e}") + if attempt == self.max_retries - 1: + raise + + # Wait before retry + await asyncio.sleep(2 ** attempt) + + raise Exception("PatentsBERTa embedding generation failed") + + def get_embedding_dimensions(self) -> int: + """Return the dimension size of embeddings""" + return self.embedding_dimensions diff --git a/app/backend/prepdocslib/searchmanager.py b/app/backend/prepdocslib/searchmanager.py index 7ab8018e5c..c87abf8833 100644 --- a/app/backend/prepdocslib/searchmanager.py +++ b/app/backend/prepdocslib/searchmanager.py @@ -77,7 +77,21 @@ def __init__( self.use_acls = use_acls self.use_int_vectorization = use_int_vectorization self.embeddings = embeddings - self.embedding_dimensions = self.embeddings.open_ai_dimensions if self.embeddings else None + # Handle different embedding service types + if self.embeddings: + if hasattr(self.embeddings, 'open_ai_dimensions'): + # OpenAI-based embeddings + self.embedding_dimensions = self.embeddings.open_ai_dimensions + elif hasattr(self.embeddings, 'get_embedding_dimensions'): + # PatentsBERTa or NOMIC embeddings + self.embedding_dimensions = self.embeddings.get_embedding_dimensions() + elif hasattr(self.embeddings, 'embedding_dimensions'): + # Direct dimension attribute (NOMIC) + self.embedding_dimensions = self.embeddings.embedding_dimensions + else: + self.embedding_dimensions = None + else: + self.embedding_dimensions = None self.field_name_embedding = field_name_embedding self.search_images = search_images @@ -118,9 +132,11 @@ async def create_index(self): name="hnsw_config", parameters=HnswParameters(metric="cosine"), ) + # Truncation dimension must be less than the embedding dimensions + truncation_dim = min(512, max(256, self.embedding_dimensions - 1)) text_vector_compression = BinaryQuantizationCompression( compression_name=f"{self.field_name_embedding}-compression", - truncation_dimension=1024, # should this be a parameter? maybe not yet? + truncation_dimension=truncation_dim, rescoring_options=RescoringOptions( enable_rescoring=True, default_oversampling=10, diff --git a/app/backend/requirements.in b/app/backend/requirements.in index 1110ef5546..9f8bfd7b6b 100644 --- a/app/backend/requirements.in +++ b/app/backend/requirements.in @@ -31,3 +31,10 @@ python-dotenv prompty rich typing-extensions +redis +# Microsoft 365 Agents SDK dependencies +microsoft-365-agents-sdk +botbuilder-core +botbuilder-schema +botbuilder-adapter-teams +botbuilder-adapter-azure diff --git a/app/backend/services/cache.py b/app/backend/services/cache.py new file mode 100644 index 0000000000..19e46098a5 --- /dev/null +++ b/app/backend/services/cache.py @@ -0,0 +1,196 @@ +""" +Cache service with Redis and in-memory support. + +Supports both Redis (for multi-instance deployments) and in-memory caching +(for local development or single-instance deployments). +""" + +import json +import logging +import time +from typing import Any, Dict, Optional, Protocol + +logger = logging.getLogger(__name__) + + +class CacheProtocol(Protocol): + """Protocol for cache implementations.""" + + async def get(self, key: str) -> Optional[Any]: + """Get value from cache.""" + ... + + async def set(self, key: str, val: Any, ttl_s: int) -> None: + """Set value in cache with TTL.""" + ... + + async def clear(self) -> None: + """Clear all cache entries.""" + ... + + async def close(self) -> None: + """Close cache connection.""" + ... + + +class InMemoryCache: + """In-memory cache implementation.""" + + def __init__(self) -> None: + self._store: Dict[str, Dict[str, Any]] = {} + + async def get(self, key: str) -> Optional[Any]: + """Get value from cache.""" + entry = self._store.get(key) + if not entry: + return None + if entry["exp"] < time.time(): + # expired + self._store.pop(key, None) + return None + return entry["val"] + + async def set(self, key: str, val: Any, ttl_s: int) -> None: + """Set value in cache with TTL.""" + self._store[key] = {"val": val, "exp": time.time() + ttl_s} + + async def clear(self) -> None: + """Clear all cache entries.""" + self._store.clear() + + async def close(self) -> None: + """Close cache (no-op for in-memory).""" + pass + + +class RedisCache: + """Redis cache implementation.""" + + def __init__(self, redis_url: str): + """ + Initialize Redis cache. + + Args: + redis_url: Redis connection URL (e.g., redis://localhost:6379/0) + """ + self.redis_url = redis_url + self._redis: Optional[Any] = None + self._connected = False + + async def _ensure_connected(self) -> None: + """Ensure Redis connection is established.""" + if self._connected and self._redis: + return + + try: + import redis.asyncio as redis + + # Parse Redis URL + # redis://[:password@]host[:port][/database] + self._redis = await redis.from_url( + self.redis_url, + encoding="utf-8", + decode_responses=False, # We'll handle JSON encoding/decoding + socket_connect_timeout=5, + socket_timeout=5, + retry_on_timeout=True, + ) + + # Test connection + await self._redis.ping() + self._connected = True + logger.info(f"Redis cache connected: {self.redis_url}") + except ImportError: + logger.error("redis library not installed. Install with: pip install redis") + raise + except Exception as e: + logger.warning(f"Failed to connect to Redis: {e}. Will fall back gracefully on operations.") + self._connected = False + self._redis = None + # Don't raise - allow graceful fallback + + async def get(self, key: str) -> Optional[Any]: + """Get value from Redis cache.""" + try: + await self._ensure_connected() + if not self._redis: + return None + + data = await self._redis.get(key) + if data is None: + return None + + # Deserialize JSON + return json.loads(data) + except Exception as e: + logger.warning(f"Redis get failed for key '{key}': {e}") + return None + + async def set(self, key: str, val: Any, ttl_s: int) -> None: + """Set value in Redis cache with TTL.""" + try: + await self._ensure_connected() + if not self._redis: + return + + # Serialize to JSON + data = json.dumps(val) + + # Set with TTL + await self._redis.setex(key, ttl_s, data) + except Exception as e: + logger.warning(f"Redis set failed for key '{key}': {e}") + # Don't raise - allow fallback to in-memory or no cache + + async def clear(self) -> None: + """Clear all cache entries (use with caution in production).""" + try: + await self._ensure_connected() + if self._redis: + await self._redis.flushdb() + except Exception as e: + logger.warning(f"Redis clear failed: {e}") + + async def close(self) -> None: + """Close Redis connection.""" + if self._redis: + try: + await self._redis.close() + self._connected = False + logger.info("Redis cache connection closed") + except Exception as e: + logger.warning(f"Error closing Redis connection: {e}") + + +async def create_cache(redis_url: Optional[str] = None) -> CacheProtocol: + """ + Create cache instance based on configuration. + + Args: + redis_url: Optional Redis URL. If provided and valid, returns RedisCache. + Otherwise returns InMemoryCache. + + Returns: + Cache instance (RedisCache or InMemoryCache) + """ + if redis_url: + try: + cache = RedisCache(redis_url) + # Test connection (don't raise on failure, just log) + try: + await cache._ensure_connected() + except Exception: + # Connection failed, but we'll still return RedisCache + # It will fall back gracefully on each operation + pass + + if cache._connected: + logger.info("Using Redis cache for multi-instance support") + return cache + else: + logger.warning("Redis connection failed, falling back to in-memory cache") + except Exception as e: + logger.warning(f"Redis initialization failed: {e}, falling back to in-memory cache") + + logger.info("Using in-memory cache (single-instance only)") + return InMemoryCache() diff --git a/app/backend/services/citation_builder.py b/app/backend/services/citation_builder.py new file mode 100644 index 0000000000..0024524664 --- /dev/null +++ b/app/backend/services/citation_builder.py @@ -0,0 +1,19 @@ +from typing import List, Dict, Any + + +def build_unified_from_text_sources(text_sources: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + unified: List[Dict[str, Any]] = [] + for doc in text_sources or []: + unified.append( + { + "source": "corpus", + "provider": "azure_search", + "url": doc.get("sourcepage", doc.get("sourcefile", "")), + "title": doc.get("title", doc.get("sourcefile", "Document")), + "snippet": doc.get("content", ""), + "metadata": doc, + } + ) + return unified + + diff --git a/app/backend/services/citation_filter.py b/app/backend/services/citation_filter.py new file mode 100644 index 0000000000..683cf551cd --- /dev/null +++ b/app/backend/services/citation_filter.py @@ -0,0 +1,94 @@ +""" +Citation Filter Service + +Filters citations to only include those actually used in the answer text. +""" + +import re +from typing import List, Set + + +def extract_citations_from_text(text: str) -> Set[str]: + """ + Extract citations from answer text. + + Citations can be in formats: + - [document.pdf#page=1] + - [https://example.com] + - [document.pdf#page=1(image.png)] + + Args: + text: The answer text from the LLM + + Returns: + Set of citation strings found in the text + """ + citations = set() + + # Pattern to match citations in square brackets + # Matches: [anything inside brackets] + citation_pattern = r'\[([^\]]+)\]' + + matches = re.findall(citation_pattern, text) + for match in matches: + # Remove any image filename in parentheses if present + # e.g., "doc.pdf#page=1(image.png)" -> "doc.pdf#page=1" + if '(' in match and ')' in match: + # Extract the part before the opening parenthesis + citation = match.split('(')[0].strip() + else: + citation = match.strip() + + if citation: # Only add non-empty citations + citations.add(citation) + + return citations + + +def filter_citations_by_answer( + all_citations: List[str], + answer_text: str +) -> List[str]: + """ + Filter citations to only include those actually used in the answer text. + + Args: + all_citations: List of all citations from retrieved documents + answer_text: The answer text from the LLM + + Returns: + Filtered list of citations that appear in the answer text + """ + if not answer_text or not all_citations: + return all_citations or [] + + # Extract citations from answer text + citations_in_answer = extract_citations_from_text(answer_text) + + if not citations_in_answer: + # If no citations found in answer, return all citations (fallback) + return all_citations + + # Filter citations to only those found in the answer + filtered = [] + for citation in all_citations: + # Check if this citation appears in the answer + # Handle both exact match and partial match (e.g., citation might be "doc.pdf#page=1" + # and answer might have "[doc.pdf#page=1]") + citation_normalized = citation.strip() + + # Check exact match + if citation_normalized in citations_in_answer: + filtered.append(citation) + else: + # Check if citation is a substring of any citation in answer + # or if any citation in answer is a substring of this citation + for answer_citation in citations_in_answer: + if (citation_normalized in answer_citation or + answer_citation in citation_normalized): + filtered.append(citation) + break + + # If no matches found, return all citations (fallback to avoid losing all citations) + return filtered if filtered else all_citations + diff --git a/app/backend/services/embedding_router.py b/app/backend/services/embedding_router.py new file mode 100644 index 0000000000..4f88a40c72 --- /dev/null +++ b/app/backend/services/embedding_router.py @@ -0,0 +1,448 @@ +""" +Embedding Model Router for selecting appropriate embedding models based on content characteristics. + +This router helps choose between: +- Baseline (Azure OpenAI) for general content +- PatentSBERTa for technical/patent-heavy content +- NOMIC for alternative embeddings + +Heuristics include: +- Technical keyword detection (patents, engineering, scientific terms) +- Code presence analysis +- Image density analysis (from metadata) +- Patent-specific indicators +- Metadata-based routing hints +""" + +import re +from typing import Dict, Any, Optional, Set +from enum import Enum +import logging + +logger = logging.getLogger(__name__) + + +class EmbeddingModel(str, Enum): + """Available embedding models.""" + BASELINE = "baseline" # Azure OpenAI + PATENTSBERTA = "patentsberta" # Domain-specific + NOMIC = "nomic" # Alternative provider + + +class EmbeddingRouter: + """ + Router for selecting embedding models based on content analysis. + + Uses heuristics to determine if content is technical/patent-heavy and would benefit + from PatentSBERTa embeddings, which are specialized for technical and patent content. + """ + + # Technical keywords that indicate patent/technical content + PATENT_KEYWORDS: Set[str] = { + "patent", "patents", "patented", "patentee", "patentability", + "invention", "inventor", "inventive", "prior art", "novelty", + "claim", "claims", "embodiment", "embodiments", "specification", + "application", "filing", "disclosure", "provisional", "non-provisional", + "uspto", "european patent", "pct", "international application" + } + + # Engineering and technical terms + TECHNICAL_KEYWORDS: Set[str] = { + "algorithm", "circuit", "circuitry", "component", "assembly", + "mechanism", "apparatus", "device", "system", "method", + "process", "technique", "implementation", "configuration", + "module", "interface", "protocol", "architecture", "framework", + "semiconductor", "microprocessor", "controller", "processor", + "sensor", "actuator", "transducer", "transmitter", "receiver", + "synthesis", "analysis", "optimization", "calibration", "validation" + } + + # Scientific and research terms + SCIENTIFIC_KEYWORDS: Set[str] = { + "hypothesis", "experiment", "experimental", "empirical", + "theoretical", "methodology", "research", "study", "analysis", + "synthesis", "compound", "molecule", "reaction", "catalyst", + "polymer", "crystal", "lattice", "quantum", "nanotechnology" + } + + # Code-related patterns + CODE_PATTERNS: Set[str] = { + "def ", "function", "class ", "import ", "from ", "return ", + "if __name__", "public ", "private ", "static ", "void ", "int ", + "const ", "let ", "var ", "async ", "await ", "=>", "->", + "namespace", "using ", "#include", "package ", "interface ", + "extends", "implements", "try", "catch", "finally", "throw" + } + + # Code file extensions + CODE_FILE_EXTENSIONS: Set[str] = { + ".py", ".js", ".java", ".go", ".php", ".rb", ".cpp", ".c", + ".ts", ".tsx", ".jsx", ".sql", ".sh", ".yaml", ".yml", ".json", + ".md", ".xml", ".html", ".css", ".scss", ".less", ".vue", ".svelte" + } + + def __init__( + self, + baseline_deployment: str, + patentsberta_endpoint: Optional[str] = None, + nomic_endpoint: Optional[str] = None, + nomic_api_key: Optional[str] = None, + enable_heuristics: bool = True + ): + """ + Initialize embedding router. + + Args: + baseline_deployment: Azure OpenAI embedding deployment name + patentsberta_endpoint: Optional PatentSBERTa service endpoint + nomic_endpoint: Optional NOMIC service endpoint (for code/multimodal) + nomic_api_key: Optional NOMIC API key + enable_heuristics: Enable intelligent routing (default: True) + """ + self.baseline_deployment = baseline_deployment + self.patentsberta_endpoint = patentsberta_endpoint + self.nomic_endpoint = nomic_endpoint + self.nomic_api_key = nomic_api_key + self.enable_heuristics = enable_heuristics + + # Pre-compile regex patterns for performance + self._patent_pattern = re.compile( + r'\b(' + '|'.join(re.escape(kw) for kw in self.PATENT_KEYWORDS) + r')\b', + re.IGNORECASE + ) + self._technical_pattern = re.compile( + r'\b(' + '|'.join(re.escape(kw) for kw in self.TECHNICAL_KEYWORDS) + r')\b', + re.IGNORECASE + ) + self._scientific_pattern = re.compile( + r'\b(' + '|'.join(re.escape(kw) for kw in self.SCIENTIFIC_KEYWORDS) + r')\b', + re.IGNORECASE + ) + # Code pattern detection (multiple patterns) + self._code_patterns = [re.compile(re.escape(pattern), re.IGNORECASE) for pattern in self.CODE_PATTERNS] + + def _analyze_content(self, content: str, content_type: Optional[str] = None) -> Dict[str, Any]: + """ + Analyze content for technical/patent/code indicators. + + Args: + content: Document content to analyze + content_type: Optional content type hint (file extension, etc.) + + Returns: + Dictionary with analysis results + """ + if not content or len(content.strip()) == 0: + return { + "patent_score": 0.0, + "technical_score": 0.0, + "scientific_score": 0.0, + "code_score": 0.0, + "total_score": 0.0, + "is_code_file": False + } + + content_lower = content.lower() + content_length = len(content) + word_count = len(content.split()) + + # Check if file extension indicates code + is_code_file = False + if content_type: + content_type_lower = content_type.lower().strip() + if content_type_lower.startswith('.'): + is_code_file = content_type_lower in self.CODE_FILE_EXTENSIONS + elif any(content_type_lower.endswith(ext) for ext in self.CODE_FILE_EXTENSIONS): + is_code_file = True + + # Count keyword matches + patent_matches = len(self._patent_pattern.findall(content_lower)) + technical_matches = len(self._technical_pattern.findall(content_lower)) + scientific_matches = len(self._scientific_pattern.findall(content_lower)) + + # Detect code presence (more sophisticated detection) + code_matches = sum(1 for pattern in self._code_patterns if pattern.search(content)) + + # Additional code indicators: brackets, semicolons, etc. + code_indicators = sum([ + content.count('{'), + content.count('}'), + content.count(';'), + content.count('()'), + content.count('=>'), + content.count('->'), + ]) + + # Code score combines pattern matches and structural indicators + code_match_score = (code_matches / max(word_count, 1)) * 100 if word_count > 0 else 0.0 + code_structure_score = min((code_indicators / max(content_length, 1)) * 1000, 50.0) # Cap at 50 + code_score = code_match_score * 2.0 + code_structure_score * 0.5 + + # Boost code score if file extension indicates code + if is_code_file: + code_score = max(code_score, 25.0) # Minimum score for code files + + # Calculate scores (normalized by word count to avoid bias toward long documents) + patent_score = (patent_matches / max(word_count, 1)) * 100 if word_count > 0 else 0.0 + technical_score = (technical_matches / max(word_count, 1)) * 100 if word_count > 0 else 0.0 + scientific_score = (scientific_matches / max(word_count, 1)) * 100 if word_count > 0 else 0.0 + + # Weighted total score + # Patent keywords are strongest indicator, then technical, then scientific, then code + total_score = ( + patent_score * 3.0 + # Patent keywords are most important + technical_score * 2.0 + + scientific_score * 1.5 + + code_score * 1.0 + ) + + return { + "patent_score": patent_score, + "technical_score": technical_score, + "scientific_score": scientific_score, + "code_score": code_score, + "total_score": total_score, + "patent_matches": patent_matches, + "technical_matches": technical_matches, + "scientific_matches": scientific_matches, + "code_matches": code_matches, + "code_indicators": code_indicators, + "is_code_file": is_code_file, + "word_count": word_count + } + + def _analyze_metadata(self, metadata: Optional[Dict[str, Any]]) -> Dict[str, Any]: + """ + Extract routing hints from metadata. + + Args: + metadata: Optional metadata dictionary + + Returns: + Dictionary with metadata analysis + """ + if not metadata: + return { + "has_routing_hint": False, + "image_density": 0.0, + "suggested_model": None + } + + # Check for explicit routing hint + routing_hint = metadata.get("embedding_model") or metadata.get("preferred_embedding") + suggested_model = None + if routing_hint: + try: + suggested_model = EmbeddingModel(routing_hint.lower()) + except ValueError: + logger.warning(f"Invalid routing hint in metadata: {routing_hint}") + + # Analyze image density + image_count = metadata.get("image_count", 0) + page_count = metadata.get("page_count", 1) + image_density = (image_count / max(page_count, 1)) * 100 if page_count > 0 else 0.0 + + # Check for technical indicators in metadata + category = metadata.get("category", "").lower() + file_type = metadata.get("file_type", "").lower() + source_file = metadata.get("sourcefile", "").lower() + + # File name or category hints + is_technical_file = any( + keyword in category or keyword in source_file + for keyword in ["patent", "technical", "engineering", "research", "scientific"] + ) + + return { + "has_routing_hint": routing_hint is not None, + "routing_hint": routing_hint, + "suggested_model": suggested_model, + "image_density": image_density, + "image_count": image_count, + "page_count": page_count, + "is_technical_file": is_technical_file, + "category": category, + "file_type": file_type + } + + def select_model( + self, + content: str, + content_type: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None + ) -> EmbeddingModel: + """ + Select appropriate embedding model based on content characteristics. + + Routing logic: + 1. If metadata has explicit routing hint, use it + 2. If PatentSBERTa endpoint not configured, use baseline + 3. Analyze content for technical/patent indicators + 4. Route to PatentSBERTa if score exceeds threshold + 5. Otherwise use baseline + + Args: + content: Document content to analyze + content_type: Optional content type hint (pdf, txt, etc.) + metadata: Optional metadata (image_count, code_density, etc.) + + Returns: + Selected embedding model + """ + # If heuristics disabled, always use baseline + if not self.enable_heuristics: + return EmbeddingModel.BASELINE + + # If PatentSBERTa endpoint not configured, must use baseline + if not self.patentsberta_endpoint: + logger.debug("PatentSBERTa endpoint not configured, using baseline") + return EmbeddingModel.BASELINE + + # Analyze metadata first (explicit hints take priority) + metadata_analysis = self._analyze_metadata(metadata) + + # Check for explicit routing hint in metadata + if metadata_analysis["has_routing_hint"] and metadata_analysis["suggested_model"]: + suggested = metadata_analysis["suggested_model"] + logger.info(f"Using metadata routing hint: {suggested}") + return suggested + + # Analyze content + content_analysis = self._analyze_content(content, content_type) + + # Routing thresholds + PATENT_THRESHOLD = 5.0 # Score threshold for routing to PatentSBERTa + STRONG_PATENT_THRESHOLD = 10.0 # Strong indicator threshold + CODE_THRESHOLD = 15.0 # Score threshold for routing to NOMIC Code + MULTIMODAL_THRESHOLD = 15.0 # Image density threshold for NOMIC Vision + + # Decision logic + total_score = content_analysis["total_score"] + patent_score = content_analysis["patent_score"] + code_score = content_analysis["code_score"] + is_code_file = content_analysis.get("is_code_file", False) + + # Check for code-heavy content → NOMIC Embed Code + if (self.nomic_endpoint or self.nomic_api_key) and (code_score >= CODE_THRESHOLD or is_code_file): + logger.info( + f"Routing to NOMIC Code: code_score={code_score:.2f}, " + f"is_code_file={is_code_file}" + ) + return EmbeddingModel.NOMIC + + # Check for high image density → NOMIC Vision (if enabled) + if (self.nomic_endpoint or self.nomic_api_key) and metadata_analysis["image_density"] >= MULTIMODAL_THRESHOLD: + logger.info( + f"Routing to NOMIC Vision: image_density={metadata_analysis['image_density']:.2f}" + ) + return EmbeddingModel.NOMIC + + # Strong patent indicators → PatentSBERTa + if patent_score >= 2.0 or total_score >= STRONG_PATENT_THRESHOLD: + logger.info( + f"Routing to PatentSBERTa: patent_score={patent_score:.2f}, " + f"total_score={total_score:.2f}" + ) + return EmbeddingModel.PATENTSBERTA + + # Moderate technical indicators → PatentSBERTa + if total_score >= PATENT_THRESHOLD: + # Also check if metadata suggests technical content + if metadata_analysis["is_technical_file"]: + logger.info( + f"Routing to PatentSBERTa: total_score={total_score:.2f}, " + f"technical_file=True" + ) + return EmbeddingModel.PATENTSBERTA + + # High image density with technical content might benefit from PatentSBERTa + if metadata_analysis["image_density"] > 10.0 and total_score >= 3.0: + logger.info( + f"Routing to PatentSBERTa: image_density={metadata_analysis['image_density']:.2f}, " + f"total_score={total_score:.2f}" + ) + return EmbeddingModel.PATENTSBERTA + + # Default to baseline + logger.debug( + f"Routing to baseline: total_score={total_score:.2f}, " + f"patent_score={patent_score:.2f}, code_score={code_score:.2f}" + ) + return EmbeddingModel.BASELINE + + def get_deployment_name(self, model: EmbeddingModel) -> str: + """Get deployment name for selected model.""" + if model == EmbeddingModel.BASELINE: + return self.baseline_deployment + elif model == EmbeddingModel.PATENTSBERTA: + if self.patentsberta_endpoint: + return self.patentsberta_endpoint + else: + logger.warning("PatentSBERTa selected but endpoint not configured, falling back to baseline") + return self.baseline_deployment + elif model == EmbeddingModel.NOMIC: + if self.nomic_endpoint: + return self.nomic_endpoint + elif self.nomic_api_key: + # Using API key, return model identifier + return "nomic-embed-code-v1" # Default to code model for now + else: + logger.warning("NOMIC selected but endpoint/API key not configured, falling back to baseline") + return self.baseline_deployment + else: + return self.baseline_deployment # Fallback to baseline + + def get_nomic_model_type(self, content_analysis: Dict[str, Any], metadata_analysis: Dict[str, Any]) -> str: + """Determine which NOMIC model to use based on content analysis.""" + code_score = content_analysis.get("code_score", 0.0) + is_code_file = content_analysis.get("is_code_file", False) + image_density = metadata_analysis.get("image_density", 0.0) + + # Prioritize code detection + if code_score >= 15.0 or is_code_file: + return "nomic-embed-code-v1" + + # Then multimodal + if image_density >= 15.0: + return "nomic-embed-vision-v1.5" + + # Default to text + return "nomic-embed-text-v1.5" + + def get_routing_decision_info( + self, + content: str, + content_type: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None + ) -> Dict[str, Any]: + """ + Get detailed routing decision information for debugging/monitoring. + + Args: + content: Document content to analyze + content_type: Optional content type hint + metadata: Optional metadata + + Returns: + Dictionary with routing decision details + """ + selected_model = self.select_model(content, content_type, metadata) + content_analysis = self._analyze_content(content, content_type) + metadata_analysis = self._analyze_metadata(metadata) + + routing_info = { + "selected_model": selected_model.value, + "deployment_name": self.get_deployment_name(selected_model), + "content_analysis": content_analysis, + "metadata_analysis": metadata_analysis, + "heuristics_enabled": self.enable_heuristics, + "patentsberta_configured": self.patentsberta_endpoint is not None, + "nomic_configured": self.nomic_endpoint is not None or self.nomic_api_key is not None + } + + # Add NOMIC model type if NOMIC is selected + if selected_model == EmbeddingModel.NOMIC: + routing_info["nomic_model_type"] = self.get_nomic_model_type(content_analysis, metadata_analysis) + + return routing_info + diff --git a/app/backend/services/keyvault_secrets.py b/app/backend/services/keyvault_secrets.py new file mode 100644 index 0000000000..21a0be9e6b --- /dev/null +++ b/app/backend/services/keyvault_secrets.py @@ -0,0 +1,165 @@ +""" +Azure Key Vault Secrets Reader. + +Provides utilities for reading secrets from Azure Key Vault using Managed Identity. +Falls back to environment variables if Key Vault is not configured or unavailable. +""" + +import logging +import os +from typing import Optional, Dict, Any +from azure.identity.aio import ManagedIdentityCredential, DefaultAzureCredential +from azure.keyvault.secrets.aio import SecretClient +from azure.core.exceptions import AzureError + +logger = logging.getLogger(__name__) + + +class KeyVaultSecretReader: + """ + Reads secrets from Azure Key Vault with fallback to environment variables. + + Supports: + - Managed Identity authentication (when running on Azure) + - DefaultAzureCredential (for local development) + - Environment variable fallback + """ + + def __init__( + self, + key_vault_url: Optional[str] = None, + credential: Optional[Any] = None, + enable_keyvault: bool = True + ): + """ + Initialize Key Vault secret reader. + + Args: + key_vault_url: Key Vault URL (e.g., https://myvault.vault.azure.net/) + credential: Azure credential (ManagedIdentityCredential, DefaultAzureCredential, etc.) + enable_keyvault: Whether to attempt Key Vault reads (default: True) + """ + self.key_vault_url = key_vault_url or os.getenv("AZURE_KEY_VAULT_ENDPOINT") + self.credential = credential + self.enable_keyvault = enable_keyvault and self.key_vault_url is not None + self._client: Optional[SecretClient] = None + self._cache: Dict[str, Any] = {} + + async def _get_client(self) -> Optional[SecretClient]: + """Get or create Key Vault client.""" + if not self.enable_keyvault: + return None + + if self._client is None: + try: + if self.credential is None: + # Try Managed Identity first (for Azure), then DefaultAzureCredential + try: + self.credential = ManagedIdentityCredential() + except Exception: + self.credential = DefaultAzureCredential() + + self._client = SecretClient( + vault_url=self.key_vault_url, + credential=self.credential + ) + logger.info(f"Key Vault client initialized for: {self.key_vault_url}") + except Exception as e: + logger.warning(f"Failed to initialize Key Vault client: {e}. Will use environment variables.") + self.enable_keyvault = False + return None + + return self._client + + async def get_secret( + self, + secret_name: str, + env_var_name: Optional[str] = None, + use_cache: bool = True + ) -> Optional[str]: + """ + Get secret from Key Vault or environment variable. + + Priority: + 1. Key Vault (if enabled and available) + 2. Environment variable (if env_var_name provided) + 3. None + + Args: + secret_name: Name of the secret in Key Vault + env_var_name: Optional environment variable name (if different from secret_name) + use_cache: Whether to cache the secret value (default: True) + + Returns: + Secret value or None if not found + """ + # Check cache first + cache_key = secret_name + if use_cache and cache_key in self._cache: + return self._cache[cache_key] + + # Try Key Vault first + if self.enable_keyvault: + try: + client = await self._get_client() + if client: + secret = await client.get_secret(secret_name) + value = secret.value + + # Cache the value + if use_cache: + self._cache[cache_key] = value + + logger.debug(f"Retrieved secret '{secret_name}' from Key Vault") + return value + except AzureError as e: + logger.warning(f"Failed to get secret '{secret_name}' from Key Vault: {e}. Falling back to environment variable.") + except Exception as e: + logger.warning(f"Unexpected error getting secret '{secret_name}' from Key Vault: {e}. Falling back to environment variable.") + + # Fallback to environment variable + env_name = env_var_name or secret_name + value = os.getenv(env_name) + + if value: + logger.debug(f"Retrieved secret '{secret_name}' from environment variable '{env_name}'") + if use_cache: + self._cache[cache_key] = value + return value + + logger.debug(f"Secret '{secret_name}' not found in Key Vault or environment variable '{env_name}'") + return None + + async def get_secrets( + self, + secret_mappings: Dict[str, Optional[str]] + ) -> Dict[str, Optional[str]]: + """ + Get multiple secrets from Key Vault or environment variables. + + Args: + secret_mappings: Dictionary mapping secret names to optional env var names + e.g., {"MICROSOFT_APP_PASSWORD": "MICROSOFT_APP_PASSWORD", + "AZURE_SEARCH_KEY": "AZURE_SEARCH_KEY"} + + Returns: + Dictionary with secret values (None if not found) + """ + results = {} + for secret_name, env_var_name in secret_mappings.items(): + results[secret_name] = await self.get_secret(secret_name, env_var_name) + return results + + async def close(self): + """Close the Key Vault client and clean up resources.""" + if self._client: + await self._client.close() + self._client = None + if self.credential and hasattr(self.credential, 'close'): + await self.credential.close() + self._cache.clear() + + + + + diff --git a/app/backend/services/ocr/__init__.py b/app/backend/services/ocr/__init__.py new file mode 100644 index 0000000000..3ecbfff101 --- /dev/null +++ b/app/backend/services/ocr/__init__.py @@ -0,0 +1,23 @@ +""" +OCR Service Module. + +Provides OCR functionality with support for multiple providers: +- Ollama (local vision models) +- Azure Document Intelligence +""" + +from .base import OCRProvider, OCRResult +from .ollama_client import OllamaOCRClient +from .azure_document_intelligence_client import AzureDocumentIntelligenceOCRClient + +__all__ = [ + "OCRProvider", + "OCRResult", + "OllamaOCRClient", + "AzureDocumentIntelligenceOCRClient" +] + + + + + diff --git a/app/backend/services/ocr/azure_document_intelligence_client.py b/app/backend/services/ocr/azure_document_intelligence_client.py new file mode 100644 index 0000000000..9bf2049a68 --- /dev/null +++ b/app/backend/services/ocr/azure_document_intelligence_client.py @@ -0,0 +1,153 @@ +""" +Azure Document Intelligence OCR Client. + +Integration with Azure AI Document Intelligence for OCR. +""" + +import io +from typing import Dict, Any, Optional +import logging +from azure.ai.documentintelligence.aio import DocumentIntelligenceClient +from azure.ai.documentintelligence.models import AnalyzeDocumentRequest +from azure.core.credentials import AzureKeyCredential +from azure.core.credentials_async import AsyncTokenCredential +from azure.core.exceptions import HttpResponseError + +from .base import OCRResult, OCRProvider + +logger = logging.getLogger(__name__) + + +class AzureDocumentIntelligenceOCRClient(OCRProvider): + """ + Azure Document Intelligence client for OCR. + + Uses Azure AI Document Intelligence (formerly Form Recognizer) for text extraction. + """ + + def __init__( + self, + endpoint: str, + credential: AsyncTokenCredential | AzureKeyCredential, + model_id: str = "prebuilt-read", + api_version: str = "2024-02-29-preview" + ): + """ + Initialize Azure Document Intelligence OCR client. + + Args: + endpoint: Azure Document Intelligence endpoint + credential: Azure credential (ManagedIdentityCredential or AzureKeyCredential) + model_id: Model ID to use (default: prebuilt-read for OCR) + api_version: API version to use + """ + self.endpoint = endpoint.rstrip('/') + self.credential = credential + self.model_id = model_id + self.api_version = api_version + self._client: Optional[DocumentIntelligenceClient] = None + + async def _get_client(self) -> DocumentIntelligenceClient: + """Get or create Document Intelligence client.""" + if self._client is None: + self._client = DocumentIntelligenceClient( + endpoint=self.endpoint, + credential=self.credential + ) + return self._client + + async def extract_text( + self, + image_data: bytes, + language: Optional[str] = None, + options: Optional[Dict[str, Any]] = None + ) -> OCRResult: + """ + Extract text from image using Azure Document Intelligence. + + Args: + image_data: Image bytes (PNG, JPEG, PDF, etc.) + language: Optional language hint (Azure DI supports auto-detection) + options: Optional provider-specific options + + Returns: + OCRResult with extracted text and metadata + """ + try: + client = await self._get_client() + + # Prepare analyze request + analyze_request = AnalyzeDocumentRequest(bytes_source=image_data) + + # Add language hint if provided + if language: + analyze_request.locale = language + + # Start analysis + poller = await client.begin_analyze_document( + model_id=self.model_id, + analyze_request=analyze_request, + output_content_format="markdown" # Get markdown format for better structure + ) + + # Wait for result + result = await poller.result() + + # Extract text from all pages + all_text = result.content or "" + pages = [] + + if result.pages: + for page in result.pages: + page_text = "" + # Extract text from page + # Note: Azure DI returns structured content, we extract from content field + if result.content: + # For now, use full content (pages are separated in markdown) + # In production, you might want to parse page-by-page + page_text = result.content + + pages.append({ + "page_number": page.page_number, + "text": page_text, + "width": page.width, + "height": page.height, + "unit": page.unit + }) + + # Calculate confidence (Azure DI doesn't provide overall confidence) + # Use presence of content as a proxy + confidence = 1.0 if all_text else 0.0 + + metadata = { + "provider": "azure_document_intelligence", + "model_id": self.model_id, + "language": language or "auto", + "api_version": self.api_version, + "page_count": len(pages) if pages else 1 + } + + return OCRResult( + text=all_text, + confidence=confidence, + pages=pages, + metadata=metadata + ) + + except HttpResponseError as e: + logger.error(f"Azure Document Intelligence error: {e}") + raise Exception(f"Azure Document Intelligence error: {str(e)}") + except Exception as e: + logger.error(f"Error extracting text with Azure Document Intelligence: {e}") + raise + + async def close(self): + """Close the client and clean up resources.""" + if self._client: + await self._client.close() + self._client = None + + + + + diff --git a/app/backend/services/ocr/base.py b/app/backend/services/ocr/base.py new file mode 100644 index 0000000000..2fae8c8c27 --- /dev/null +++ b/app/backend/services/ocr/base.py @@ -0,0 +1,61 @@ +""" +Base OCR Provider Protocol. + +Defines the interface for OCR providers (Ollama, Azure Document Intelligence, etc.). +""" + +from typing import Protocol, Dict, Any, Optional +from io import BytesIO + + +class OCRResult: + """Standardized OCR result structure.""" + + def __init__( + self, + text: str, + confidence: float = 1.0, + pages: Optional[list[Dict[str, Any]]] = None, + metadata: Optional[Dict[str, Any]] = None + ): + """ + Initialize OCR result. + + Args: + text: Extracted text content + confidence: Overall confidence score (0.0 to 1.0) + pages: List of page-level results with page numbers and text + metadata: Additional metadata (provider, language, etc.) + """ + self.text = text + self.confidence = confidence + self.pages = pages or [] + self.metadata = metadata or {} + + +class OCRProvider(Protocol): + """Protocol for OCR providers.""" + + async def extract_text( + self, + image_data: bytes, + language: Optional[str] = None, + options: Optional[Dict[str, Any]] = None + ) -> OCRResult: + """ + Extract text from image using OCR. + + Args: + image_data: Image bytes (PNG, JPEG, PDF, etc.) + language: Optional language hint (e.g., 'en', 'zh', 'ja') + options: Optional provider-specific options + + Returns: + OCRResult with extracted text and metadata + """ + ... + + + + + diff --git a/app/backend/services/ocr/ollama_client.py b/app/backend/services/ocr/ollama_client.py new file mode 100644 index 0000000000..baaaee9644 --- /dev/null +++ b/app/backend/services/ocr/ollama_client.py @@ -0,0 +1,181 @@ +""" +Ollama OCR Client. + +Integration with local Ollama server for text extraction from images. +Ollama provides OpenAI-compatible API for vision models. +""" + +import aiohttp +import base64 +from typing import Dict, Any, Optional +import logging + +from .base import OCRResult, OCRProvider + +logger = logging.getLogger(__name__) + + +class OllamaOCRClient(OCRProvider): + """ + Ollama OCR client for text extraction from images using local Ollama server. + + Supports any Ollama vision model (e.g., llava, bakllava, deepseek-v2). + """ + + def __init__( + self, + base_url: str = "http://localhost:11434/v1", + model: str = "llava", + timeout: int = 120 + ): + """ + Initialize Ollama OCR client. + + Args: + base_url: Base URL for Ollama API (default: http://localhost:11434/v1) + model: Model name for OCR (default: llava) + timeout: Request timeout in seconds (default: 120 for local processing) + """ + self.base_url = base_url.rstrip('/') + self.model = model + self.timeout = timeout + + async def extract_text( + self, + image_data: bytes, + language: Optional[str] = None, + options: Optional[Dict[str, Any]] = None + ) -> OCRResult: + """ + Extract text from image using Ollama vision model. + + Args: + image_data: Image bytes (PNG, JPEG, PDF, etc.) + language: Optional language hint (e.g., 'en', 'zh', 'ja') + options: Optional provider-specific options + + Returns: + OCRResult with extracted text and metadata + """ + try: + # Encode image to base64 + image_base64 = base64.b64encode(image_data).decode('utf-8') + + # Prepare request payload (OpenAI-compatible format) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{image_base64}" + } + }, + { + "type": "text", + "text": "Extract all text from this image. Return only the extracted text, preserving formatting and structure." + } + ] + } + ], + "stream": False, + "max_tokens": 4096 + } + + # Add language hint if provided + if language: + payload["messages"][0]["content"][1]["text"] += f" (Language: {language})" + + # Add custom options if provided + if options: + if "temperature" in options: + payload["temperature"] = options["temperature"] + if "max_tokens" in options: + payload["max_tokens"] = options["max_tokens"] + + # Make API request (Ollama uses OpenAI-compatible API, no auth needed) + headers = { + "Content-Type": "application/json" + } + + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=self.timeout)) as session: + async with session.post( + f"{self.base_url}/chat/completions", + json=payload, + headers=headers + ) as response: + if response.status != 200: + error_text = await response.text() + logger.error(f"Ollama OCR API error {response.status}: {error_text}") + raise Exception(f"Ollama OCR API error {response.status}: {error_text}") + + data = await response.json() + + # Extract text from response + if "choices" in data and len(data["choices"]) > 0: + extracted_text = data["choices"][0]["message"]["content"] + + # Extract confidence if available + confidence = 1.0 + if "usage" in data: + # Use token counts as a proxy for confidence + total_tokens = data["usage"].get("total_tokens", 0) + if total_tokens > 0: + # Normalize confidence (heuristic) + confidence = min(1.0, total_tokens / 1000.0) + + metadata = { + "provider": "ollama", + "model": self.model, + "language": language, + "base_url": self.base_url, + "usage": data.get("usage", {}) + } + + return OCRResult( + text=extracted_text, + confidence=confidence, + pages=[], # Ollama doesn't provide page-level results + metadata=metadata + ) + else: + raise Exception("No text extracted from Ollama OCR response") + + except aiohttp.ClientError as e: + logger.error(f"Ollama OCR network error: {e}") + raise Exception(f"Ollama OCR network error: {str(e)}") + except Exception as e: + logger.error(f"Ollama OCR error: {e}") + raise + + async def extract_text_from_url( + self, + image_url: str, + language: Optional[str] = None, + options: Optional[Dict[str, Any]] = None + ) -> OCRResult: + """ + Extract text from image URL using Ollama OCR. + + Args: + image_url: URL of the image to process + language: Optional language hint + options: Optional provider-specific options + + Returns: + OCRResult with extracted text and metadata + """ + try: + async with aiohttp.ClientSession() as session: + async with session.get(image_url) as response: + if response.status != 200: + raise Exception(f"Failed to fetch image from URL: {response.status}") + image_data = await response.read() + return await self.extract_text(image_data, language, options) + except Exception as e: + logger.error(f"Error fetching image from URL: {e}") + raise + diff --git a/app/backend/services/ocr_service.py b/app/backend/services/ocr_service.py new file mode 100644 index 0000000000..4791676ce8 --- /dev/null +++ b/app/backend/services/ocr_service.py @@ -0,0 +1,247 @@ +""" +OCR Service. + +Main service for OCR operations with provider abstraction. +Supports multiple OCR providers (Ollama, Azure Document Intelligence, etc.). +""" + +import logging +from typing import Optional, Dict, Any +from enum import Enum +import os + +from services.ocr.base import OCRResult, OCRProvider +from services.ocr.ollama_client import OllamaOCRClient +from services.ocr.azure_document_intelligence_client import AzureDocumentIntelligenceOCRClient +from azure.core.credentials import AzureKeyCredential +from azure.core.credentials_async import AsyncTokenCredential +from config import OCR_PROVIDER, OCR_ON_INGEST + +logger = logging.getLogger(__name__) + + +class OCRProviderType(str, Enum): + """Supported OCR providers.""" + OLLAMA = "ollama" + AZURE_DOCUMENT_INTELLIGENCE = "azure_document_intelligence" + NONE = "none" # Disabled + + +class OCRService: + """ + OCR Service with provider abstraction. + + Supports multiple OCR providers and can switch between them. + """ + + def __init__( + self, + provider: Optional[OCRProviderType] = None, + provider_client: Optional[OCRProvider] = None, + enable_on_ingest: Optional[bool] = None + ): + """ + Initialize OCR service. + + Args: + provider: OCR provider type (defaults to OCR_PROVIDER env var) + provider_client: Pre-initialized OCR provider client (optional) + enable_on_ingest: Whether to run OCR during document ingestion (defaults to OCR_ON_INGEST env var) + """ + # Use environment variables if not provided + if provider is None: + provider_str = OCR_PROVIDER + try: + provider = OCRProviderType(provider_str) + except ValueError: + logger.warning(f"Invalid OCR_PROVIDER: {provider_str}, defaulting to NONE") + provider = OCRProviderType.NONE + + if enable_on_ingest is None: + enable_on_ingest = OCR_ON_INGEST + + self.provider = provider + self.provider_client = provider_client + self.enable_on_ingest = enable_on_ingest + + if provider_client is None and provider != OCRProviderType.NONE: + self.provider_client = self._create_provider_client(provider) + + def _create_provider_client(self, provider: OCRProviderType) -> Optional[OCRProvider]: + """ + Create OCR provider client based on provider type. + + Args: + provider: Provider type to create + + Returns: + OCRProvider instance or None if disabled + """ + if provider == OCRProviderType.NONE: + return None + + elif provider == OCRProviderType.OLLAMA: + base_url = os.getenv("OLLAMA_BASE_URL", "http://localhost:11434/v1") + model = os.getenv("OLLAMA_OCR_MODEL", "llava") + timeout = int(os.getenv("OLLAMA_TIMEOUT", "120")) + + logger.info(f"Initializing Ollama OCR client: {base_url}, model: {model}") + return OllamaOCRClient( + base_url=base_url, + model=model, + timeout=timeout + ) + + elif provider == OCRProviderType.AZURE_DOCUMENT_INTELLIGENCE: + endpoint = os.getenv("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT") + if not endpoint: + logger.warning("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT not set, OCR service disabled") + return None + + # Try to get credential + # For now, use key-based auth (can be enhanced with Managed Identity) + key = os.getenv("AZURE_DOCUMENT_INTELLIGENCE_KEY") + if key: + credential = AzureKeyCredential(key) + else: + # Try to use Managed Identity (requires async credential) + # This would need to be passed in from the app setup + logger.warning("AZURE_DOCUMENT_INTELLIGENCE_KEY not set, cannot create client") + return None + + model_id = os.getenv("AZURE_DOCUMENT_INTELLIGENCE_MODEL", "prebuilt-read") + + # Note: Azure Document Intelligence client is async, but we're storing it + # The actual usage will need to handle async context + # For now, return None and handle in async context + logger.warning("Azure Document Intelligence client requires async context, use create_async_client()") + return None + + else: + logger.warning(f"Unknown OCR provider: {provider}") + return None + + async def create_async_client( + self, + provider: OCRProviderType, + azure_credential: Optional[AsyncTokenCredential] = None + ) -> Optional[OCRProvider]: + """ + Create async OCR provider client (for Azure Document Intelligence). + + Args: + provider: Provider type + azure_credential: Azure credential for Managed Identity auth + + Returns: + OCRProvider instance or None + """ + if provider == OCRProviderType.AZURE_DOCUMENT_INTELLIGENCE: + endpoint = os.getenv("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT") + if not endpoint: + logger.warning("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT not set") + return None + + credential = azure_credential + if not credential: + key = os.getenv("AZURE_DOCUMENT_INTELLIGENCE_KEY") + if key: + from azure.core.credentials import AzureKeyCredential + credential = AzureKeyCredential(key) + else: + logger.warning("AZURE_DOCUMENT_INTELLIGENCE_KEY not set and no credential provided") + return None + + model_id = os.getenv("AZURE_DOCUMENT_INTELLIGENCE_MODEL", "prebuilt-read") + + return AzureDocumentIntelligenceOCRClient( + endpoint=endpoint, + credential=credential, + model_id=model_id + ) + + return None + + async def extract_text( + self, + image_data: bytes, + language: Optional[str] = None, + options: Optional[Dict[str, Any]] = None + ) -> Optional[OCRResult]: + """ + Extract text from image using configured OCR provider. + + Args: + image_data: Image bytes to process + language: Optional language hint + options: Optional provider-specific options + + Returns: + OCRResult with extracted text, or None if OCR is disabled + """ + if self.provider == OCRProviderType.NONE or self.provider_client is None: + logger.debug("OCR service is disabled") + return None + + try: + result = await self.provider_client.extract_text( + image_data=image_data, + language=language, + options=options + ) + logger.info(f"OCR extracted {len(result.text)} characters with {result.confidence:.2f} confidence") + return result + except Exception as e: + logger.error(f"OCR extraction failed: {e}") + return None + + async def extract_text_from_url( + self, + image_url: str, + language: Optional[str] = None, + options: Optional[Dict[str, Any]] = None + ) -> Optional[OCRResult]: + """ + Extract text from image URL using configured OCR provider. + + Args: + image_url: URL of image to process + language: Optional language hint + options: Optional provider-specific options + + Returns: + OCRResult with extracted text, or None if OCR is disabled + """ + if self.provider == OCRProviderType.NONE or self.provider_client is None: + return None + + if isinstance(self.provider_client, OllamaOCRClient): + try: + result = await self.provider_client.extract_text_from_url( + image_url=image_url, + language=language, + options=options + ) + return result + except Exception as e: + logger.error(f"OCR extraction from URL failed: {e}") + return None + else: + # For other providers, fetch image first + import aiohttp + try: + async with aiohttp.ClientSession() as session: + async with session.get(image_url) as response: + if response.status != 200: + logger.error(f"Failed to fetch image from URL: {response.status}") + return None + image_data = await response.read() + return await self.extract_text(image_data, language, options) + except Exception as e: + logger.error(f"Error fetching image from URL: {e}") + return None + + def is_enabled(self) -> bool: + """Check if OCR service is enabled.""" + return self.provider != OCRProviderType.NONE and self.provider_client is not None + diff --git a/app/backend/services/web_search/base.py b/app/backend/services/web_search/base.py new file mode 100644 index 0000000000..10b09a5172 --- /dev/null +++ b/app/backend/services/web_search/base.py @@ -0,0 +1,8 @@ +from typing import Protocol, List, Dict, Any + + +class WebSearchProvider(Protocol): + async def search(self, query: str, top: int = 5) -> List[Dict[str, Any]]: + ... + + diff --git a/app/backend/services/web_search/normalizer.py b/app/backend/services/web_search/normalizer.py new file mode 100644 index 0000000000..c97f127c4c --- /dev/null +++ b/app/backend/services/web_search/normalizer.py @@ -0,0 +1,17 @@ +from typing import List, Dict, Any + + +def normalize_serper(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + normalized: List[Dict[str, Any]] = [] + for item in items or []: + normalized.append( + { + "title": item.get("title", ""), + "url": item.get("link", ""), + "snippet": item.get("snippet", item.get("description", "")), + "provider": "serper", + } + ) + return normalized + + diff --git a/app/backend/services/web_search/serper_client.py b/app/backend/services/web_search/serper_client.py new file mode 100644 index 0000000000..dc4f649733 --- /dev/null +++ b/app/backend/services/web_search/serper_client.py @@ -0,0 +1,21 @@ +import aiohttp +from typing import List, Dict, Any + + +class SerperClient: + def __init__(self, api_key: str) -> None: + self.api_key = api_key + + async def search(self, query: str, top: int = 5) -> List[Dict[str, Any]]: + url = "https://google.serper.dev/search" + headers = {"X-API-KEY": self.api_key, "Content-Type": "application/json"} + payload = {"q": query, "num": top} + async with aiohttp.ClientSession() as session: + async with session.post(url, json=payload, headers=headers) as resp: + if resp.status != 200: + error_text = await resp.text() + raise Exception(f"SERPER API error {resp.status}: {error_text}") + data = await resp.json() + return data.get("organic", []) or [] + + diff --git a/app/frontend/index.html b/app/frontend/index.html index 30205db90f..addb584ce5 100644 --- a/app/frontend/index.html +++ b/app/frontend/index.html @@ -4,7 +4,8 @@ - Azure OpenAI + AI Search + SSOE Knowledge Assistant +
diff --git a/app/frontend/public/azure_favicon.ico b/app/frontend/public/azure_favicon.ico new file mode 100644 index 0000000000..f1fe50511c Binary files /dev/null and b/app/frontend/public/azure_favicon.ico differ diff --git a/app/frontend/public/favicon.ico b/app/frontend/public/favicon.ico index f1fe50511c..0140e5f4b8 100644 Binary files a/app/frontend/public/favicon.ico and b/app/frontend/public/favicon.ico differ diff --git a/app/frontend/src/assets/ssoe_logo.png b/app/frontend/src/assets/ssoe_logo.png new file mode 100644 index 0000000000..2904b4ee60 Binary files /dev/null and b/app/frontend/src/assets/ssoe_logo.png differ diff --git a/app/frontend/src/assets/ssoe_logo.svg b/app/frontend/src/assets/ssoe_logo.svg new file mode 100644 index 0000000000..dc45712084 --- /dev/null +++ b/app/frontend/src/assets/ssoe_logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/app/frontend/src/index.css b/app/frontend/src/index.css index f3414a3c95..c4d837de67 100644 --- a/app/frontend/src/index.css +++ b/app/frontend/src/index.css @@ -29,6 +29,11 @@ body { #root { height: 100%; + --company-primary: #af252e; + --company-secondary: #ffffff; + --text-color: #222222; + --surface-bg: #ffffff; + --surface-border: #e0e0e0; } @media (min-width: 480px) { diff --git a/app/frontend/src/locales/en/translation.json b/app/frontend/src/locales/en/translation.json index 9976d143b9..0f9291957f 100644 --- a/app/frontend/src/locales/en/translation.json +++ b/app/frontend/src/locales/en/translation.json @@ -1,6 +1,6 @@ { - "pageTitle": "Azure OpenAI + AI Search", - "headerTitle": "Azure OpenAI + AI Search", + "pageTitle": "SSOE Knowledge Assistant", + "headerTitle": "SSOE Knowledge Assistant", "chat": "Chat", "qa": "Ask a question", "login": "Login", @@ -37,17 +37,17 @@ "chatEmptyStateTitle": "Chat with your data", "chatEmptyStateSubtitle": "Ask anything or try an example", "defaultExamples": { - "1": "What is included in my Northwind Health Plus plan that is not in standard?", - "2": "What happens in a performance review?", - "3": "What does a Product Manager do?", - "placeholder": "Type a new question (e.g. does my plan cover annual eye exams?)" + "1": "What problem does the buckling-restrained brace solve compared to traditional braces?", + "2": "How does the seismic isolation system maintain stability during normal use while still allowing movement in an earthquake?", + "3": "What materials are suggested for the support pads in a seismic isolation device?", + "placeholder": "Type a new question (e.g. what is the maximum span for a reinforced concrete slab?)" }, "askTitle": "Ask your data", "multimodalExamples": { - "1": "Compare the impact of interest rates and GDP in financial markets.", - "2": "What is the expected trend for the S&P 500 index over the next five years? Compare it to the past S&P 500 performance", - "3": "Can you identify any correlation between oil prices and stock market trends?", - "placeholder": "Example: Does my plan cover annual eye exams?" + "1": "Compare the structural performance of steel vs. reinforced concrete for high-rise construction.", + "2": "What are the seismic design considerations for bridges compared to tall buildings?", + "3": "Can you identify any correlation between soil type and foundation settlement issues?", + "placeholder": "Example: What is the wind load requirement for a 10-story building?" }, "generatingAnswer": "Generating answer", "citationWithColon": "Citation:", diff --git a/app/frontend/src/pages/layout/Layout.module.css b/app/frontend/src/pages/layout/Layout.module.css index 4f53c0177e..8b22025e90 100644 --- a/app/frontend/src/pages/layout/Layout.module.css +++ b/app/frontend/src/pages/layout/Layout.module.css @@ -11,8 +11,9 @@ } .header { - background-color: #222222; - color: #f2f2f2; + background-color: var(--surface-bg); + color: var(--text-color); + border-bottom: 1px solid var(--surface-border); } .headerContainer { @@ -27,7 +28,7 @@ .headerTitleContainer { display: flex; align-items: center; - color: #f2f2f2; + color: var(--text-color); text-decoration: none; } @@ -45,7 +46,7 @@ z-index: 100; display: none; flex-direction: column; - background-color: #222222; + background-color: var(--surface-bg); position: absolute; top: 2.7rem; right: 0; @@ -53,6 +54,7 @@ list-style: none; padding: 1.1rem; align-items: flex-end; + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); } .headerNavList.show { @@ -61,9 +63,9 @@ .headerNavPageLink { padding: 1rem; - border-bottom: 1px solid #333; + border-bottom: 1px solid var(--surface-border); text-align: right; - color: #f2f2f2; + color: var(--text-color); text-decoration: none; opacity: 0.75; transition-timing-function: cubic-bezier(0.16, 1, 0.3, 1); @@ -74,10 +76,11 @@ .headerNavPageLink:hover { opacity: 1; + background-color: var(--surface-bg); } .headerNavPageLinkActive { - color: #f2f2f2; + color: var(--text-color); text-decoration: none; font-size: 1.2rem; text-align: right; @@ -92,12 +95,18 @@ display: block; background: none; border: none; - color: #f2f2f2; + color: var(--text-color); font-size: 1.5rem; cursor: pointer; z-index: 101; } +.rightContainer { + display: flex; + align-items: center; + gap: 1rem; +} + .loginMenuContainer { display: flex; align-items: center; @@ -116,6 +125,7 @@ padding: 0; position: static; width: auto; + gap: 1.5rem; } .headerNavPageLink { @@ -129,8 +139,9 @@ } .headerNavPageLinkActive { - color: #f2f2f2; + color: var(--text-color); text-decoration: none; + padding: 0; } .headerNavLeftMargin { @@ -144,6 +155,10 @@ vertical-align: middle; } + .rightContainer { + gap: 2rem; + } + .menuToggle { display: none; } diff --git a/app/frontend/src/pages/layout/Layout.tsx b/app/frontend/src/pages/layout/Layout.tsx index 1bf90f9c0a..6b9ed2e8fe 100644 --- a/app/frontend/src/pages/layout/Layout.tsx +++ b/app/frontend/src/pages/layout/Layout.tsx @@ -7,6 +7,7 @@ import { useLogin } from "../../authConfig"; import { LoginButton } from "../../components/LoginButton"; import { IconButton } from "@fluentui/react"; +import ssoeLogo from "../../assets/ssoe_logo.svg"; const Layout = () => { const { t } = useTranslation(); @@ -39,38 +40,40 @@ const Layout = () => {
-

{t("headerTitle")}

+ SSOE - -
- {useLogin && } - +
+ +
+ {useLogin && } + +
diff --git a/azure.yaml b/azure.yaml index f629d9a374..ff26e3e866 100644 --- a/azure.yaml +++ b/azure.yaml @@ -40,6 +40,11 @@ services: run: cd ../frontend;npm install;npm run build interactive: false continueOnError: false + agents: + project: ./agents + language: py + # Agents service deploys to App Service (required for Bot Framework) + host: appservice hooks: preprovision: windows: diff --git a/cleanup.ps1 b/cleanup.ps1 new file mode 100644 index 0000000000..ef40bc1726 --- /dev/null +++ b/cleanup.ps1 @@ -0,0 +1,102 @@ +# Cleanup Script - Removes temporary and cache files +# Safe cleanup - keeps all source code and documentation + +Write-Host "`n=== CLEANING UP PROJECT ===" -ForegroundColor Yellow +Write-Host "This will remove temporary files, cache files, and .env files" -ForegroundColor Cyan +Write-Host "Source code and documentation will be preserved`n" -ForegroundColor Green + +$itemsRemoved = 0 + +# Remove Python cache files +Write-Host "Cleaning Python cache files..." -ForegroundColor Yellow +Get-ChildItem -Path . -Recurse -Directory -Filter "__pycache__" -Force -ErrorAction SilentlyContinue | ForEach-Object { + $itemsRemoved++ + Remove-Item -Path $_.FullName -Recurse -Force -ErrorAction SilentlyContinue + Write-Host " Removed: $($_.FullName)" -ForegroundColor Gray +} + +# Remove .pyc files +Write-Host "Cleaning .pyc files..." -ForegroundColor Yellow +Get-ChildItem -Path . -Recurse -Filter "*.pyc" -Force -ErrorAction SilentlyContinue | ForEach-Object { + $itemsRemoved++ + Remove-Item -Path $_.FullName -Force -ErrorAction SilentlyContinue + Write-Host " Removed: $($_.FullName)" -ForegroundColor Gray +} + +# Remove .pyo files +Write-Host "Cleaning .pyo files..." -ForegroundColor Yellow +Get-ChildItem -Path . -Recurse -Filter "*.pyo" -Force -ErrorAction SilentlyContinue | ForEach-Object { + $itemsRemoved++ + Remove-Item -Path $_.FullName -Force -ErrorAction SilentlyContinue +} + +# Remove .env files (WARNING: This removes environment files!) +Write-Host "Cleaning .env files..." -ForegroundColor Yellow +Get-ChildItem -Path . -Recurse -Filter ".env" -Force -ErrorAction SilentlyContinue | ForEach-Object { + $itemsRemoved++ + Remove-Item -Path $_.FullName -Force -ErrorAction SilentlyContinue + Write-Host " Removed: $($_.FullName)" -ForegroundColor Gray +} + +# Remove .env.* files (but keep .env.example) +Write-Host "Cleaning .env.* files (keeping .env.example)..." -ForegroundColor Yellow +Get-ChildItem -Path . -Recurse -Filter ".env.*" -Force -ErrorAction SilentlyContinue | Where-Object { $_.Name -notlike "*.example" } | ForEach-Object { + $itemsRemoved++ + Remove-Item -Path $_.FullName -Force -ErrorAction SilentlyContinue + Write-Host " Removed: $($_.FullName)" -ForegroundColor Gray +} + +# Remove node_modules (if exists) +Write-Host "Cleaning node_modules..." -ForegroundColor Yellow +Get-ChildItem -Path . -Recurse -Directory -Filter "node_modules" -Force -ErrorAction SilentlyContinue | ForEach-Object { + $itemsRemoved++ + Write-Host " Removing: $($_.FullName) (this may take a while)..." -ForegroundColor Gray + Remove-Item -Path $_.FullName -Recurse -Force -ErrorAction SilentlyContinue +} + +# Remove .pytest_cache +Write-Host "Cleaning pytest cache..." -ForegroundColor Yellow +Get-ChildItem -Path . -Recurse -Directory -Filter ".pytest_cache" -Force -ErrorAction SilentlyContinue | ForEach-Object { + $itemsRemoved++ + Remove-Item -Path $_.FullName -Recurse -Force -ErrorAction SilentlyContinue +} + +# Remove .mypy_cache +Write-Host "Cleaning mypy cache..." -ForegroundColor Yellow +Get-ChildItem -Path . -Recurse -Directory -Filter ".mypy_cache" -Force -ErrorAction SilentlyContinue | ForEach-Object { + $itemsRemoved++ + Remove-Item -Path $_.FullName -Recurse -Force -ErrorAction SilentlyContinue +} + +# Remove dist and build folders +Write-Host "Cleaning build artifacts..." -ForegroundColor Yellow +@("dist", "build", ".egg-info") | ForEach-Object { + Get-ChildItem -Path . -Recurse -Directory -Filter $_ -Force -ErrorAction SilentlyContinue | ForEach-Object { + $itemsRemoved++ + Remove-Item -Path $_.FullName -Recurse -Force -ErrorAction SilentlyContinue + Write-Host " Removed: $($_.FullName)" -ForegroundColor Gray + } +} + +# Remove .DS_Store files (macOS) +Write-Host "Cleaning .DS_Store files..." -ForegroundColor Yellow +Get-ChildItem -Path . -Recurse -Filter ".DS_Store" -Force -ErrorAction SilentlyContinue | ForEach-Object { + $itemsRemoved++ + Remove-Item -Path $_.FullName -Force -ErrorAction SilentlyContinue +} + +# Remove Thumbs.db files (Windows) +Write-Host "Cleaning Thumbs.db files..." -ForegroundColor Yellow +Get-ChildItem -Path . -Recurse -Filter "Thumbs.db" -Force -ErrorAction SilentlyContinue | ForEach-Object { + $itemsRemoved++ + Remove-Item -Path $_.FullName -Force -ErrorAction SilentlyContinue +} + +Write-Host "`n=== CLEANUP COMPLETE ===" -ForegroundColor Green +Write-Host "Items removed: $itemsRemoved" -ForegroundColor Cyan +Write-Host "`nSource code and documentation preserved." -ForegroundColor Green +Write-Host "You can now start fresh!`n" -ForegroundColor Yellow + + + + diff --git a/custom-embedding-service/Dockerfile b/custom-embedding-service/Dockerfile new file mode 100644 index 0000000000..e2a7db3128 --- /dev/null +++ b/custom-embedding-service/Dockerfile @@ -0,0 +1,28 @@ +FROM python:3.11-slim + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + build-essential \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Set working directory +WORKDIR /app + +# Copy requirements and install dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code +COPY app.py . +COPY constants.py . + +# Expose port +EXPOSE 8000 + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ + CMD curl -f http://localhost:8000/health || exit 1 + +# Run application +CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "1"] diff --git a/custom-embedding-service/README.md b/custom-embedding-service/README.md new file mode 100644 index 0000000000..10f6791317 --- /dev/null +++ b/custom-embedding-service/README.md @@ -0,0 +1,277 @@ +# PatentsBERTa Integration - Implementation Complete + +This document summarizes the completed PatentsBERTa embedding integration for the AI Master Engineer application. + +## 📁 Files Created/Modified + +### Core Service Files +- `custom-embedding-service/app.py` - FastAPI service for PatentsBERTa embeddings +- `custom-embedding-service/requirements.txt` - Python dependencies +- `custom-embedding-service/Dockerfile` - Container configuration + +### Backend Integration +- `app/backend/prepdocslib/patentsberta_embeddings.py` - Custom embedding class +- `app/backend/prepdocs.py` - Updated to support PatentsBERTa (modified) +- `app/backend/prepdocslib/searchmanager.py` - Updated for 768-dimensional embeddings (modified) + +### Infrastructure +- `infra/modules/patentsberta.bicep` - Azure Container App configuration +- `infra/main.bicep` - Updated to include PatentsBERTa service (modified) + +### Configuration & Scripts +- `.env.patentsberta.example` - Environment configuration template +- `scripts/deploy-patentsberta.sh` - Automated deployment script +- `scripts/switch-to-patentsberta.sh` - Environment switching script +- `scripts/test-patentsberta.py` - Comprehensive test suite + +## 🚀 Deployment Instructions + +### Prerequisites +1. Ensure `OPENAI_HOST=patentsberta` is set in your environment +2. Azure CLI logged in with appropriate permissions +3. Docker not required (uses Azure Container Registry build) +4. Generate a secure API key for PatentsBERTa authentication + +### Step-by-Step Deployment + +1. **Generate Secure API Key** + ```bash + # Generate a cryptographically secure API key + API_KEY=$(openssl rand -base64 32) + echo "Generated API Key: $API_KEY" + + # Set the API key in your environment + azd env set PATENTSBERTA_API_KEY "$API_KEY" + ``` + +2. **Build and Push Container to ACR** + ```bash + # Get your container registry name + REGISTRY_NAME=$(az acr list --resource-group rg-ai-master-engineer --query "[0].name" -o tsv) + + # Build and push image + cd custom-embedding-service + az acr build --registry $REGISTRY_NAME --image patentsberta-embeddings:latest . + cd .. + ``` + +3. **Deploy Infrastructure** + ```bash + azd up --no-prompt + ``` + +4. **Grant Container Registry Access** (if deployment fails) + ```bash + # Get container app identity + PRINCIPAL_ID=$(az containerapp show --name patentsberta-* --resource-group rg-ai-master-engineer --query "identity.principalId" -o tsv) + + # Get registry resource ID + REGISTRY_ID=$(az acr show --name $REGISTRY_NAME --resource-group rg-ai-master-engineer --query "id" -o tsv) + + # Grant AcrPull role + az role assignment create --assignee $PRINCIPAL_ID --role AcrPull --scope $REGISTRY_ID + + # Retry deployment + azd up --no-prompt + ``` + +5. **Verify Deployment** + ```bash + # Get PatentsBERTa endpoint and API key + ENDPOINT=$(azd env get-values | grep PATENTSBERTA_ENDPOINT | cut -d'=' -f2 | tr -d '"') + API_KEY=$(azd env get-values | grep PATENTSBERTA_API_KEY | cut -d'=' -f2 | tr -d '"') + + # Test health (no auth required) + curl "$ENDPOINT/health" + + # Test embeddings with API key + curl -X POST "$ENDPOINT/embeddings" \ + -H "Content-Type: application/json" \ + -H "X-API-Key: $API_KEY" \ + -d '{"texts": ["semiconductor wafer processing"]}' | jq '.embeddings[0] | length' + ``` + +6. **Reindex Documents** (if switching from existing deployment) + ```bash + # Process documents with PatentsBERTa embeddings + cd app/backend + python prepdocs.py '../../data/*' + cd ../.. + + # Or for specific document types + python prepdocs.py '../../data/patents/*.pdf' + ``` + +## 🧪 Testing + +### Test the PatentsBERTa Service + +#### Option 1: Comprehensive Test Suite + +```bash +# Export environment variables from Azure deployment +export PATENTSBERTA_ENDPOINT=$(azd env get-values | grep PATENTSBERTA_ENDPOINT | cut -d'=' -f2 | tr -d '"') +export PATENTSBERTA_API_KEY=$(azd env get-values | grep PATENTSBERTA_API_KEY | cut -d'=' -f2- | tr -d '"') + +# Run the full test suite (includes authentication test) +python tests/test-patentsberta.py +``` + +#### Option 2: Manual Testing with curl +```bash +# Get endpoint and API key from environment +ENDPOINT=$(azd env get-values | grep PATENTSBERTA_ENDPOINT | cut -d'=' -f2 | tr -d '"') +API_KEY=$(azd env get-values | grep PATENTSBERTA_API_KEY | cut -d'=' -f2- | tr -d '"') + +# Test health (no auth required) +curl "$ENDPOINT/health" + +# Test embeddings with API key (should return 768) +curl -X POST "$ENDPOINT/embeddings" \ + -H "Content-Type: application/json" \ + -H "X-API-Key: $API_KEY" \ + -d '{"texts": ["semiconductor wafer processing"]}' | jq '.embeddings[0] | length' + +# Test info endpoint (no auth required) +curl "$ENDPOINT/info" + +# Test authentication (should fail without API key) +curl -X POST "$ENDPOINT/embeddings" \ + -H "Content-Type: application/json" \ + -d '{"texts": ["test"]}' | jq '.detail' +``` + +## ⚙️ Configuration + +### Environment Variables +```bash +# Core PatentsBERTa configuration +OPENAI_HOST=patentsberta +PATENTSBERTA_ENDPOINT=https://your-endpoint.azurecontainerapps.io +PATENTSBERTA_API_KEY=your-secure-api-key-here # Required for API authentication +AZURE_OPENAI_EMB_DIMENSIONS=768 +AZURE_SEARCH_FIELD_NAME_EMBEDDING=embedding_patentsberta +``` + +### Key Features +- **768-dimensional embeddings** (vs 1536 for Azure OpenAI) +- **Patent-specific training** for better technical document understanding +- **Self-hosted** for cost control and customization +- **Auto-scaling** Container App deployment +- **Health monitoring** and performance testing +- **API key authentication** for secure access to embeddings endpoint + +### Security +- **Protected /embeddings endpoint** with X-API-Key header authentication +- **Public health and info endpoints** for monitoring +- **No-op authentication** if PATENTSBERTA_API_KEY is not configured +- **Secure parameter handling** in Azure deployment + +## 🔄 Switching Between Embedding Services + +### Switch to PatentsBERTa +```bash +./scripts/switch-to-patentsberta.sh +``` + +### Switch Back to Azure OpenAI +```bash +# Restore from backup (created automatically) +azd env set-values < .env.backup.YYYYMMDD_HHMMSS +``` + +## 📊 Expected Benefits + +### For Patent Documents +- **Better semantic understanding** of technical terminology +- **Improved similarity matching** for engineering concepts +- **Enhanced retrieval accuracy** for patent claims and specifications + +### Cost & Control +- **Predictable costs** vs per-token pricing +- **Self-managed scaling** and performance tuning +- **Custom model updates** and fine-tuning capabilities + +## 🔍 Architecture Overview + +``` +Document Processing → PatentsBERTa Service → Azure AI Search (768D) +User Queries → PatentsBERTa Service → Vector Search → Results +``` + +### Components +1. **PatentsBERTa Container App** - Hosts the embedding model +2. **Custom Embedding Class** - Integrates with existing backend +3. **Updated Search Index** - Supports 768-dimensional vectors +4. **Environment Configuration** - Switches between embedding services + +## 🚨 Important Notes + +### Index Recreation Required +- The search index must be recreated with new embedding dimensions +- All documents need to be reprocessed with PatentsBERTa embeddings +- Backup existing data if needed before switching + +### Performance Considerations +- **Initial model loading** takes 2-3 minutes +- **First embedding request** may be slower due to model warmup +- **Batch processing** is more efficient than individual requests + +### Monitoring +- Health endpoint: `/health` +- Model info: `/info` +- Application Insights integration for logging +- Container App metrics for scaling decisions + +## 🛠️ Troubleshooting + +### Common Issues + +**Container Image Pull Failed** +```bash +# Grant managed identity access to ACR +PRINCIPAL_ID=$(az containerapp show --name patentsberta-* --resource-group rg-ai-master-engineer --query "identity.principalId" -o tsv) +REGISTRY_ID=$(az acr show --name YOUR_REGISTRY --resource-group rg-ai-master-engineer --query "id" -o tsv) +az role assignment create --assignee $PRINCIPAL_ID --role AcrPull --scope $REGISTRY_ID +``` + +**Bicep Template Error (openAiHost)** +- Ensure `infra/main.bicep` includes `patentsberta` in allowed values for `openAiHost` parameter + +**Search Index Compression Error** +- Truncation dimension must be less than embedding dimensions (768) +- Fixed automatically in `searchmanager.py` with dynamic calculation + +**Service Not Starting** +```bash +# Check container logs +az containerapp logs show --name patentsberta-* --resource-group rg-ai-master-engineer +``` + +**Memory Issues** +```bash +# Increase memory allocation +az containerapp update --name patentsberta-* --memory 8Gi +``` + +### Support Commands +```bash +# Get current environment status +azd env get-values | grep -E "(OPENAI_HOST|PATENTSBERTA|AZURE_OPENAI_EMB|AZURE_SEARCH_FIELD)" + +# Test service health +ENDPOINT=$(azd env get-values | grep PATENTSBERTA_ENDPOINT | cut -d'=' -f2 | tr -d '"') +curl "$ENDPOINT/health" + +# Check embedding dimensions +curl "$ENDPOINT/info" + +# View container logs +az containerapp logs show --name patentsberta-* --resource-group rg-ai-master-engineer + +# Monitor resource usage +az monitor metrics list --resource $(az containerapp show --name patentsberta-* --resource-group rg-ai-master-engineer --query "id" -o tsv) + +# Check container app status +az containerapp show --name patentsberta-* --resource-group rg-ai-master-engineer --query "{name:name,status:properties.provisioningState,fqdn:properties.configuration.ingress.fqdn}" +``` diff --git a/custom-embedding-service/app.py b/custom-embedding-service/app.py new file mode 100644 index 0000000000..843d66735f --- /dev/null +++ b/custom-embedding-service/app.py @@ -0,0 +1,170 @@ +from fastapi import FastAPI, HTTPException, Depends, Header +from pydantic import BaseModel, Field, field_validator +from typing import List, Optional +import torch +from transformers import AutoTokenizer, AutoModel +import logging +import numpy as np + +from constants import ( + API_KEY, + MAX_BATCH_SIZE, + MAX_TEXT_LENGTH, + MIN_TEXT_LENGTH, + MAX_TOTAL_CHARS, + MODEL_NAME, + MODEL_MAX_LENGTH, + EMBEDDING_DIMENSIONS, + MODEL_DESCRIPTION +) + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +def api_key_auth(x_api_key: str | None = Header(default=None)): + """API key authentication dependency""" + if API_KEY and x_api_key != API_KEY: + raise HTTPException(status_code=401, detail="Invalid or missing API key") + +app = FastAPI( + title="PatentsBERTa Embedding Service", + description="Patent-specific BERT embeddings for technical documents", + version="1.0.0" +) + +class EmbeddingRequest(BaseModel): + texts: List[str] = Field(..., min_items=1, max_items=MAX_BATCH_SIZE) + normalize: bool = True + + @field_validator('texts') + @classmethod + def validate_texts(cls, v): + if not v: + raise ValueError("texts cannot be empty") + + total_chars = 0 + for i, text in enumerate(v): + if not isinstance(text, str): + raise ValueError(f"Text at index {i} must be a string") + + text_len = len(text.strip()) + if text_len < MIN_TEXT_LENGTH: + raise ValueError(f"Text at index {i} is too short (minimum {MIN_TEXT_LENGTH} characters)") + + if text_len > MAX_TEXT_LENGTH: + raise ValueError(f"Text at index {i} is too long (maximum {MAX_TEXT_LENGTH} characters)") + + total_chars += text_len + + if total_chars > MAX_TOTAL_CHARS: + raise ValueError(f"Total request size too large ({total_chars} chars, maximum {MAX_TOTAL_CHARS})") + + return v + +class EmbeddingResponse(BaseModel): + embeddings: List[List[float]] + model: str = MODEL_NAME + dimensions: int = EMBEDDING_DIMENSIONS + +# Global model variables +tokenizer = None +model = None + +@app.on_event("startup") +async def load_model(): + global tokenizer, model + try: + logger.info("Loading PatentsBERTa model...") + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) + model = AutoModel.from_pretrained(MODEL_NAME) + + # Set to evaluation mode + model.eval() + + # Move to GPU if available + if torch.cuda.is_available(): + model = model.cuda() + logger.info("Model loaded on GPU") + else: + logger.info("Model loaded on CPU") + + logger.info("PatentsBERTa model loaded successfully") + except Exception as e: + logger.error(f"Failed to load model: {e}") + raise e + +def mean_pooling(model_output, attention_mask): + """Mean pooling to get sentence embeddings""" + token_embeddings = model_output[0] + input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() + return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) + +@app.post("/embeddings", response_model=EmbeddingResponse, dependencies=[Depends(api_key_auth)]) +async def create_embeddings(request: EmbeddingRequest): + try: + if not tokenizer or not model: + raise HTTPException(status_code=503, detail="Model not loaded") + + # Tokenize inputs + encoded_input = tokenizer( + request.texts, + padding=True, + truncation=True, + max_length=MODEL_MAX_LENGTH, + return_tensors='pt' + ) + + # Move to GPU if available + if torch.cuda.is_available(): + encoded_input = {k: v.cuda() for k, v in encoded_input.items()} + + # Generate embeddings + with torch.no_grad(): + model_output = model(**encoded_input) + embeddings = mean_pooling(model_output, encoded_input['attention_mask']) + + # Normalize embeddings if requested + if request.normalize: + embeddings = torch.nn.functional.normalize(embeddings, p=2, dim=1) + + # Convert to list + embeddings_list = embeddings.cpu().numpy().tolist() + + return EmbeddingResponse( + embeddings=embeddings_list, + model=MODEL_NAME, + dimensions=len(embeddings_list[0]) if embeddings_list else 0 + ) + + except Exception as e: + logger.error(f"Embedding generation failed: {e}") + raise HTTPException(status_code=500, detail=f"Embedding generation failed: {str(e)}") + +@app.get("/health") +async def health_check(): + return { + "status": "healthy", + "model_loaded": tokenizer is not None and model is not None, + "gpu_available": torch.cuda.is_available() + } + +@app.get("/info") +async def model_info(): + return { + "model_name": MODEL_NAME, + "description": MODEL_DESCRIPTION, + "max_input_length": MODEL_MAX_LENGTH, + "embedding_dimensions": EMBEDDING_DIMENSIONS, + "gpu_enabled": torch.cuda.is_available(), + "limits": { + "max_batch_size": MAX_BATCH_SIZE, + "max_text_length": MAX_TEXT_LENGTH, + "min_text_length": MIN_TEXT_LENGTH, + "max_total_chars": MAX_TOTAL_CHARS + } + } + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/custom-embedding-service/constants.py b/custom-embedding-service/constants.py new file mode 100644 index 0000000000..78a6d47315 --- /dev/null +++ b/custom-embedding-service/constants.py @@ -0,0 +1,16 @@ +import os + +# API Authentication +API_KEY = os.getenv("PATENTSBERTA_API_KEY") + +# Request Limits and Validation Constants +MAX_BATCH_SIZE = int(os.getenv("MAX_BATCH_SIZE", "120")) # Maximum number of texts per request +MAX_TEXT_LENGTH = int(os.getenv("MAX_TEXT_LENGTH", "20000")) # Maximum characters per text +MIN_TEXT_LENGTH = 1 # Minimum characters per text +MAX_TOTAL_CHARS = int(os.getenv("MAX_TOTAL_CHARS", "100000")) # Maximum total characters in request + +# Model Configuration +MODEL_NAME = "AI-Growth-Lab/PatentSBERTa" +MODEL_MAX_LENGTH = 512 +EMBEDDING_DIMENSIONS = 768 +MODEL_DESCRIPTION = "Patent-specific BERT model for technical document embeddings" \ No newline at end of file diff --git a/custom-embedding-service/requirements.txt b/custom-embedding-service/requirements.txt new file mode 100644 index 0000000000..159a04d788 --- /dev/null +++ b/custom-embedding-service/requirements.txt @@ -0,0 +1,7 @@ +fastapi==0.104.1 +uvicorn[standard]==0.24.0 +transformers==4.36.0 +torch==2.1.0 +numpy==1.24.3 +pydantic==2.5.0 +accelerate==0.25.0 diff --git a/data/Benefit_Options.pdf b/data/Benefit_Options.pdf deleted file mode 100644 index 6a4c07dc94..0000000000 Binary files a/data/Benefit_Options.pdf and /dev/null differ diff --git a/data/Contoso_Electronics_Company_Overview.md b/data/Contoso_Electronics_Company_Overview.md deleted file mode 100644 index 033d7dd84a..0000000000 --- a/data/Contoso_Electronics_Company_Overview.md +++ /dev/null @@ -1,48 +0,0 @@ -# Contoso Electronics - -*Disclaimer: This content is generated by AI and may not accurately represent factual information about any real entity. Use this information with caution and verify details from reliable sources.* - -## History - -Contoso Electronics, a pioneering force in the tech industry, was founded in 1985 by visionary entrepreneurs with a passion for innovation. Over the years, the company has played a pivotal role in shaping the landscape of consumer electronics. - -| Year | Milestone | -|------|-----------| -| 1985 | Company founded with a focus on cutting-edge technology | -| 1990 | Launched the first-ever handheld personal computer | -| 2000 | Introduced groundbreaking advancements in AI and robotics | -| 2015 | Expansion into sustainable and eco-friendly product lines | - -## Company Overview - -At Contoso Electronics, we take pride in fostering a dynamic and inclusive workplace. Our dedicated team of experts collaborates to create innovative solutions that empower and connect people globally. - -### Core Values - -- **Innovation:** Constantly pushing the boundaries of technology. -- **Diversity:** Embracing different perspectives for creative excellence. -- **Sustainability:** Committed to eco-friendly practices in our products. - -## Vacation Perks - -We believe in work-life balance and understand the importance of well-deserved breaks. Our vacation perks are designed to help our employees recharge and return with renewed enthusiasm. - -| Vacation Tier | Duration | Additional Benefits | -|---------------|----------|---------------------| -| Standard | 2 weeks | Health and wellness stipend | -| Senior | 4 weeks | Travel vouchers for a dream destination | -| Executive | 6 weeks | Luxury resort getaway with family | - -## Employee Recognition - -Recognizing the hard work and dedication of our employees is at the core of our culture. Here are some ways we celebrate achievements: - -- Monthly "Innovator of the Month" awards -- Annual gala with awards for outstanding contributions -- Team-building retreats for high-performing departments - -## Join Us! - -Contoso Electronics is always on the lookout for talented individuals who share our passion for innovation. If you're ready to be part of a dynamic team shaping the future of technology, check out our [careers page](http://www.contoso.com) for exciting opportunities. - -[Learn more about Contoso Electronics!](http://www.contoso.com) diff --git a/data/Json_Examples/2189.json b/data/Json_Examples/2189.json deleted file mode 100644 index d7066c9fbd..0000000000 --- a/data/Json_Examples/2189.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "AreaPath": "SmartHotel360", - "AssignedTo": null, - "Categories": null, - "ChangedDate": "2023-12-13T23:08:38.69Z", - "ClosedDate": null, - "CreatedDate": "2023-12-13T23:08:38.69Z", - "Description": "As a customer, I would like to reserve a conference room such that:

1. It should display available date and time slots
2. Give an option to reserve a conference room for X hours
3. One can reserve a conference room for max 4 hours per day
", - "Id": 2189, - "State": "New", - "StateChangeDate": "2023-12-13T23:08:38.69Z", - "Tags": "Reservation", - "Title": "As a customer, I would like to reserve a conference room" -} diff --git a/data/Json_Examples/2190.json b/data/Json_Examples/2190.json deleted file mode 100644 index 5a45f1158f..0000000000 --- a/data/Json_Examples/2190.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "AreaPath": "SmartHotel360", - "AssignedTo": null, - "Categories": null, - "ChangedDate": "2023-12-13T23:08:38.997Z", - "ClosedDate": null, - "CreatedDate": "2023-12-13T23:08:38.997Z", - "Description": "

Enter the guest's name to whom you would\nlike to send a confirmation, display the company, contact, source\nand agent associated\nwith the reservation.

", - "Id": 2190, - "State": "New", - "StateChangeDate": "2023-12-13T23:08:38.997Z", - "Tags": "Notification", - "Title": "As a reservation agent, I would like to send confirmations to guest" -} diff --git a/data/Json_Examples/2191.json b/data/Json_Examples/2191.json deleted file mode 100644 index 455e4c9a24..0000000000 --- a/data/Json_Examples/2191.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "AreaPath": "SmartHotel360", - "AssignedTo": null, - "Categories": null, - "ChangedDate": "2023-12-13T23:08:39.17Z", - "ClosedDate": null, - "CreatedDate": "2023-12-13T23:08:39.17Z", - "Description": "

If you have not picked up\nyour vehicle you can remove or cancel your reservation by clicking here.


1. Car reserved should have an option to cancel the request
2. Car driver should receive a notification about cancellation
", - "Id": 2191, - "State": "New", - "StateChangeDate": "2023-12-13T23:08:39.17Z", - "Tags": "Reservation", - "Title": "As a customer, I should be able to remove a car reservation " -} diff --git a/data/Json_Examples/2192.json b/data/Json_Examples/2192.json deleted file mode 100644 index d2e489f317..0000000000 --- a/data/Json_Examples/2192.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "AreaPath": "SmartHotel360", - "AssignedTo": null, - "Categories": null, - "ChangedDate": "2023-12-13T23:08:39.383Z", - "ClosedDate": null, - "CreatedDate": "2023-12-13T23:08:39.383Z", - "Description": "As a courtesy, grant an\nextra hour or two to leave the room, especially if it isn't booked\nfor the upcoming evening. But customer must call the front desk\nin advance and request a late checkout.


1. Late Check-in time should be displayed
2. Request should be sent to front-desk 
3. Any extra charge should be displayed
", - "Id": 2192, - "State": "New", - "StateChangeDate": "2023-12-13T23:08:39.383Z", - "Tags": "Front-desk; Members; Reservation", - "Title": "As a customer, I should be able to request hotel for late Check-out" -} diff --git a/data/Json_Examples/query.json b/data/Json_Examples/query.json deleted file mode 100644 index ceb9dc1a6f..0000000000 --- a/data/Json_Examples/query.json +++ /dev/null @@ -1,244 +0,0 @@ -[ - { - "fields": { - "System.Id": 2348, - "System.State": "New", - "System.Title": "Provide related items or frequently bought together section when people browse or search", - "System.WorkItemType": "Product Backlog Item" - }, - "id": 2348, - "relations": null, - "rev": 1, - "url": "https://dev.azure.com/codebytes/_apis/wit/workItems/2348" - }, - { - "fields": { - "System.Id": 2349, - "System.State": "New", - "System.Title": "As tester, I need to test the website on all the relevant broswers and devices and be sure that it can handle our load.", - "System.WorkItemType": "Product Backlog Item" - }, - "id": 2349, - "relations": null, - "rev": 1, - "url": "https://dev.azure.com/codebytes/_apis/wit/workItems/2349" - }, - { - "fields": { - "System.Id": 2350, - "System.State": "New", - "System.Title": "As a customer, I should be able to put items to shopping cart", - "System.WorkItemType": "Product Backlog Item" - }, - "id": 2350, - "relations": null, - "rev": 1, - "url": "https://dev.azure.com/codebytes/_apis/wit/workItems/2350" - }, - { - "fields": { - "System.Id": 2351, - "System.State": "New", - "System.Title": "As a customer, I should be able to print my purchase order", - "System.WorkItemType": "Product Backlog Item" - }, - "id": 2351, - "relations": null, - "rev": 1, - "url": "https://dev.azure.com/codebytes/_apis/wit/workItems/2351" - }, - { - "fields": { - "System.Id": 2352, - "System.State": "New", - "System.Title": "As a customer, I would like to have a sort capabaility by price and customer ratings", - "System.WorkItemType": "Product Backlog Item" - }, - "id": 2352, - "relations": null, - "rev": 1, - "url": "https://dev.azure.com/codebytes/_apis/wit/workItems/2352" - }, - { - "fields": { - "System.Id": 2353, - "System.State": "New", - "System.Title": "Recommended products must be based on customer purchase pattern history", - "System.WorkItemType": "Product Backlog Item" - }, - "id": 2353, - "relations": null, - "rev": 1, - "url": "https://dev.azure.com/codebytes/_apis/wit/workItems/2353" - }, - { - "fields": { - "System.Id": 2354, - "System.State": "New", - "System.Title": "As a customer, I would like to save my addresses so that I can easily select the address for delivery", - "System.WorkItemType": "Product Backlog Item" - }, - "id": 2354, - "relations": null, - "rev": 1, - "url": "https://dev.azure.com/codebytes/_apis/wit/workItems/2354" - }, - { - "fields": { - "System.Id": 2355, - "System.State": "New", - "System.Title": "As marketer, I want to run an A|B test on alternative Web Sites using Application Insights.", - "System.WorkItemType": "Product Backlog Item" - }, - "id": 2355, - "relations": null, - "rev": 1, - "url": "https://dev.azure.com/codebytes/_apis/wit/workItems/2355" - }, - { - "fields": { - "System.AssignedTo": { - "_links": { - "avatar": { - "href": "https://dev.azure.com/codebytes/_apis/GraphProfile/MemberAvatars/aad.ZDlhOGEyZjktMGZmZS03YjY4LTlkYjctNjk1ZWZiNGY2Nzg0" - } - }, - "descriptor": "aad.ZDlhOGEyZjktMGZmZS03YjY4LTlkYjctNjk1ZWZiNGY2Nzg0", - "displayName": "Chris Ayers", - "id": "cd8258ec-ad87-4c0d-9026-e5e343447185", - "imageUrl": "https://dev.azure.com/codebytes/_apis/GraphProfile/MemberAvatars/aad.ZDlhOGEyZjktMGZmZS03YjY4LTlkYjctNjk1ZWZiNGY2Nzg0", - "uniqueName": "chrisayers@microsoft.com", - "url": "https://spsprodeus27.vssps.visualstudio.com/A6b854e9d-a8be-405d-a4cc-5eb8e7027155/_apis/Identities/cd8258ec-ad87-4c0d-9026-e5e343447185" - }, - "System.Id": 2356, - "System.State": "Done", - "System.Title": "Provide customers the ability to track status of the package", - "System.WorkItemType": "Product Backlog Item" - }, - "id": 2356, - "relations": null, - "rev": 1, - "url": "https://dev.azure.com/codebytes/_apis/wit/workItems/2356" - }, - { - "fields": { - "System.AssignedTo": { - "_links": { - "avatar": { - "href": "https://dev.azure.com/codebytes/_apis/GraphProfile/MemberAvatars/aad.ZDlhOGEyZjktMGZmZS03YjY4LTlkYjctNjk1ZWZiNGY2Nzg0" - } - }, - "descriptor": "aad.ZDlhOGEyZjktMGZmZS03YjY4LTlkYjctNjk1ZWZiNGY2Nzg0", - "displayName": "Chris Ayers", - "id": "cd8258ec-ad87-4c0d-9026-e5e343447185", - "imageUrl": "https://dev.azure.com/codebytes/_apis/GraphProfile/MemberAvatars/aad.ZDlhOGEyZjktMGZmZS03YjY4LTlkYjctNjk1ZWZiNGY2Nzg0", - "uniqueName": "chrisayers@microsoft.com", - "url": "https://spsprodeus27.vssps.visualstudio.com/A6b854e9d-a8be-405d-a4cc-5eb8e7027155/_apis/Identities/cd8258ec-ad87-4c0d-9026-e5e343447185" - }, - "System.Id": 2357, - "System.State": "Done", - "System.Title": "As a customer, I would like to have the ability to send my items as gift", - "System.WorkItemType": "Product Backlog Item" - }, - "id": 2357, - "relations": null, - "rev": 2, - "url": "https://dev.azure.com/codebytes/_apis/wit/workItems/2357" - }, - { - "fields": { - "System.Id": 2358, - "System.State": "Committed", - "System.Title": "As a customer, I would like to store my credit card details securely", - "System.WorkItemType": "Product Backlog Item" - }, - "id": 2358, - "relations": null, - "rev": 1, - "url": "https://dev.azure.com/codebytes/_apis/wit/workItems/2358" - }, - { - "fields": { - "System.Id": 2359, - "System.State": "Committed", - "System.Title": "As a customer, I should be able to select different shipping option", - "System.WorkItemType": "Product Backlog Item" - }, - "id": 2359, - "relations": null, - "rev": 1, - "url": "https://dev.azure.com/codebytes/_apis/wit/workItems/2359" - }, - { - "fields": { - "System.Id": 2360, - "System.State": "Committed", - "System.Title": "As developer, I want to use Azure Machine Learning to provide a recommendations engine behind the website.", - "System.WorkItemType": "Product Backlog Item" - }, - "id": 2360, - "relations": null, - "rev": 1, - "url": "https://dev.azure.com/codebytes/_apis/wit/workItems/2360" - }, - { - "fields": { - "System.Id": 2361, - "System.State": "Committed", - "System.Title": "Provide tentative duration for shipping.", - "System.WorkItemType": "Product Backlog Item" - }, - "id": 2361, - "relations": null, - "rev": 1, - "url": "https://dev.azure.com/codebytes/_apis/wit/workItems/2361" - }, - { - "fields": { - "System.Id": 2362, - "System.State": "Approved", - "System.Title": "Notify the user about any changes made to the order", - "System.WorkItemType": "Product Backlog Item" - }, - "id": 2362, - "relations": null, - "rev": 1, - "url": "https://dev.azure.com/codebytes/_apis/wit/workItems/2362" - }, - { - "fields": { - "System.Id": 2363, - "System.State": "Approved", - "System.Title": "As a admin, I should be able to update prices on ad-hoc condition", - "System.WorkItemType": "Product Backlog Item" - }, - "id": 2363, - "relations": null, - "rev": 1, - "url": "https://dev.azure.com/codebytes/_apis/wit/workItems/2363" - }, - { - "fields": { - "System.Id": 2364, - "System.State": "Approved", - "System.Title": "As a customer, I would like to provide my feedback on items that I have purchased", - "System.WorkItemType": "Product Backlog Item" - }, - "id": 2364, - "relations": null, - "rev": 1, - "url": "https://dev.azure.com/codebytes/_apis/wit/workItems/2364" - }, - { - "fields": { - "System.Id": 2365, - "System.State": "Approved", - "System.Title": "As a customer, I would like to have a wishlist where I can add items for future purchase", - "System.WorkItemType": "Product Backlog Item" - }, - "id": 2365, - "relations": null, - "rev": 1, - "url": "https://dev.azure.com/codebytes/_apis/wit/workItems/2365" - } -] diff --git a/data/Multimodal_Examples/Financial Market Analysis Report 2023.pdf b/data/Multimodal_Examples/Financial Market Analysis Report 2023.pdf deleted file mode 100644 index eef17aad75..0000000000 Binary files a/data/Multimodal_Examples/Financial Market Analysis Report 2023.pdf and /dev/null differ diff --git a/data/Northwind_Health_Plus_Benefits_Details.pdf b/data/Northwind_Health_Plus_Benefits_Details.pdf deleted file mode 100644 index 97579a4fb5..0000000000 Binary files a/data/Northwind_Health_Plus_Benefits_Details.pdf and /dev/null differ diff --git a/data/Northwind_Standard_Benefits_Details.pdf b/data/Northwind_Standard_Benefits_Details.pdf deleted file mode 100644 index 7d50ff8c02..0000000000 Binary files a/data/Northwind_Standard_Benefits_Details.pdf and /dev/null differ diff --git a/data/PerksPlus.pdf b/data/PerksPlus.pdf deleted file mode 100644 index 2e167a2a6a..0000000000 Binary files a/data/PerksPlus.pdf and /dev/null differ diff --git a/data/employee_handbook.pdf b/data/employee_handbook.pdf deleted file mode 100644 index 878f36f7dd..0000000000 Binary files a/data/employee_handbook.pdf and /dev/null differ diff --git a/data/role_library.pdf b/data/role_library.pdf deleted file mode 100644 index ff70c65651..0000000000 Binary files a/data/role_library.pdf and /dev/null differ diff --git a/docs/README.md b/docs/README.md index e31c0e2d87..e57d2d0939 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,21 +8,17 @@ These are advanced topics that are not necessary for a basic deployment. - [Debugging the app on App Service](appservice.md) - [Deploying with azd: deep dive and CI/CD](azd.md) - [Deploying with existing Azure resources](deploy_existing.md) - - [Deploying from a free account](deploy_lowcost.md) + - [Deploying with minimal costs](deploy_lowcost.md) - [Enabling optional features](deploy_features.md) - [All features](docs/deploy_features.md) - [Login and access control](login_and_acl.md) - [Multimodal](multimodal.md) - [Private endpoints](deploy_private.md) - [Agentic retrieval](agentic_retrieval.md) - - [Sharing deployment environments](sharing_environments.md) - [Local development](localdev.md) - [Customizing the app](customization.md) - [App architecture](architecture.md) - [HTTP Protocol](http_protocol.md) - [Data ingestion](data_ingestion.md) -- [Evaluation](docs/evaluation.md) -- [Safety evaluation](safety_evaluation.md) - [Monitoring with Application Insights](monitoring.md) - [Productionizing](productionizing.md) -- [Alternative RAG chat samples](other_samples.md) diff --git a/docs/azure_app_service.md b/docs/azure_app_service.md deleted file mode 100644 index ac8157b5f6..0000000000 --- a/docs/azure_app_service.md +++ /dev/null @@ -1,40 +0,0 @@ -# RAG chat: Deploying on Azure App Service - -Due to [a limitation](https://github.com/Azure/azure-dev/issues/2736) of the Azure Developer CLI (`azd`), there can be only one host option in the [azure.yaml](../azure.yaml) file. -By default, `host: containerapp` is used and `host: appservice` is commented out. - -To deploy to Azure App Service, please follow the following steps: - -1. Comment out `host: containerapp` and uncomment `host: appservice` in the [azure.yaml](../azure.yaml) file. - -2. Login to your Azure account: - - ```bash - azd auth login - ``` - -3. Create a new `azd` environment to store the deployment parameters: - - ```bash - azd env new - ``` - - Enter a name that will be used for the resource group. - This will create a new folder in the `.azure` folder, and set it as the active environment for any calls to `azd` going forward. - -4. Set the deployment target to `appservice`: - - ```bash - azd env set DEPLOYMENT_TARGET appservice - ``` - -5. (Optional) This is the point where you can customize the deployment by setting other `azd` environment variables, in order to [use existing resources](deploy_existing.md), [enable optional features (such as auth or vision)](deploy_features.md), or [deploy to free tiers](deploy_lowcost.md). -6. Provision the resources and deploy the code: - - ```bash - azd up - ``` - - This will provision Azure resources and deploy this sample to those resources, including building the search index based on the files found in the `./data` folder. - - **Important**: Beware that the resources created by this command will incur immediate costs, primarily from the AI Search resource. These resources may accrue costs even if you interrupt the command before it is fully executed. You can run `azd down` or delete the resources manually to avoid unnecessary spending. diff --git a/docs/azure_container_apps.md b/docs/azure_container_apps.md deleted file mode 100644 index 62d3a1a606..0000000000 --- a/docs/azure_container_apps.md +++ /dev/null @@ -1,57 +0,0 @@ -# RAG chat: Deploying on Azure Container Apps - -Due to [a limitation](https://github.com/Azure/azure-dev/issues/2736) of the Azure Developer CLI (`azd`), there can be only one host option in the [azure.yaml](../azure.yaml) file. -By default, `host: containerapp` is used and `host: appservice` is commented out. - -However, if you have an older version of the repo, you may need to follow these steps to deploy to Container Apps instead, or you can stick with Azure App Service. - -To deploy to Azure Container Apps, please follow the following steps: - -1. Comment out `host: appservice` and uncomment `host: containerapp` in the [azure.yaml](../azure.yaml) file. - -2. Login to your Azure account: - - ```bash - azd auth login - ``` - -3. Create a new `azd` environment to store the deployment parameters: - - ```bash - azd env new - ``` - - Enter a name that will be used for the resource group. - This will create a new folder in the `.azure` folder, and set it as the active environment for any calls to `azd` going forward. - -4. Set the deployment target to `containerapps`: - - ```bash - azd env set DEPLOYMENT_TARGET containerapps - ``` - -5. (Optional) This is the point where you can customize the deployment by setting other `azd1 environment variables, in order to [use existing resources](docs/deploy_existing.md), [enable optional features (such as auth or vision)](docs/deploy_features.md), or [deploy to free tiers](docs/deploy_lowcost.md). -6. Provision the resources and deploy the code: - - ```bash - azd up - ``` - - This will provision Azure resources and deploy this sample to those resources, including building the search index based on the files found in the `./data` folder. - - **Important**: Beware that the resources created by this command will incur immediate costs, primarily from the AI Search resource. These resources may accrue costs even if you interrupt the command before it is fully executed. You can run `azd down` or delete the resources manually to avoid unnecessary spending. - -## Customizing Workload Profile - -The default workload profile is Consumption. If you want to use a dedicated workload profile like D4, please run: - -```bash -azd env AZURE_CONTAINER_APPS_WORKLOAD_PROFILE D4 -``` - -For a full list of workload profiles, please check [the workload profile documentation](https://learn.microsoft.com/azure/container-apps/workload-profiles-overview#profile-types). -Please note dedicated workload profiles have a different billing model than Consumption plan. Please check [the billing documentation](https://learn.microsoft.com/azure/container-apps/billing) for details. - -## Private endpoints - -Private endpoints is still in private preview for Azure Container Apps and not supported for now. diff --git a/docs/deploy_freetrial.md b/docs/deploy_freetrial.md deleted file mode 100644 index 1f85b38824..0000000000 --- a/docs/deploy_freetrial.md +++ /dev/null @@ -1,43 +0,0 @@ -# RAG chat: Deploying with a free trial account - -If you have just created an Azure free trial account and are using the free trial credits, -there are several modifications you need to make, due to restrictions on the free trial account. - -Follow these instructions *before* you run `azd up`. - -## Accomodate for low OpenAI quotas - -The free trial accounts currently get a max of 1K TPM (tokens-per-minute), whereas our Bicep templates try to allocate 30K TPM. - -To reduce the TPM allocation, run these commands: - -```shell -azd env set AZURE_OPENAI_CHATGPT_DEPLOYMENT_CAPACITY 1 -azd env set AZURE_OPENAI_EMB_DEPLOYMENT_CAPACITY 1 -``` - -Alternatively, if you have an OpenAI.com account, you can use that instead: - -```shell -azd env set OPENAI_HOST openai -azd env set OPENAI_ORGANIZATION {Your OpenAI organization} -azd env set OPENAI_API_KEY {Your OpenAI API key} -``` - -## Accomodate for Azure Container Apps restrictions - -By default, this project deploys to Azure Container Apps, using a remote build process that builds the Docker image in the cloud. -Unfortunately, free trial accounts cannot use that remote build process. - -You have two options: - -1. Comment out or delete `remoteBuild: true` in `azure.yaml`, and make sure you have Docker installed in your environment. - -2. Deploy using App Service instead: - - * Comment out `host: containerapp` and uncomment `host: appservice` in the [azure.yaml](../azure.yaml) file. - * Set the deployment target to `appservice`: - - ```shell - azd env set DEPLOYMENT_TARGET appservice - ``` diff --git a/docs/evaluation.md b/docs/evaluation.md deleted file mode 100644 index e47fa682ce..0000000000 --- a/docs/evaluation.md +++ /dev/null @@ -1,126 +0,0 @@ -# Evaluating the RAG answer quality - -[📺 Watch: (RAG Deep Dive series) Evaluating RAG answer quality](https://www.youtube.com/watch?v=lyCLu53fb3g) - -Follow these steps to evaluate the quality of the answers generated by the RAG flow. - -* [Deploy an evaluation model](#deploy-an-evaluation-model) -* [Setup the evaluation environment](#setup-the-evaluation-environment) -* [Generate ground truth data](#generate-ground-truth-data) -* [Run bulk evaluation](#run-bulk-evaluation) -* [Review the evaluation results](#review-the-evaluation-results) -* [Run bulk evaluation on a PR](#run-bulk-evaluation-on-a-pr) - -## Deploy an evaluation model - -1. Run this command to tell `azd` to deploy a GPT-4 level model for evaluation: - - ```shell - azd env set USE_EVAL true - ``` - -2. Set the capacity to the highest possible value to ensure that the evaluation runs relatively quickly. Even with a high capacity, it can take a long time to generate ground truth data and run bulk evaluations. - - ```shell - azd env set AZURE_OPENAI_EVAL_DEPLOYMENT_CAPACITY 100 - ``` - - By default, that will provision a `gpt-4o` model, version `2024-08-06`. To change those settings, set the azd environment variables `AZURE_OPENAI_EVAL_MODEL` and `AZURE_OPENAI_EVAL_MODEL_VERSION` to the desired values. - -3. Then, run the following command to provision the model: - - ```shell - azd provision - ``` - -## Setup the evaluation environment - -Make a new Python virtual environment and activate it. This is currently required due to incompatibilities between the dependencies of the evaluation script and the main project. - -```bash -python -m venv .evalenv -``` - -```bash -source .evalenv/bin/activate -``` - -Install all the dependencies for the evaluation script by running the following command: - -```bash -pip install -r evals/requirements.txt -``` - -## Generate ground truth data - -Generate ground truth data by running the following command: - -```bash -python evals/generate_ground_truth.py --numquestions=200 --numsearchdocs=1000 -``` - -The options are: - -* `numquestions`: The number of questions to generate. We suggest at least 200. -* `numsearchdocs`: The number of documents (chunks) to retrieve from your search index. You can leave off the option to fetch all documents, but that will significantly increase time it takes to generate ground truth data. You may want to at least start with a subset. -* `kgfile`: An existing RAGAS knowledge base JSON file, which is usually `ground_truth_kg.json`. You may want to specify this if you already created a knowledge base and just want to tweak the question generation steps. -* `groundtruthfile`: The file to write the generated ground truth answwers. By default, this is `evals/ground_truth.jsonl`. - -🕰️ This may take a long time, possibly several hours, depending on the size of the search index. - -Review the generated data in `evals/ground_truth.jsonl` after running that script, removing any question/answer pairs that don't seem like realistic user input. - -## Run bulk evaluation - -Review the configuration in `evals/evaluate_config.json` to ensure that everything is correctly setup. You may want to adjust the metrics used. See [the ai-rag-chat-evaluator README](https://github.com/Azure-Samples/ai-rag-chat-evaluator) for more information on the available metrics. - -By default, the evaluation script will evaluate every question in the ground truth data. -Run the evaluation script by running the following command: - -```bash -python evals/evaluate.py -``` - -The options are: - -* `numquestions`: The number of questions to evaluate. By default, this is all questions in the ground truth data. -* `resultsdir`: The directory to write the evaluation results. By default, this is a timestamped folder in `evals/results`. This option can also be specified in `evaluate_config.json`. -* `targeturl`: The URL of the running application to evaluate. By default, this is `http://localhost:50505`. This option can also be specified in `evaluate_config.json`. - -🕰️ This may take a long time, possibly several hours, depending on the number of ground truth questions, the TPM capacity of the evaluation model, and the number of LLM-based metrics requested. - -## Review the evaluation results - -The evaluation script will output a summary of the evaluation results, inside the `evals/results` directory. - -You can see a summary of results across all evaluation runs by running the following command: - -```bash -python -m evaltools summary evals/results -``` - -Compare answers to the ground truth by running the following command: - -```bash -python -m evaltools diff evals/results/baseline/ -``` - -Compare answers across two runs by running the following command: - -```bash -python -m evaltools diff evals/results/baseline/ evals/results/SECONDRUNHERE -``` - -## Run bulk evaluation on a PR - -This repository includes a GitHub Action workflow `evaluate.yaml` that can be used to run the evaluation on the changes in a PR. - -In order for the workflow to run successfully, you must first set up [continuous integration](./azd.md#github-actions) for the repository. - -To run the evaluation on the changes in a PR, a repository member can post a `/evaluate` comment to the PR. This will trigger the evaluation workflow to run the evaluation on the PR changes and will post the results to the PR. - -## Evaluate multimodal RAG answers - -The repository also includes an `evaluate_config_multimodal.json` file specifically for evaluating multimodal RAG answers. This configuration uses a different ground truth file, `ground_truth_multimodal.jsonl`, which includes questions based off the sample data that require both text and image sources to answer. - -Note that the "groundedness" evaluator is not reliable for multimodal RAG, since it does not currently incorporate the image sources. We still include it in the metrics, but the more reliable metrics are "relevance" and "citations matched". diff --git a/docs/keyvault-setup.md b/docs/keyvault-setup.md new file mode 100644 index 0000000000..e2fa9f9727 --- /dev/null +++ b/docs/keyvault-setup.md @@ -0,0 +1,246 @@ +# Azure Key Vault Setup Guide + +This guide explains how to set up Azure Key Vault for secret management in the AI Master Engineer application. + +## Overview + +Azure Key Vault provides secure storage for application secrets such as: +- Bot Framework credentials (`MICROSOFT_APP_ID`, `MICROSOFT_APP_PASSWORD`) +- Azure service keys (`AZURE_SEARCH_KEY`, `AZURE_OPENAI_API_KEY`) +- Web search API keys (`SERPER_API_KEY`, `FIRECRAWL_API_KEY`, etc.) + +## Prerequisites + +- Azure subscription with appropriate permissions +- Azure CLI or Azure Portal access +- Application deployed (or ready to deploy) + +## Setup Steps + +### 1. Enable Key Vault in Bicep Deployment + +Update `infra/main.parameters.json` or pass parameter during deployment: + +```json +{ + "enableKeyVault": { + "value": true + }, + "keyVaultName": { + "value": "" // Optional: will be auto-generated if empty + } +} +``` + +Or use `azd` command: + +```bash +azd up --parameter enableKeyVault=true +``` + +### 2. Deploy Infrastructure + +The Key Vault will be created automatically with the deployment: + +```bash +azd up +``` + +### 3. Grant Access to App Services + +After deployment, you need to grant the App Service Managed Identities access to Key Vault. + +#### Option A: Using Azure Portal + +1. Navigate to your Key Vault in Azure Portal +2. Go to **Access policies** → **Add access policy** +3. For each App Service (backend and agents): + - **Select principal**: Search for the App Service name + - **Secret permissions**: Select `Get` and `List` + - Click **Add** +4. Click **Save** + +#### Option B: Using Azure CLI + +```bash +# Get Key Vault name +KEY_VAULT_NAME=$(az keyvault list --query "[?contains(name, 'kv-')].name" -o tsv | head -n 1) + +# Get App Service principal IDs +BACKEND_PRINCIPAL_ID=$(az webapp identity show --name --resource-group --query principalId -o tsv) +AGENTS_PRINCIPAL_ID=$(az webapp identity show --name --resource-group --query principalId -o tsv) + +# Grant access +az keyvault set-policy \ + --name $KEY_VAULT_NAME \ + --object-id $BACKEND_PRINCIPAL_ID \ + --secret-permissions get list + +az keyvault set-policy \ + --name $KEY_VAULT_NAME \ + --object-id $AGENTS_PRINCIPAL_ID \ + --secret-permissions get list +``` + +#### Option C: Using PowerShell Script + +```powershell +# Get Key Vault name +$keyVaultName = (Get-AzKeyVault -ResourceGroupName "").VaultName + +# Get App Service principal IDs +$backendPrincipalId = (Get-AzWebApp -ResourceGroupName "" -Name "").Identity.PrincipalId +$agentsPrincipalId = (Get-AzWebApp -ResourceGroupName "" -Name "").Identity.PrincipalId + +# Grant access +Set-AzKeyVaultAccessPolicy ` + -VaultName $keyVaultName ` + -ObjectId $backendPrincipalId ` + -PermissionsToSecrets Get,List + +Set-AzKeyVaultAccessPolicy ` + -VaultName $keyVaultName ` + -ObjectId $agentsPrincipalId ` + -PermissionsToSecrets Get,List +``` + +### 4. Store Secrets in Key Vault + +#### Using Azure Portal + +1. Navigate to Key Vault → **Secrets** → **Generate/Import** +2. Create secrets with these names: + - `MICROSOFT-APP-ID` + - `MICROSOFT-APP-PASSWORD` + - `AZURE-SEARCH-KEY` + - `AZURE-OPENAI-API-KEY` + - `AZURE-CLIENT-SECRET` + - `SERPER-API-KEY` (optional) + - `FIRECRAWL-API-KEY` (optional) + - `COHERE-API-KEY` (optional) + - `DEEPSEEK-API-KEY` (optional) + +#### Using Azure CLI + +```bash +KEY_VAULT_NAME="" + +az keyvault secret set \ + --vault-name $KEY_VAULT_NAME \ + --name "MICROSOFT-APP-ID" \ + --value "" + +az keyvault secret set \ + --vault-name $KEY_VAULT_NAME \ + --name "MICROSOFT-APP-PASSWORD" \ + --value "" + +az keyvault secret set \ + --vault-name $KEY_VAULT_NAME \ + --name "AZURE-SEARCH-KEY" \ + --value "" + +# ... repeat for other secrets +``` + +#### Using PowerShell + +```powershell +$keyVaultName = "" + +Set-AzKeyVaultSecret ` + -VaultName $keyVaultName ` + -Name "MICROSOFT-APP-ID" ` + -SecretValue (ConvertTo-SecureString "" -AsPlainText -Force) + +Set-AzKeyVaultSecret ` + -VaultName $keyVaultName ` + -Name "MICROSOFT-APP-PASSWORD" ` + -SecretValue (ConvertTo-SecureString "" -AsPlainText -Force) + +# ... repeat for other secrets +``` + +### 5. Update Application Code + +The application code already supports Key Vault. It will automatically: +1. Try to read from Key Vault using Managed Identity +2. Fall back to environment variables if Key Vault is not available + +The Key Vault URL is automatically set in App Service configuration as `AZURE_KEY_VAULT_ENDPOINT`. + +### 6. Verify Secret Access + +#### Test from App Service Logs + +Check Application Insights or App Service logs for: +- "Key Vault client initialized" (success) +- "Failed to initialize Key Vault client" (fallback to env vars) + +#### Test via API + +```bash +# Check backend health (should show healthy status) +curl https://.azurewebsites.net/health + +# Check agents health (should show healthy status) +curl https://.azurewebsites.net/api/health +``` + +## Secret Naming Convention + +Key Vault secrets use **hyphens** (`-`) instead of underscores (`_`): +- Environment variable: `MICROSOFT_APP_ID` +- Key Vault secret: `MICROSOFT-APP-ID` + +The code automatically converts between these formats. + +## Troubleshooting + +### Secrets not found + +1. **Check access policies**: Ensure App Service Managed Identity has `Get` and `List` permissions +2. **Check secret names**: Use hyphens, not underscores +3. **Check Key Vault URL**: Verify `AZURE_KEY_VAULT_ENDPOINT` is set in App Service configuration +4. **Check logs**: Application will log warnings if Key Vault access fails + +### Access denied + +1. **Verify Managed Identity**: Ensure App Service has System Assigned Managed Identity enabled +2. **Verify permissions**: Check Key Vault access policies include the App Service principal ID +3. **Wait for propagation**: Access policy changes may take a few minutes to propagate + +### Fallback to environment variables + +If Key Vault is not configured or unavailable, the application will automatically fall back to environment variables. This is expected behavior for: +- Local development +- When Key Vault is disabled +- When Key Vault is temporarily unavailable + +## Best Practices + +1. **Enable soft delete**: Already enabled by default (7 days retention) +2. **Enable purge protection**: For production (currently disabled by default) +3. **Rotate secrets regularly**: Update secrets in Key Vault, no code changes needed +4. **Monitor access**: Use Key Vault access logs to track secret access +5. **Use separate Key Vaults**: Consider separate Key Vaults for dev/staging/production + +## Security Considerations + +- **Network access**: Key Vault allows Azure services by default +- **Private endpoints**: Can be configured for private network access +- **Access policies**: Use least privilege (only `Get` and `List` for secrets) +- **Audit logging**: Enable diagnostic logs for Key Vault access + +## Next Steps + +After Key Vault is set up: +1. Remove secrets from App Service configuration (optional, but recommended) +2. Test application functionality +3. Monitor Key Vault access logs +4. Set up secret rotation policies (if applicable) + + + + + diff --git a/docs/nomic-embeddings-implementation.md b/docs/nomic-embeddings-implementation.md new file mode 100644 index 0000000000..c9865a5409 --- /dev/null +++ b/docs/nomic-embeddings-implementation.md @@ -0,0 +1,285 @@ +# NOMIC Embeddings Implementation + +## Overview + +NOMIC embeddings have been fully implemented in the AI Master Engineer application. This implementation supports multiple NOMIC models (text, code, and vision) with intelligent routing based on content analysis. + +## Implementation Summary + +### 1. NOMIC Embeddings Client (`app/backend/prepdocslib/nomic_embeddings.py`) + +**Features:** +- Support for multiple NOMIC models: + - `nomic-embed-text-v1.5`: General text embeddings + - `nomic-embed-code-v1`: Code-specific embeddings + - `nomic-embed-vision-v1.5`: Multimodal (text + image) embeddings +- Dual access modes: + - **API Mode**: Uses NOMIC API endpoint (default) + - **SDK Mode**: Uses NOMIC Python SDK (optional, for local inference) +- Batch processing with configurable batch size +- Retry logic with exponential backoff +- Compatible interface with existing embedding services + +**Key Methods:** +- `create_embedding(text)`: Single text embedding +- `create_embeddings(texts)`: Batch embeddings +- `get_embedding_dimensions()`: Returns 768 (standard NOMIC dimension) + +### 2. Enhanced Embedding Router (`app/backend/services/embedding_router.py`) + +**New Features:** +- **Code Detection**: Detects code-heavy content using: + - File extension matching (`.py`, `.js`, `.java`, `.go`, etc.) + - Code pattern detection (keywords, syntax patterns) + - Structural indicators (brackets, semicolons, etc.) +- **Multimodal Detection**: Routes high image-density documents to NOMIC Vision +- **Intelligent Routing**: + ``` + Code-heavy content (score >= 15.0 or code file) → NOMIC Embed Code + High image density (>= 15%) → NOMIC Embed Vision + Technical/Patent content → PatentSBERTa + General content → Baseline (Azure OpenAI) + ``` + +**Routing Logic:** +1. Check for explicit metadata routing hints +2. Detect code-heavy content → NOMIC Code +3. Detect high image density → NOMIC Vision +4. Detect patent/technical content → PatentSBERTa +5. Default → Baseline (Azure OpenAI) + +### 3. Configuration (`app/backend/config.py`) + +**New Environment Variables:** +```python +NOMIC_API_KEY # NOMIC API key (required for API mode) +NOMIC_ENDPOINT # Optional custom endpoint URL +NOMIC_USE_SDK # Use Python SDK instead of API (default: false) +NOMIC_INFERENCE_MODE # local or remote (SDK only, default: remote) +ENABLE_NOMIC_EMBEDDINGS # Feature flag to enable NOMIC (default: false) +``` + +### 4. Document Ingestion Integration (`app/backend/prepdocs.py`) + +**Changes:** +- Added `NOMIC` to `OpenAIHost` enum +- Updated `setup_embeddings_service()` to support NOMIC +- Added NOMIC parameters to embedding service setup +- Environment variable support for NOMIC configuration + +### 5. Search Manager Compatibility (`app/backend/prepdocslib/searchmanager.py`) + +**Changes:** +- Enhanced embedding dimension detection to support NOMIC +- Added fallback for direct `embedding_dimensions` attribute + +## Usage + +### 1. Basic Setup + +**Using NOMIC API (Recommended):** +```bash +export NOMIC_API_KEY="your-api-key" +export OPENAI_HOST="nomic" +export NOMIC_MODEL="nomic-embed-text-v1.5" # Optional, defaults to text +``` + +**Using NOMIC SDK (Local Inference):** +```bash +export NOMIC_USE_SDK="true" +export NOMIC_INFERENCE_MODE="local" +pip install nomic # Install SDK +``` + +### 2. Automatic Routing (Recommended) + +The embedding router automatically selects the best model: + +```python +from services.embedding_router import EmbeddingRouter + +router = EmbeddingRouter( + baseline_deployment="text-embedding-3-large", + nomic_api_key=os.getenv("NOMIC_API_KEY"), + nomic_endpoint=os.getenv("NOMIC_ENDPOINT"), # Optional +) + +# Automatically routes based on content +model = router.select_model( + content="def hello_world():\n print('Hello')", + content_type=".py", + metadata={"image_count": 0} +) +# Returns: EmbeddingModel.NOMIC + +# Get routing decision details +info = router.get_routing_decision_info(content, content_type, metadata) +# Returns detailed analysis and selected model +``` + +### 3. Manual Selection + +**For Code Documents:** +```python +from prepdocslib.nomic_embeddings import create_nomic_code_embeddings + +embeddings = create_nomic_code_embeddings( + api_key=os.getenv("NOMIC_API_KEY") +) + +result = await embeddings.create_embeddings([ + "def fibonacci(n):\n if n <= 1:\n return n\n return fibonacci(n-1) + fibonacci(n-2)" +]) +``` + +**For Multimodal Content:** +```python +from prepdocslib.nomic_embeddings import create_nomic_vision_embeddings + +embeddings = create_nomic_vision_embeddings( + api_key=os.getenv("NOMIC_API_KEY") +) + +result = await embeddings.create_embeddings([ + "This diagram shows the system architecture with three layers..." +]) +``` + +### 4. Document Ingestion + +**Using NOMIC for all embeddings:** +```bash +export OPENAI_HOST="nomic" +export NOMIC_API_KEY="your-api-key" +python prepdocs.py data/ +``` + +**Using automatic routing (requires embedding router integration):** +- Set up embedding router with NOMIC credentials +- Router will automatically select NOMIC for code-heavy or multimodal content +- PatentSBERTa for technical/patent content +- Baseline for general content + +## Configuration Examples + +### Example 1: Code-Heavy Repository +```bash +# Router will automatically use NOMIC Code for .py, .js, etc. +export NOMIC_API_KEY="your-key" +export ENABLE_NOMIC_EMBEDDINGS="true" +# Router configured in app.py +``` + +### Example 2: Cost Optimization +```bash +# Use NOMIC for general content instead of Azure OpenAI +export OPENAI_HOST="nomic" +export NOMIC_API_KEY="your-key" +export NOMIC_MODEL="nomic-embed-text-v1.5" +``` + +### Example 3: Multimodal Documents +```bash +# Router will use NOMIC Vision for high image-density docs +export NOMIC_API_KEY="your-key" +export ENABLE_NOMIC_EMBEDDINGS="true" +# Documents with >15% image density will use NOMIC Vision +``` + +## File Structure + +``` +app/backend/ +├── prepdocslib/ +│ └── nomic_embeddings.py # NOMIC embeddings client +├── services/ +│ └── embedding_router.py # Enhanced with NOMIC routing +├── config.py # NOMIC configuration +└── prepdocs.py # Document ingestion with NOMIC support +``` + +## Testing + +### Test NOMIC Embeddings Client +```python +import asyncio +from prepdocslib.nomic_embeddings import NomicEmbeddings + +async def test(): + embeddings = NomicEmbeddings( + model="nomic-embed-text-v1.5", + api_key="your-api-key" + ) + result = await embeddings.create_embedding("Hello world") + print(f"Embedding dimensions: {len(result)}") # Should be 768 + +asyncio.run(test()) +``` + +### Test Routing Logic +```python +from services.embedding_router import EmbeddingRouter + +router = EmbeddingRouter( + baseline_deployment="baseline", + nomic_api_key="your-key" +) + +# Test code detection +result = router.select_model( + content="def hello():\n return 'world'", + content_type=".py" +) +assert result == EmbeddingModel.NOMIC +``` + +## Benefits + +1. **Code Search**: NOMIC Embed Code provides superior code understanding +2. **Cost Optimization**: Open-source alternative to Azure OpenAI +3. **Multimodal Support**: Unified text+image embeddings +4. **Automatic Routing**: Intelligent model selection based on content +5. **Flexible Deployment**: API or SDK modes + +## Next Steps + +1. **Test with real code repositories**: Index code documentation and test retrieval +2. **Benchmark performance**: Compare NOMIC vs Azure OpenAI for different content types +3. **Monitor costs**: Track cost savings when using NOMIC for general content +4. **Fine-tune thresholds**: Adjust routing thresholds based on real-world performance + +## Troubleshooting + +### Issue: "NOMIC SDK not installed" +**Solution:** Install SDK or use API mode +```bash +pip install nomic # For SDK mode +# OR +export NOMIC_USE_SDK="false" # Use API mode (default) +``` + +### Issue: "NOMIC API error 401" +**Solution:** Check API key +```bash +export NOMIC_API_KEY="your-valid-api-key" +``` + +### Issue: Routing not working +**Solution:** Ensure NOMIC is configured in embedding router +```python +router = EmbeddingRouter( + baseline_deployment="...", + nomic_api_key=os.getenv("NOMIC_API_KEY") # Required for routing +) +``` + +## References + +- [NOMIC Embeddings Documentation](https://docs.nomic.ai/atlas/capabilities/embeddings) +- [NOMIC Python SDK](https://github.com/nomic-ai/nomic) +- [Use Cases Document](./nomic-embeddings-use-cases.md) + + + + + diff --git a/docs/nomic-embeddings-use-cases.md b/docs/nomic-embeddings-use-cases.md new file mode 100644 index 0000000000..41ba465357 --- /dev/null +++ b/docs/nomic-embeddings-use-cases.md @@ -0,0 +1,337 @@ +# NOMIC Embeddings Use Cases + +## Overview + +NOMIC embeddings are open-source, high-performance embedding models that can be used as an alternative to Azure OpenAI embeddings. Currently, NOMIC is listed in the embedding router but not yet implemented. This document outlines where NOMIC embeddings would be most beneficial in the AI Master Engineer application. + +--- + +## Current Embedding Architecture + +The application currently supports: + +1. **Baseline (Azure OpenAI)** - `text-embedding-3-large` or similar + - General-purpose embeddings + - Used for most content by default + - High quality, but requires Azure OpenAI service + +2. **PatentSBERTa** - Domain-specific embeddings + - Optimized for technical/patent content + - Used when content analysis detects patent/technical indicators + - Better semantic understanding for engineering/patent documents + +3. **NOMIC** - Listed but not implemented + - Open-source alternative + - Multiple variants (text, code, multimodal) + +--- + +## Where NOMIC Embeddings Would Be Helpful + +### 1. **Code-Heavy Documents** ⭐⭐⭐⭐⭐ +**Best Use Case** + +**Scenario:** +- Software documentation (API docs, code comments, technical specifications) +- Code repositories being indexed +- Programming tutorials or guides +- Configuration files with code snippets + +**Why NOMIC:** +- NOMIC Embed Code provides state-of-the-art code embeddings +- Supports multiple languages: Python, JavaScript, Java, Go, PHP, Ruby +- Better semantic understanding of code structure than general-purpose embeddings +- Superior for code search and code-to-text retrieval + +**Implementation:** +```python +# In embedding_router.py, detect code-heavy content +if code_score >= threshold: + return EmbeddingModel.NOMIC # Use NOMIC Embed Code +``` + +**Example Documents:** +- `api-reference.md` with code examples +- `README.md` with installation scripts +- `config.yaml` or `settings.json` files +- Software architecture documentation + +--- + +### 2. **Multimodal Content (Text + Images)** ⭐⭐⭐⭐ +**Strong Use Case** + +**Scenario:** +- Documents with diagrams, charts, and images +- Technical manuals with figures +- Presentations (PPTX) with embedded images +- PDFs with mixed text and visual content + +**Why NOMIC:** +- NOMIC Embed Vision v1.5 supports multimodal embeddings +- Aligns text and image data into unified embedding space +- Enables image retrieval using text queries +- Better than baseline for documents with high image density + +**Current Limitation:** +- Application already has Azure Vision for image embeddings +- NOMIC would provide unified text+image embeddings in one model + +**Implementation:** +```python +# In embedding_router.py +if metadata_analysis["image_density"] > 15.0: # High image density + return EmbeddingModel.NOMIC # Use NOMIC Embed Vision +``` + +**Example Documents:** +- Engineering diagrams with annotations +- Technical manuals with screenshots +- Architecture diagrams with descriptions +- Product catalogs with images + +--- + +### 3. **Cost Optimization** ⭐⭐⭐⭐ +**Practical Use Case** + +**Scenario:** +- Large-scale document ingestion +- High-volume indexing operations +- Budget constraints +- Open-source preference + +**Why NOMIC:** +- Open-source (no per-token costs) +- Self-hosted or API-based (flexible pricing) +- Good performance-to-cost ratio +- Suitable for general-purpose content + +**When to Use:** +- Non-critical content that doesn't need Azure OpenAI quality +- Bulk indexing where cost per document matters +- Development/testing environments +- Content that doesn't fit patent/technical categories + +**Implementation:** +```python +# Route general content to NOMIC instead of Azure OpenAI +if not is_technical and not is_patent: + return EmbeddingModel.NOMIC # Cost-effective alternative +``` + +--- + +### 4. **General-Purpose Content (Non-Technical)** ⭐⭐⭐ +**Moderate Use Case** + +**Scenario:** +- Business documents (emails, reports, memos) +- Marketing materials +- Legal documents (non-patent) +- General knowledge articles +- News articles or blog posts + +**Why NOMIC:** +- Good general-purpose performance +- Comparable to Azure OpenAI for non-specialized content +- Lower cost option +- Open-source flexibility + +**When to Use:** +- Documents that don't benefit from PatentSBERTa specialization +- Content where Azure OpenAI is overkill +- General knowledge base content + +**Example Documents:** +- Company policies and procedures +- HR documentation +- Marketing brochures +- General FAQs + +--- + +### 5. **Fallback/Redundancy** ⭐⭐⭐ +**Reliability Use Case** + +**Scenario:** +- Azure OpenAI service unavailable +- Rate limiting issues +- Regional availability constraints +- Multi-cloud deployments + +**Why NOMIC:** +- Provides alternative embedding source +- Reduces dependency on single provider +- Can be self-hosted for complete control +- Good backup option + +**Implementation:** +```python +# Fallback logic +try: + return EmbeddingModel.BASELINE # Azure OpenAI +except ServiceUnavailable: + return EmbeddingModel.NOMIC # Fallback +``` + +--- + +### 6. **Data Visualization and Exploration** ⭐⭐ +**Niche Use Case** + +**Scenario:** +- Large document collections needing clustering +- Topic modeling and discovery +- Anomaly detection in documents +- Similarity analysis across corpus + +**Why NOMIC:** +- NOMIC Atlas platform uses embeddings for visualization +- Good for creating interactive document maps +- Useful for exploratory data analysis +- Can help identify document relationships + +**When to Use:** +- Initial document analysis +- Understanding document corpus structure +- Finding similar documents across categories +- Quality assurance during ingestion + +--- + +## Routing Logic Recommendations + +### Current Routing (PatentSBERTa) +``` +Technical/Patent Content → PatentSBERTa +Everything Else → Baseline (Azure OpenAI) +``` + +### Recommended Routing with NOMIC +``` +Technical/Patent Content → PatentSBERTa +Code-Heavy Content → NOMIC (Code) +High Image Density → NOMIC (Vision) or Baseline +General Content → NOMIC (Text) or Baseline +Fallback → NOMIC (Text) +``` + +### Implementation Priority + +1. **High Priority:** + - Code detection and routing to NOMIC Embed Code + - Fallback mechanism when Azure OpenAI unavailable + +2. **Medium Priority:** + - Multimodal routing for high image-density documents + - Cost optimization for general content + +3. **Low Priority:** + - Data visualization features + - Advanced clustering analysis + +--- + +## Content Detection Heuristics for NOMIC + +### Code Detection (for NOMIC Embed Code) +```python +CODE_KEYWORDS = { + "function", "class", "def ", "import", "from", "return", + "public", "private", "static", "void", "const", "let", "var", + "if __name__", "namespace", "package", "interface", "extends" +} + +CODE_FILE_EXTENSIONS = { + ".py", ".js", ".java", ".go", ".php", ".rb", ".cpp", ".c", + ".ts", ".tsx", ".jsx", ".sql", ".sh", ".yaml", ".yml", ".json" +} + +def detect_code_content(content: str, metadata: dict) -> bool: + # Check file extension + if metadata.get("file_type") in CODE_FILE_EXTENSIONS: + return True + + # Check for code patterns + code_density = sum(1 for pattern in CODE_KEYWORDS if pattern in content.lower()) + return code_density > threshold +``` + +### Multimodal Detection (for NOMIC Embed Vision) +```python +def detect_multimodal_content(metadata: dict) -> bool: + image_density = metadata.get("image_density", 0) + page_count = metadata.get("page_count", 1) + + # High image density (>15% images per page) + if image_density > 15.0: + return True + + # Presentation files (typically image-heavy) + if metadata.get("file_type") in [".pptx", ".ppt"]: + return True + + return False +``` + +--- + +## Comparison Matrix + +| Use Case | Baseline (Azure OpenAI) | PatentSBERTa | NOMIC | +|----------|------------------------|--------------|-------| +| **General Text** | ✅ Excellent | ⚠️ Overkill | ✅ Good | +| **Technical/Patents** | ✅ Good | ✅✅ Excellent | ⚠️ Good | +| **Code** | ⚠️ Fair | ❌ Not Designed | ✅✅ Excellent | +| **Multimodal** | ⚠️ Separate Service | ❌ Not Designed | ✅✅ Excellent | +| **Cost** | 💰💰 Paid | 💰💰💰 Custom | ✅ Free/Open-source | +| **Latency** | ✅ Fast | ⚠️ Varies | ✅ Fast | +| **Availability** | ✅ Azure | ⚠️ Custom | ✅ Flexible | + +--- + +## Implementation Steps + +### Phase 1: Code Detection & Routing +1. Add code detection heuristics to `embedding_router.py` +2. Implement NOMIC Embed Code client +3. Route code-heavy content to NOMIC +4. Test with code documentation + +### Phase 2: Multimodal Support +1. Implement NOMIC Embed Vision client +2. Add multimodal detection logic +3. Route high image-density documents +4. Test with diagram-heavy documents + +### Phase 3: Cost Optimization +1. Add general content routing to NOMIC +2. Implement fallback mechanism +3. Add configuration flags +4. Monitor cost savings + +--- + +## Summary + +**NOMIC embeddings would be most helpful for:** + +1. **Code-heavy documents** - Best performance for code search and retrieval +2. **Cost optimization** - Open-source alternative for general content +3. **Multimodal content** - Unified text+image embeddings +4. **Fallback/redundancy** - Alternative when Azure OpenAI unavailable +5. **General content** - Good performance for non-specialized documents + +**Priority Implementation:** +- Start with code detection and NOMIC Embed Code integration +- Add fallback mechanism for reliability +- Consider multimodal support if image-heavy documents are common +- Use for cost optimization in non-critical content + +The current embedding router already has the infrastructure to support NOMIC - it just needs the actual NOMIC client implementation and routing logic. + + + + + diff --git a/docs/other_samples.md b/docs/other_samples.md deleted file mode 100644 index 09c4b234a1..0000000000 --- a/docs/other_samples.md +++ /dev/null @@ -1,85 +0,0 @@ -# RAG chat: Alternative RAG chat samples - -There are an increasingly large number of ways to build RAG chat apps. - -* [Most similar to this repo](#most-similar-to-this-repo) -* [azurechat](#azurechat) -* [sample-app-aoai-chatGPT](#sample-app-aoai-chatgpt) - -## Most similar to this repo - -Inspired by this repo, there are similar RAG chat apps for other languages: - -* [**JavaScript**](https://aka.ms/azai/js/code) -* [**.NET**](https://aka.ms/azai/net/code) -* [**Java**](https://aka.ms/azai/java/code) - -They do not all support the same features as this repo, but they provide a good starting point for building a RAG chat app in your preferred language. - -## azurechat - -Another popular sample is the Azure Chat Solution Accelerator: -[https://github.com/microsoft/azurechat](https://github.com/microsoft/azurechat) - -AzureChat deploys a private chat app with a ChatGPT-like UX on Azure, with built‑in capabilities for chatting over internal data and files and optional extensions. - -Key differences versus this repository: - -* **Technology stack**: AzureChat uses a full JavaScript/TypeScript stack with a Node.js backend; this repo uses Python (Quart) for backend services. -* **Use case emphasis**: AzureChat offers more features around user personalization, while this repo offers more features needed for enterprise scenarios like data ACLs and evaluation. - -Feature comparison: - -| Feature | azure-search-openai-demo | azurechat | -| --- | --- | --- | -| Vector support | ✅ Yes | ✅ Yes | -| Data ingestion | ✅ Yes ([Many formats](data_ingestion.md#supported-document-formats)) | ✅ Yes | -| Persistent chat history | ✅ Yes | ✅ Yes | -| Multimodal | ✅ Yes | ✅ Yes | -| Voice/Speech I/O | ✅ Yes | ✅ Yes | -| File upload | ✅ Yes | ✅ Yes | -| Auth + ACL | ✅ Yes (Enterprise-focused) | ✅ Yes (Personal-focused) | -| Access control | ✅ Yes (Document-level) | ❌ Limited | - -Technology comparison: - -| Tech | azure-search-openai-demo | azurechat | -| --- | --- | --- | -| Frontend | React (TypeScript) | React (TypeScript) | -| Backend | Python (Quart) | Node.js (TypeScript) | -| Database | Azure AI Search | Azure AI Search | -| Deployment | Azure Developer CLI (azd) | Azure Developer CLI (azd) | - -## sample-app-aoai-chatGPT - -Another popular repository for this use case is: -[https://github.com/Microsoft/sample-app-aoai-chatGPT/](https://github.com/Microsoft/sample-app-aoai-chatGPT/) - -That repository is designed for use by customers using Azure OpenAI studio and Azure Portal for setup. It also includes `azd` support for folks who want to deploy it completely from scratch. - -The primary differences: - -* This repository includes multiple RAG (retrieval-augmented generation) approaches that chain the results of multiple API calls (to Azure OpenAI and ACS) together in different ways. The other repository uses only the built-in data sources option for the ChatCompletions API, which uses a RAG approach on the specified ACS index. That should work for most uses, but if you needed more flexibility, this sample may be a better option. -* This repository is also a bit more experimental in other ways, since it's not tied to the Azure OpenAI Studio like the other repository. - -Feature comparison: - -| Feature | azure-search-openai-demo | sample-app-aoai-chatGPT | -| --- | --- | --- | -| Vector support | ✅ Yes | ✅ Yes | -| Data ingestion | ✅ Yes ([Many formats](data_ingestion.md#supported-document-formats)) | ✅ Yes ([Many formats](https://learn.microsoft.com/azure/ai-services/openai/concepts/use-your-data?tabs=ai-search#data-formats-and-file-types)) | -| Persistent chat history | ✅ Yes | ✅ Yes | -| User feedback | ❌ No | ✅ Yes | -| GPT-4-vision | ✅ Yes | ❌ No | -| Auth + ACL | ✅ Yes | ✅ Yes | -| User upload | ✅ Yes | ❌ No | -| Speech I/O | ✅ Yes | ❌ No | - -Technology comparison: - -| Tech | azure-search-openai-demo | sample-app-aoai-chatGPT | -| --- | --- | --- | -| Frontend | React | React | -| Backend | Python (Quart) | Python (Quart) | -| Vector DB | Azure AI Search | Azure AI Search, CosmosDB Mongo vCore, ElasticSearch, Pinecone, AzureML | -| Deployment | Azure Developer CLI (azd) | Azure Portal, az, azd | diff --git a/docs/patentsberta-api-key-explanation.md b/docs/patentsberta-api-key-explanation.md new file mode 100644 index 0000000000..819b773809 --- /dev/null +++ b/docs/patentsberta-api-key-explanation.md @@ -0,0 +1,223 @@ +# PatentSBERTa API Key - Why It's Optional + +## How It Works + +The PatentSBERTa API key is **optional** because the service implements **conditional authentication**: + +### The Logic + +Looking at the service code (`custom-embedding-service/app.py`): + +```python +def api_key_auth(x_api_key: str | None = Header(default=None)): + """API key authentication dependency""" + if API_KEY and x_api_key != API_KEY: + raise HTTPException(status_code=401, detail="Invalid or missing API key") +``` + +**Key Point:** The check only fails if: +1. `API_KEY` is set in the service (from `PATENTSBERTA_API_KEY` environment variable) +2. AND the provided key doesn't match + +### Two Scenarios + +#### Scenario 1: API Key NOT Configured (Works Without Key) + +**Service Side:** +```python +# In custom-embedding-service/constants.py +API_KEY = os.getenv("PATENTSBERTA_API_KEY") # Returns None if not set +``` + +**What Happens:** +- `API_KEY` is `None` +- The check `if API_KEY and x_api_key != API_KEY:` evaluates to `False` (because `API_KEY` is `None`) +- **Authentication is bypassed** - service accepts requests without API key +- ✅ **Works without API key** + +**Client Side:** +```python +# In app/backend/prepdocslib/patentsberta_embeddings.py +headers = {'Content-Type': 'application/json'} +if self.api_key: # This is None, so header not added + headers['X-API-Key'] = self.api_key +``` + +#### Scenario 2: API Key IS Configured (Requires Key) + +**Service Side:** +```bash +# Set in service environment +export PATENTSBERTA_API_KEY="your-secret-key" +``` + +**What Happens:** +- `API_KEY` is set to `"your-secret-key"` +- The check `if API_KEY and x_api_key != API_KEY:` will enforce authentication +- Requests without matching key get **401 Unauthorized** +- ✅ **Requires API key** - client must provide matching key + +**Client Side:** +```bash +# Must set matching key +export PATENTSBERTA_API_KEY="your-secret-key" +``` + +```python +# Client includes header +headers = {'Content-Type': 'application/json', 'X-API-Key': 'your-secret-key'} +``` + +--- + +## Why This Design? + +This **conditional authentication** pattern is useful because: + +1. **Development/Testing**: You can run the service locally without authentication for easier testing +2. **Internal Networks**: If the service is behind a VPN/firewall, you might not need API key authentication +3. **Azure Managed Identity**: If using Azure authentication, you might not need API keys +4. **Production Security**: You can enable authentication by simply setting the environment variable + +--- + +## When Does It Work Without API Key? + +✅ **Works without API key** when: +- `PATENTSBERTA_API_KEY` is **NOT set** in the service environment +- Service is behind a VPN/firewall (network-level security) +- Service uses Azure Managed Identity for authentication +- Development/testing environment + +❌ **Requires API key** when: +- `PATENTSBERTA_API_KEY` **IS set** in the service environment +- Service is publicly accessible and needs protection +- Production environment requiring authentication + +--- + +## How to Check Your Service Configuration + +### Check if Service Requires API Key + +**Option 1: Check Service Environment** +```bash +# If service is deployed, check environment variables +az containerapp show \ + --name your-patentsberta-service \ + --resource-group your-rg \ + --query "properties.template.containers[0].env" +``` + +**Option 2: Test the Endpoint** +```bash +# Try without API key +curl -X POST https://your-service.azurewebsites.net/embeddings \ + -H "Content-Type: application/json" \ + -d '{"texts": ["test"], "normalize": true}' + +# If you get 401, service requires API key +# If you get 200, service doesn't require API key +``` + +**Option 3: Check Service Logs** +```bash +# Look for authentication-related errors +az containerapp logs show \ + --name your-patentsberta-service \ + --resource-group your-rg \ + --tail 50 +``` + +--- + +## Configuration Examples + +### Example 1: Service Without API Key (No Authentication) + +**Service Deployment:** +```bash +# Don't set PATENTSBERTA_API_KEY when deploying +az containerapp create \ + --name patentsberta-service \ + --resource-group rg-ai-master-engineer \ + # ... other settings ... + # No PATENTSBERTA_API_KEY in environment variables +``` + +**Client Configuration:** +```bash +# Client doesn't need API key +export PATENTSBERTA_ENDPOINT="https://patentsberta-service.azurewebsites.net" +# No PATENTSBERTA_API_KEY needed +``` + +**Result:** ✅ Works without API key + +--- + +### Example 2: Service With API Key (Authentication Required) + +**Service Deployment:** +```bash +# Set API key when deploying service +API_KEY=$(openssl rand -base64 32) +az containerapp create \ + --name patentsberta-service \ + --resource-group rg-ai-master-engineer \ + --env-vars "PATENTSBERTA_API_KEY=$API_KEY" \ + # ... other settings ... +``` + +**Client Configuration:** +```bash +# Client MUST provide matching API key +export PATENTSBERTA_ENDPOINT="https://patentsberta-service.azurewebsites.net" +export PATENTSBERTA_API_KEY="$API_KEY" # Same key as service +``` + +**Result:** ✅ Works with API key (won't work without it) + +--- + +## Security Recommendations + +### For Development +```bash +# Optional - skip API key for easier development +# Service: Don't set PATENTSBERTA_API_KEY +# Client: Don't set PATENTSBERTA_API_KEY +``` + +### For Production +```bash +# Recommended - use API key for security +# Service: Set PATENTSBERTA_API_KEY +# Client: Set matching PATENTSBERTA_API_KEY +``` + +### For Internal Networks +```bash +# Optional - rely on network security +# Service: Don't set PATENTSBERTA_API_KEY (if behind VPN/firewall) +# Client: Don't set PATENTSBERTA_API_KEY +``` + +--- + +## Summary + +| Service `PATENTSBERTA_API_KEY` | Client `PATENTSBERTA_API_KEY` | Result | +|-------------------------------|-------------------------------|--------| +| Not set | Not set | ✅ Works (no authentication) | +| Not set | Set | ✅ Works (key ignored) | +| Set | Not set | ❌ 401 Unauthorized | +| Set | Set (matches) | ✅ Works | +| Set | Set (doesn't match) | ❌ 401 Unauthorized | + +**Bottom Line:** The API key is optional because the service only enforces authentication **if** the `PATENTSBERTA_API_KEY` environment variable is set on the service side. If it's not set, the service accepts requests without authentication. + + + + + diff --git a/docs/safety_evaluation.md b/docs/safety_evaluation.md deleted file mode 100644 index 811a2210a4..0000000000 --- a/docs/safety_evaluation.md +++ /dev/null @@ -1,77 +0,0 @@ -# Evaluating RAG answer safety - -When deploying a RAG app to production, you should evaluate the safety of the answers generated by the RAG flow. This is important to ensure that the answers are appropriate and do not contain any harmful or sensitive content. This project includes scripts that use Azure AI services to simulate an adversarial user and evaluate the safety of the answers generated in response to those adversarial queries. - -* [Deploy an Azure AI project](#deploy-an-azure-ai-project) -* [Simulate and evaluate adversarial users](#simulate-and-evaluate-adversarial-users) -* [Review the safety evaluation results](#review-the-safety-evaluation-results) - -## Deploy an Azure AI project - -In order to use the adversarial simulator and safety evaluators, you need an Azure AI project inside an Azure AI Hub. - -1. Run this command to tell `azd` to provision an Azure AI project and hub: - - ```shell - azd env set USE_AI_PROJECT true - ``` - -2. Then, run the following command to provision the project: - - ```shell - azd provision - ``` - -## Simulate and evaluate adversarial users - -Run the following command to simulate adversarial queries and evaluate the safety of the answers generated in response to those queries: - -```shell -python evals/safety_evaluation.py --target_url --max_simulations -``` - -* `--target_url`: The target URL for the callback. Default is `http://localhost:50505/chat`. -* `--max_simulations`: The maximum number of simulated user queries. Default is `200`. The higher the number, the longer the evaluation will take. The default of `200` simulations will take about 25 minutes to run, which includes both the time to generate the simulated data and the time to evaluate it. - -## Review the safety evaluation results - -That command will generate a report in `safety_results.json`. -Results will look like this: - -```json -{ - "hate_unfairness": { - "low_count": 20, - "score_total": 8, - "mean_score": 0.4, - "low_rate": 1.0 - }, - "sexual": { - "low_count": 20, - "score_total": 9, - "mean_score": 0.45, - "low_rate": 1.0 - }, - "violence": { - "low_count": 20, - "score_total": 9, - "mean_score": 0.45, - "low_rate": 1.0 - }, - "self_harm": { - "low_count": 20, - "score_total": 10, - "mean_score": 0.5, - "low_rate": 1.0 - } -} -``` - -The ideal score is `low_rate` of 1.0 and `mean_score` of 0.0. The `low_rate` indicates the fraction of answers that were reported as "Low" or "Very low" by an evaluator. The `mean_score` is the average score of all the answers, where 0 is a very safe answer and 7 is a very unsafe answer. - -## Resources - -To learn more about the Azure AI services used in this project, look through the script and reference the following documentation: - -* [Generate simulated data for evaluation](https://learn.microsoft.com/azure/ai-studio/how-to/develop/simulator-interaction-data) -* [Evaluate with the Azure AI Evaluation SDK](https://learn.microsoft.com/azure/ai-studio/how-to/develop/evaluate-sdk) diff --git a/docs/sharing_environments.md b/docs/sharing_environments.md deleted file mode 100644 index dcb7de4b31..0000000000 --- a/docs/sharing_environments.md +++ /dev/null @@ -1,11 +0,0 @@ -# RAG chat: Sharing deployment environments - -If you've deployed the RAG chat solution already following the steps in the [deployment guide](../README.md#deploying), you may want to share the environment with a colleague. -Either you or they can follow these steps: - -1. Install the [Azure CLI](https://learn.microsoft.com/cli/azure/install-azure-cli) -1. Run `azd init -t azure-search-openai-demo` or clone this repository. -1. Run `azd env refresh -e {environment name}` - They will need the azd environment name, subscription ID, and location to run this command. You can find those values in your `.azure/{env name}/.env` file. This will populate their azd environment's `.env` file with all the settings needed to run the app locally. -1. Set the environment variable `AZURE_PRINCIPAL_ID` either in that `.env` file or in the active shell to their Azure ID, which they can get with `az ad signed-in-user show`. -1. Run `./scripts/roles.ps1` or `.scripts/roles.sh` to assign all of the necessary roles to the user. If they do not have the necessary permission to create roles in the subscription, then you may need to run this script for them. Once the script runs, they should be able to run the app locally. diff --git a/infra/main.bicep b/infra/main.bicep index 146ba32ff4..7f1e3c7047 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -11,6 +11,7 @@ param location string param appServicePlanName string = '' // Set in main.parameters.json param backendServiceName string = '' // Set in main.parameters.json +param agentsServiceName string = '' // Set in main.parameters.json param resourceGroupName string = '' // Set in main.parameters.json param applicationInsightsDashboardName string = '' // Set in main.parameters.json @@ -51,7 +52,7 @@ param imageStorageContainerName string = 'images' param appServiceSkuName string // Set in main.parameters.json -@allowed(['azure', 'openai', 'azure_custom']) +@allowed(['azure', 'openai', 'azure_custom', 'patentsberta']) param openAiHost string // Set in main.parameters.json param isAzureOpenAiHost bool = startsWith(openAiHost, 'azure') param deployAzureOpenAi bool = openAiHost == 'azure' @@ -121,6 +122,11 @@ param openAiSkuName string = 'S0' param openAiApiKey string = '' param openAiApiOrganization string = '' +@secure() +param patentsbertaApiKey string = '' +@description('Immutable image tag for PatentsBERTa container (e.g., v1.0.0-20241201)') +param patentsbertaImageTag string = 'v1.0.0' + param documentIntelligenceServiceName string = '' // Set in main.parameters.json param documentIntelligenceResourceGroupName string = '' // Set in main.parameters.json @@ -291,6 +297,11 @@ param azureContainerAppsWorkloadProfile string @allowed(['appservice', 'containerapps']) param deploymentTarget string = 'appservice' +@description('Enable Key Vault for secret management') +param enableKeyVault bool = false +@description('Key Vault name (optional, will be generated if not provided)') +param keyVaultName string = '' + // RAG Configuration Parameters @description('Whether to use text embeddings for RAG search') param ragSearchTextEmbeddings bool = true @@ -372,6 +383,23 @@ module monitoring 'core/monitor/monitoring.bicep' = if (useApplicationInsights) } } +// Create Key Vault for secret management (optional) +module keyVault 'modules/keyvault.bicep' = if (enableKeyVault) { + name: 'keyvault' + scope: resourceGroup + params: { + name: !empty(keyVaultName) ? keyVaultName : '${abbrs.keyVaultVaults}${resourceToken}' + location: location + tags: tags + tenantId: tenantId + accessPolicies: [] // Will be updated after App Services are created + enableSoftDelete: true + softDeleteRetentionInDays: 7 + enablePurgeProtection: false + sku: 'standard' + } +} + module applicationInsightsDashboard 'backend-dashboard.bicep' = if (useApplicationInsights) { name: 'application-insights-dashboard' scope: resourceGroup @@ -483,9 +511,13 @@ var appEnvVariables = { module backend 'core/host/appservice.bicep' = if (deploymentTarget == 'appservice') { name: 'web' scope: resourceGroup + dependsOn: [ + enableKeyVault ? keyVault : null + ] params: { name: !empty(backendServiceName) ? backendServiceName : '${abbrs.webSitesAppService}backend-${resourceToken}' location: location + keyVaultName: enableKeyVault ? keyVault.outputs.name : '' tags: union(tags, { 'azd-service-name': 'backend' }) // Need to check deploymentTarget again due to https://github.com/Azure/bicep/issues/3990 appServicePlanId: deploymentTarget == 'appservice' ? appServicePlan.outputs.id : '' @@ -512,6 +544,77 @@ module backend 'core/host/appservice.bicep' = if (deploymentTarget == 'appservic } } +// App Service for the Agents service (Teams Bot) +var agentsEnvVariables = { + // Bot Framework Configuration + MICROSOFT_APP_ID: '' // Set via Azure Portal or Key Vault + MICROSOFT_APP_PASSWORD: '' // Set via Azure Portal or Key Vault + // Microsoft 365 Configuration + AZURE_TENANT_ID: tenantId + AZURE_CLIENT_ID: clientAppId // Use same client ID as backend + AZURE_CLIENT_SECRET: clientAppSecret // Use same client secret as backend + // Backend API Configuration - reference backend service + BACKEND_URL: deploymentTarget == 'appservice' + ? 'https://${backend.outputs.name}.azurewebsites.net' + : 'https://${acaBackend.outputs.name}.azurecontainerapps.io' + // Azure Services (inherit from backend) + AZURE_OPENAI_ENDPOINT: isAzureOpenAiHost && deployAzureOpenAi + ? 'https://${openAi.outputs.name}.openai.azure.com' + : '' + AZURE_OPENAI_API_KEY: azureOpenAiApiKey + AZURE_OPENAI_CHATGPT_DEPLOYMENT: chatGpt.deploymentName + AZURE_SEARCH_ENDPOINT: 'https://${searchService.outputs.name}.search.windows.net' + AZURE_SEARCH_KEY: '' // Set via Key Vault or use Managed Identity + AZURE_SEARCH_INDEX: searchIndexName + // Application Insights + APPLICATIONINSIGHTS_CONNECTION_STRING: useApplicationInsights + ? monitoring.outputs.applicationInsightsConnectionString + : '' + // Port configuration + PORT: '3978' + // Agent Settings + AGENT_NAME: 'RAG Assistant' + MAX_CONVERSATION_TURNS: '20' + ENABLE_TYPING_INDICATOR: 'true' + // Channel Settings + ENABLE_TEAMS: 'true' + ENABLE_COPILOT: 'true' + ENABLE_WEB_CHAT: 'true' + // CORS + ALLOWED_ORIGINS: join(['https://teams.microsoft.com', 'https://teams.microsoft.com/*'], ';') + RUNNING_IN_PRODUCTION: 'true' +} + +module agents 'core/host/appservice.bicep' = if (deploymentTarget == 'appservice') { + name: 'agents' + scope: resourceGroup + dependsOn: [ + backend + enableKeyVault ? keyVault : null + ] + params: { + name: !empty(agentsServiceName) ? agentsServiceName : '${abbrs.webSitesAppService}agents-${resourceToken}' + location: location + tags: union(tags, { 'azd-service-name': 'agents' }) + appServicePlanId: deploymentTarget == 'appservice' ? appServicePlan.outputs.id : '' + runtimeName: 'python' + runtimeVersion: '3.11' + appCommandLine: 'python main.py' + scmDoBuildDuringDeployment: true + managedIdentity: true + keyVaultName: enableKeyVault ? keyVault.outputs.name : '' + virtualNetworkSubnetId: usePrivateEndpoint ? isolation.outputs.appSubnetId : '' + publicNetworkAccess: publicNetworkAccess + allowedOrigins: ['https://teams.microsoft.com'] + // Disable Easy Auth for Agents (uses Bot Framework auth) + enableUnauthenticatedAccess: true + disableAppServicesAuthentication: true + use32BitWorkerProcess: appServiceSkuName == 'F1' + alwaysOn: appServiceSkuName != 'F1' + appSettings: agentsEnvVariables + } +} + // Azure container apps resources (Only deployed if deploymentTarget is 'containerapps') // User-assigned identity for pulling images from ACR @@ -540,6 +643,23 @@ module containerApps 'core/host/container-apps.bicep' = if (deploymentTarget == } } +// PatentsBERTa Container App for custom embeddings +module patentsbertaService 'modules/patentsberta.bicep' = if (deploymentTarget == 'containerapps') { + name: 'patentsberta-service' + scope: resourceGroup + dependsOn: [ + containerApps + ] + params: { + environmentName: environmentName + location: location + containerAppsEnvironmentName: containerApps.outputs.environmentName + containerRegistryName: containerApps.outputs.registryName + patentsbertaApiKey: patentsbertaApiKey + imageTag: patentsbertaImageTag + } +} + // Container Apps for the web application (Python Quart app with JS frontend) module acaBackend 'core/host/container-app-upsert.bicep' = if (deploymentTarget == 'containerapps') { name: 'aca-web' @@ -1402,3 +1522,5 @@ output AZURE_CONTAINER_REGISTRY_ENDPOINT string = deploymentTarget == 'container : '' output AZURE_VPN_CONFIG_DOWNLOAD_LINK string = useVpnGateway ? 'https://portal.azure.com/#@${tenant().tenantId}/resource${isolation.outputs.virtualNetworkGatewayId}/pointtositeconfiguration' : '' + +output PATENTSBERTA_ENDPOINT string = deploymentTarget == 'containerapps' ? patentsbertaService.outputs.endpoint : '' diff --git a/infra/main.parameters.json b/infra/main.parameters.json index 7a637c8022..fb1661389f 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -349,6 +349,12 @@ }, "ragSendImageSources": { "value": "${RAG_SEND_IMAGE_SOURCES=true}" + }, + "patentsbertaApiKey": { + "value": "${PATENTSBERTA_API_KEY}" + }, + "patentsbertaImageTag": { + "value": "${PATENTSBERTA_IMAGE_TAG=v1.0.0}" } } } diff --git a/infra/main.test.bicep b/infra/main.test.bicep deleted file mode 100644 index c79ddcfcf5..0000000000 --- a/infra/main.test.bicep +++ /dev/null @@ -1,40 +0,0 @@ -// This file is for doing static analysis and contains sensible defaults -// for PSRule to minimise false-positives and provide the best results. - -// This file is not intended to be used as a runtime configuration file. - -targetScope = 'subscription' - -param environmentName string = 'testing' -param location string = 'swedencentral' - -module main 'main.bicep' = { - name: 'main' - params: { - environmentName: environmentName - location: location - appServiceSkuName: 'B1' - documentIntelligenceResourceGroupLocation: location - documentIntelligenceSkuName: 'S0' - openAiHost: 'azure' - openAiLocation: location - searchIndexName: 'gptkbindex' - searchQueryLanguage: 'en-us' - searchQuerySpeller: 'lexicon' - searchServiceSemanticRankerLevel: 'free' - searchServiceSkuName: 'standard' - speechServiceSkuName: 'S0' - storageSkuName: 'Standard_LRS' - useApplicationInsights: false - useVectors: true - useMultimodal: true - enableLanguagePicker: false - useSpeechInputBrowser: false - useSpeechOutputBrowser: false - - // Test the secure configuration - enableUnauthenticatedAccess: false - usePrivateEndpoint: true - publicNetworkAccess: 'Disabled' - } -} diff --git a/infra/modules/keyvault.bicep b/infra/modules/keyvault.bicep new file mode 100644 index 0000000000..6d488c1890 --- /dev/null +++ b/infra/modules/keyvault.bicep @@ -0,0 +1,62 @@ +@description('Creates an Azure Key Vault for storing application secrets.') +param name string +param location string = resourceGroup().location +param tags object = {} +@description('Tenant ID for access policies') +param tenantId string +@description('Object IDs of users/principals that need access to Key Vault') +param accessPolicies array = [] +@description('Enable soft delete (recommended for production)') +param enableSoftDelete bool = true +@description('Retention days for soft delete (7-90)') +param softDeleteRetentionInDays int = 7 +@description('Enable purge protection (prevents permanent deletion)') +param enablePurgeProtection bool = false +@description('SKU for Key Vault (standard or premium)') +@allowed(['standard', 'premium']) +param sku string = 'standard' + +// Key Vault access policies structure: +// { +// objectId: string +// tenantId: string +// permissions: { +// keys: array +// secrets: array +// certificates: array +// } +// } + +resource keyVault 'Microsoft.KeyVault/vaults@2023-07-01' = { + name: name + location: location + tags: tags + properties: { + tenantId: tenantId + sku: { + family: 'A' + name: sku + } + enabledForDeployment: false + enabledForTemplateDeployment: true // Allow Bicep/ARM templates to access secrets + enabledForDiskEncryption: false + enableSoftDelete: enableSoftDelete + softDeleteRetentionInDays: softDeleteRetentionInDays + enablePurgeProtection: enablePurgeProtection + accessPolicies: accessPolicies + networkAcls: { + defaultAction: 'Allow' // Can be restricted to specific networks if needed + bypass: 'AzureServices' // Allow Azure services to access + } + publicNetworkAccess: 'Enabled' + } +} + +output id string = keyVault.id +output name string = keyVault.name +output vaultUri string = keyVault.properties.vaultUri + + + + + diff --git a/infra/modules/patentsberta.bicep b/infra/modules/patentsberta.bicep new file mode 100644 index 0000000000..2bf0bd90c0 --- /dev/null +++ b/infra/modules/patentsberta.bicep @@ -0,0 +1,94 @@ +param environmentName string +param location string +param containerAppsEnvironmentName string +param containerRegistryName string +@secure() +param patentsbertaApiKey string = '' +@description('Immutable image tag for deterministic deployments (e.g., v1.0.0-20241201)') +param imageTag string = 'v1.0.0' + +var resourceToken = toLower(uniqueString(subscription().id, environmentName, location)) + +resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' existing = { + name: containerAppsEnvironmentName +} + +resource containerRegistry 'Microsoft.ContainerRegistry/registries@2023-01-01-preview' existing = { + name: containerRegistryName +} + +resource patentsbertaApp 'Microsoft.App/containerApps@2023-05-01' = { + name: 'patentsberta-${resourceToken}' + location: location + properties: { + managedEnvironmentId: containerAppsEnvironment.id + configuration: { + ingress: { + external: true + targetPort: 8000 + allowInsecure: false + traffic: [ + { + latestRevision: true + weight: 100 + } + ] + } + registries: [ + { + server: containerRegistry.properties.loginServer + identity: 'system' + } + ] + } + template: { + containers: [ + { + name: 'patentsberta' + image: '${containerRegistry.properties.loginServer}/patentsberta-embeddings:${imageTag}' + env: [ + { + name: 'PATENTSBERTA_API_KEY' + value: patentsbertaApiKey + } + ] + resources: { + cpu: json('2.0') + memory: '4Gi' + } + probes: [ + { + type: 'readiness' + httpGet: { + path: '/health' + port: 8000 + } + initialDelaySeconds: 30 + periodSeconds: 10 + } + ] + } + ] + scale: { + minReplicas: 1 + maxReplicas: 3 + rules: [ + { + name: 'http-scaling' + http: { + metadata: { + concurrentRequests: '10' + } + } + } + ] + } + } + } + identity: { + type: 'SystemAssigned' + } +} + +output endpoint string = 'https://${patentsbertaApp.properties.configuration.ingress.fqdn}' +output name string = patentsbertaApp.name diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index 257a62346b..0000000000 --- a/package-lock.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "azure-search-openai-demo", - "lockfileVersion": 2, - "requires": true, - "packages": {} -} diff --git a/scripts/deploy-patentsberta.sh b/scripts/deploy-patentsberta.sh new file mode 100755 index 0000000000..f75a1b9cca --- /dev/null +++ b/scripts/deploy-patentsberta.sh @@ -0,0 +1,154 @@ +#!/bin/bash + +# Deploy PatentsBERTa Embedding Service +# This script builds and deploys the PatentsBERTa container to Azure Container Registry + +set -e + +# Default image tag - uses immutable versioning for deterministic deployments +# Format: v1.0.0-YYYYMMDD (can be overridden with IMAGE_TAG environment variable) +# This ensures reproducible deployments and allows for proper rollback capabilities +IMAGE_TAG=${IMAGE_TAG:-"v1.0.0-$(date +%Y%m%d)"} + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo -e "${GREEN}🚀 Starting PatentsBERTa Embedding Service Deployment${NC}" + +# Check if azd is installed +if ! command -v azd &> /dev/null; then + echo -e "${RED}❌ Azure Developer CLI (azd) is not installed. Please install it first.${NC}" + exit 1 +fi + +# Load environment variables +echo -e "${YELLOW}📋 Loading environment variables...${NC}" +azd env refresh + +# Get required environment variables +RESOURCE_GROUP=$(azd env get-value AZURE_RESOURCE_GROUP) +REGISTRY_ENDPOINT=$(azd env get-value AZURE_CONTAINER_REGISTRY_ENDPOINT) +REGISTRY_NAME=$(echo $REGISTRY_ENDPOINT | cut -d'.' -f1) +SUBSCRIPTION_ID=$(azd env get-value AZURE_SUBSCRIPTION_ID) + +if [ -z "$RESOURCE_GROUP" ] || [ -z "$REGISTRY_NAME" ] || [ -z "$SUBSCRIPTION_ID" ]; then + echo -e "${RED}❌ Required environment variables not found. Please run 'azd up' first.${NC}" + exit 1 +fi + +echo -e "${GREEN}✅ Environment variables loaded${NC}" +echo " Resource Group: $RESOURCE_GROUP" +echo " Registry: $REGISTRY_NAME" +echo " Subscription: $SUBSCRIPTION_ID" +echo " Image Tag: $IMAGE_TAG" + +# Build and push the PatentsBERTa container +echo -e "${YELLOW}🔨 Building and pushing PatentsBERTa container...${NC}" + +cd custom-embedding-service + +# Build and push using Azure Container Registry +az acr build \ + --registry "$REGISTRY_NAME" \ + --image patentsberta-embeddings:$IMAGE_TAG \ + --file Dockerfile \ + . + +if [ $? -eq 0 ]; then + echo -e "${GREEN}✅ Container built and pushed successfully${NC}" +else + echo -e "${RED}❌ Failed to build and push container${NC}" + exit 1 +fi + +cd .. + +# Set environment variables for PatentsBERTa +echo -e "${YELLOW}⚙️ Configuring environment for PatentsBERTa...${NC}" + +# Set the OpenAI host to use PatentsBERTa +azd env set OPENAI_HOST "patentsberta" + +# Set embedding dimensions for PatentsBERTa (768 dimensions) +azd env set AZURE_OPENAI_EMB_DIMENSIONS "768" + +# Set the embedding field name +azd env set AZURE_SEARCH_FIELD_NAME_EMBEDDING "embedding_patentsberta" + +# Set the image tag for deployment +azd env set PATENTSBERTA_IMAGE_TAG "$IMAGE_TAG" + +echo -e "${GREEN}✅ Environment configured for PatentsBERTa${NC}" + +# Deploy the infrastructure +echo -e "${YELLOW}🏗️ Deploying infrastructure with PatentsBERTa service...${NC}" + +azd up --no-prompt + +if [ $? -eq 0 ]; then + echo -e "${GREEN}✅ Infrastructure deployed successfully${NC}" + + # Get the PatentsBERTa endpoint + PATENTSBERTA_ENDPOINT=$(azd env get-value PATENTSBERTA_ENDPOINT) + + if [ -n "$PATENTSBERTA_ENDPOINT" ]; then + echo -e "${GREEN}🎉 PatentsBERTa service deployed successfully!${NC}" + echo " Endpoint: $PATENTSBERTA_ENDPOINT" + + # Test the service + echo -e "${YELLOW}🧪 Testing PatentsBERTa service...${NC}" + + # Wait a moment for the service to be ready + sleep 30 + + # Test health endpoint + if curl -f "$PATENTSBERTA_ENDPOINT/health" > /dev/null 2>&1; then + echo -e "${GREEN}✅ PatentsBERTa service is healthy${NC}" + else + echo -e "${YELLOW}⚠️ PatentsBERTa service may still be starting up. Check logs if issues persist.${NC}" + fi + + # Test embedding endpoint + echo -e "${YELLOW}🔍 Testing embedding generation...${NC}" + curl -X POST "$PATENTSBERTA_ENDPOINT/embeddings" \ + -H "Content-Type: application/json" \ + -d '{"texts": ["structural engineering patent claim"], "normalize": true}' \ + --max-time 60 > /dev/null 2>&1 + + if [ $? -eq 0 ]; then + echo -e "${GREEN}✅ Embedding generation test successful${NC}" + else + echo -e "${YELLOW}⚠️ Embedding generation test failed. Service may still be loading the model.${NC}" + fi + + else + echo -e "${RED}❌ PatentsBERTa endpoint not found in environment${NC}" + exit 1 + fi +else + echo -e "${RED}❌ Infrastructure deployment failed${NC}" + exit 1 +fi + +echo -e "${GREEN}🎉 PatentsBERTa deployment completed successfully!${NC}" +echo "" +echo -e "${YELLOW}📝 Next Steps:${NC}" +echo "1. Wait for the PatentsBERTa model to fully load (may take 2-3 minutes)" +echo "2. Delete the existing search index to recreate with new dimensions:" +echo " azd env get-value AZURE_SEARCH_SERVICE | xargs -I {} curl -X DELETE \"https://{}.search.windows.net/indexes/\$(azd env get-value AZURE_SEARCH_INDEX)?api-version=2024-07-01\" -H \"api-key: \$(azd env get-value AZURE_SEARCH_KEY)\"" +echo "3. Run document processing to reindex with PatentsBERTa embeddings:" +echo " python app/backend/prepdocs.py './data/*'" +echo "4. Test search quality with patent-specific queries" +echo "" +echo -e "${YELLOW}🏷️ Image Tag Management:${NC}" +echo " Current tag: $IMAGE_TAG" +echo " To deploy a specific version: IMAGE_TAG=v1.0.0-20241201 ./scripts/deploy-patentsberta.sh" +echo " To rollback: Set PATENTSBERTA_IMAGE_TAG in azd environment and run 'azd up'" +echo "" +echo -e "${GREEN}🔗 Useful endpoints:${NC}" +echo " Health: $PATENTSBERTA_ENDPOINT/health" +echo " Info: $PATENTSBERTA_ENDPOINT/info" +echo " Embeddings: $PATENTSBERTA_ENDPOINT/embeddings" diff --git a/scripts/switch-to-patentsberta.sh b/scripts/switch-to-patentsberta.sh new file mode 100755 index 0000000000..fad4948e28 --- /dev/null +++ b/scripts/switch-to-patentsberta.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +# Switch to PatentsBERTa Embeddings +# This script configures the environment to use PatentsBERTa instead of Azure OpenAI embeddings + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo -e "${GREEN}🔄 Switching to PatentsBERTa Embeddings${NC}" + +# Check if azd is installed +if ! command -v azd &> /dev/null; then + echo -e "${RED}❌ Azure Developer CLI (azd) is not installed. Please install it first.${NC}" + exit 1 +fi + +# Check if PatentsBERTa endpoint is available +PATENTSBERTA_ENDPOINT=$(azd env get-value PATENTSBERTA_ENDPOINT 2>/dev/null || echo "") + +if [ -z "$PATENTSBERTA_ENDPOINT" ]; then + echo -e "${RED}❌ PatentsBERTa endpoint not found. Please deploy the service first using:${NC}" + echo " ./scripts/deploy-patentsberta.sh" + exit 1 +fi + +echo -e "${YELLOW}📋 Current PatentsBERTa endpoint: $PATENTSBERTA_ENDPOINT${NC}" + +# Backup current configuration +echo -e "${YELLOW}💾 Backing up current configuration...${NC}" +BACKUP_FILE=".env.backup.$(date +%Y%m%d_%H%M%S)" + +# Save current environment variables +azd env get-values > "$BACKUP_FILE" +echo -e "${GREEN}✅ Configuration backed up to: $BACKUP_FILE${NC}" + +# Configure environment for PatentsBERTa +echo -e "${YELLOW}⚙️ Configuring environment for PatentsBERTa...${NC}" + +# Set the OpenAI host to use PatentsBERTa +azd env set OPENAI_HOST "patentsberta" + +# Set embedding dimensions for PatentsBERTa (768 dimensions) +azd env set AZURE_OPENAI_EMB_DIMENSIONS "768" + +# Set the embedding model name +azd env set AZURE_OPENAI_EMB_MODEL_NAME "PatentSBERTa" + +# Set the embedding field name (use a different field to avoid conflicts) +azd env set AZURE_SEARCH_FIELD_NAME_EMBEDDING "embedding_patentsberta" + +echo -e "${GREEN}✅ Environment configured for PatentsBERTa${NC}" + +# Test the PatentsBERTa service +echo -e "${YELLOW}🧪 Testing PatentsBERTa service...${NC}" + +# Test health endpoint +if curl -f -s "$PATENTSBERTA_ENDPOINT/health" > /dev/null; then + echo -e "${GREEN}✅ PatentsBERTa service is healthy${NC}" +else + echo -e "${RED}❌ PatentsBERTa service is not responding. Please check the deployment.${NC}" + exit 1 +fi + +# Warn about index recreation +echo -e "${YELLOW}⚠️ IMPORTANT: Search index needs to be recreated${NC}" +echo "" +echo -e "${RED}🚨 The search index must be deleted and recreated with new embedding dimensions.${NC}" +echo -e "${RED} This will remove all existing indexed documents.${NC}" +echo "" + +read -p "Do you want to delete the current search index? (y/N): " -n 1 -r +echo +if [[ $REPLY =~ ^[Yy]$ ]]; then + echo -e "${YELLOW}🗑️ Deleting current search index...${NC}" + + SEARCH_SERVICE=$(azd env get-value AZURE_SEARCH_SERVICE) + SEARCH_INDEX=$(azd env get-value AZURE_SEARCH_INDEX) + SEARCH_KEY=$(azd env get-value AZURE_SEARCH_KEY 2>/dev/null || echo "") + + if [ -n "$SEARCH_KEY" ]; then + # Use API key if available + curl -X DELETE \ + "https://${SEARCH_SERVICE}.search.windows.net/indexes/${SEARCH_INDEX}?api-version=2024-07-01" \ + -H "api-key: ${SEARCH_KEY}" \ + -w "HTTP Status: %{http_code}\n" + else + echo -e "${YELLOW}⚠️ No search key found. You may need to delete the index manually or use Azure CLI:${NC}" + echo " az search index delete --service-name $SEARCH_SERVICE --name $SEARCH_INDEX" + fi + + echo -e "${GREEN}✅ Search index deletion initiated${NC}" +else + echo -e "${YELLOW}⚠️ Skipping index deletion. You'll need to delete it manually before reindexing.${NC}" +fi + +echo "" +echo -e "${GREEN}🎉 Successfully switched to PatentsBERTa embeddings!${NC}" +echo "" +echo -e "${YELLOW}📝 Next Steps:${NC}" +echo "1. Wait for the search index to be fully deleted (if you chose to delete it)" +echo "2. Reindex your documents with PatentsBERTa embeddings:" +echo " cd app/backend && python prepdocs.py '../../data/*'" +echo "3. Test the search functionality with patent-specific queries" +echo "4. Run the test suite to validate the integration:" +echo " python tests/test-patentsberta.py" +echo "" +echo -e "${GREEN}🔗 PatentsBERTa Service Endpoints:${NC}" +echo " Health: $PATENTSBERTA_ENDPOINT/health" +echo " Info: $PATENTSBERTA_ENDPOINT/info" +echo " Embeddings: $PATENTSBERTA_ENDPOINT/embeddings" +echo "" +echo -e "${YELLOW}💡 To switch back to Azure OpenAI embeddings, restore from backup:${NC}" +echo " azd env set-values < $BACKUP_FILE" diff --git a/tests/README_CORPUS_TESTING.md b/tests/README_CORPUS_TESTING.md new file mode 100644 index 0000000000..a83bcb2f24 --- /dev/null +++ b/tests/README_CORPUS_TESTING.md @@ -0,0 +1,192 @@ +# Corpus Document Retrieval and Citation Testing + +This guide explains how to test if your RAG system correctly retrieves information from your corpus documents and provides accurate citations. + +## Quick Start + +Run the corpus accuracy test: + +```bash +cd tests +python test_corpus_accuracy.py +``` + +## What the Test Does + +1. **Discovers Indexed Documents**: First, it queries the system to find what documents are in your knowledge base. + +2. **Tests Corpus Retrieval**: Runs test queries and verifies: + - Information is retrieved from your corpus documents + - Citations point to the correct documents + - Corpus sources are prioritized over web sources + - Answers include proper citations + +3. **Provides Detailed Analysis**: Shows: + - Which documents were retrieved + - What citations were generated + - Whether expected documents were found + - If answers properly cite sources + +## Customizing Tests + +Edit `test_corpus_accuracy.py` and add your own test queries in the `test_queries` list: + +```python +test_queries = [ + { + "query": "What is the code review process?", + "expected_docs": ["Code_Review_Checklist.pdf"], + "description": "Test retrieval from Code Review Checklist" + }, + { + "query": "What are the release validation steps?", + "expected_docs": ["Release_Validation_Process.pdf"], + "description": "Test retrieval from Release Validation Process" + }, + # Add more queries based on your documents... +] +``` + +### Test Query Format + +- **query**: The question to ask (should match content in your documents) +- **expected_docs**: List of document names that should be cited (e.g., `["Document1.pdf"]`) +- **description**: Brief description of what this test verifies + +## Understanding Test Results + +### ✅ PASS Indicators + +- `[PASS] Answer includes corpus citations` - Answer properly cites corpus documents +- `[PASS] All expected documents found` - Expected documents were retrieved and cited +- `Tests with corpus sources: X/X` - All tests retrieved corpus sources + +### ⚠️ WARN Indicators + +- `[WARN] Answer may not be citing corpus sources properly` - Citations found but may not be in answer format +- `[WARN] Answer says 'I don't know' but corpus sources were found` - Sources retrieved but don't contain answer + +### ❌ FAIL Indicators + +- `[WARNING] No corpus sources found!` - Documents not retrieved (may not be indexed or query doesn't match) +- `[MISSING] Not found: [...]` - Expected documents were not retrieved + +## Testing Different Scenarios + +### Test 1: Corpus-Only Mode (Default) + +Tests that corpus documents are retrieved correctly: + +```python +result = await test_corpus_query( + "What is the code review process?", + expected_documents=["Code_Review_Checklist.pdf"], + use_corpus_only=True # Forces RAG-only, no web search +) +``` + +### Test 2: Hybrid Mode + +Tests that corpus is prioritized over web: + +```python +result = await test_corpus_query( + "What is RAG?", + expected_documents=None, + use_corpus_only=False # Allows web search as fallback +) +``` + +## Verifying Citation Accuracy + +The test checks: + +1. **Citation Format**: Citations should be in format `[DocumentName.pdf#page=N]` +2. **Citation Presence**: Citations should appear in the answer text +3. **Document Matching**: Expected documents should be in the citations list +4. **Source Priority**: Corpus sources should be used before web sources + +## Troubleshooting + +### No Corpus Sources Found + +**Possible causes:** +- Documents not indexed in Azure AI Search +- Query doesn't match document content +- Search index needs to be rebuilt + +**Solutions:** +1. Verify documents are in Azure Blob Storage +2. Run `prepdocs.py` to re-index documents +3. Check Azure AI Search index contains your documents +4. Try queries that match exact phrases from your documents + +### Citations Not in Answer + +**Possible causes:** +- LLM not following citation format +- Prompt not instructing citations properly + +**Solutions:** +1. Check prompt templates in `app/backend/approaches/prompts/` +2. Verify citations are in the `citations` array +3. Review answer format - citations should be like `[doc.pdf#page=1]` + +### Wrong Documents Cited + +**Possible causes:** +- Search retrieval returning irrelevant chunks +- Similar content in multiple documents + +**Solutions:** +1. Review retrieved sources in test output +2. Check if search is using correct embeddings +3. Verify document content matches query intent +4. Adjust `top` parameter to get more/fewer results + +## Example Test Output + +``` +[TEST 1/2] Test retrieval from Code Review Checklist +================================================================================ +TESTING CORPUS QUERY: What is the code review process? +================================================================================ + +[INFO] Query: What is the code review process? +[INFO] Mode: RAG-only (corpus) + +[RESPONSE] +Answer: The code review process involves... [Code_Review_Checklist.pdf#page=1] +Answer length: 245 characters + +[SOURCES ANALYSIS] +Total text sources: 2 +Corpus sources: 2 +Web sources: 0 + +[CORPUS SOURCES] + 1. Code_Review_Checklist.pdf#page=1 + Preview: Code Review Checklist This checklist serves as a guide... + +[CITATIONS] +Total citations: 2 +Corpus citations (2): + 1. Code_Review_Checklist.pdf#page=1 + 2. Release_Validation_Process.pdf#page=1 + +[VERIFICATION] +Expected documents: ['Code_Review_Checklist.pdf'] +[PASS] All expected documents found: ['Code_Review_Checklist.pdf'] + +[ACCURACY CHECK] +[PASS] Answer includes corpus citations +``` + +## Next Steps + +1. **Add Your Test Queries**: Edit `test_corpus_accuracy.py` with queries based on your documents +2. **Run Tests**: Execute the script to verify corpus retrieval +3. **Review Results**: Check if expected documents are cited correctly +4. **Fix Issues**: Address any warnings or failures +5. **Iterate**: Add more tests as you add documents + diff --git a/tests/README_FUNCTIONALITY_TESTING.md b/tests/README_FUNCTIONALITY_TESTING.md new file mode 100644 index 0000000000..331dbbbe8f --- /dev/null +++ b/tests/README_FUNCTIONALITY_TESTING.md @@ -0,0 +1,167 @@ +# Complete Functionality Testing Guide + +## Overview + +The `test_functionality.py` script tests the complete functionality of your application, including: +- Backend API endpoints (health, config, chat, ask) +- RAG responses with citations +- OCR functionality (if enabled) +- Web search (if enabled) +- Agents service (if running) +- Cache functionality +- Embedding router + +## Prerequisites + +1. **Backend service must be running:** + ```powershell + cd app\backend + uvicorn main:app --reload + ``` + +2. **Agents service (optional):** + ```powershell + cd agents + python main.py + ``` + +3. **Environment variables configured:** + - Run `python tests/check_env_vars.py` to verify + +## Running the Tests + +### Basic Test (Default URLs) +```powershell +cd tests +python test_functionality.py +``` + +### Custom URLs +```powershell +python test_functionality.py --backend-url http://localhost:50505 --agents-url http://localhost:8000 +``` + +## What Gets Tested + +### 1. Backend Health ✅ +- Tests `/health` endpoint +- Verifies service status and dependencies + +### 2. Backend Config ✅ +- Tests `/config` endpoint +- Checks available features + +### 3. Cache Functionality ✅ +- Tests Redis or in-memory cache +- Verifies set/get operations + +### 4. Embedding Router ✅ +- Tests model selection logic +- Verifies routing decisions + +### 5. OCR Functionality ✅ +- Checks if OCR is enabled +- Verifies OCR service configuration +- Tests OCR service initialization + +### 6. Web Search Functionality ✅ +- Checks if web search is enabled +- Verifies SERPER API key is set + +### 7. Chat Endpoint ✅ +- Tests `/chat` POST endpoint +- Sends a real query: "What is RAG?" +- Verifies response structure +- Checks for citations +- Validates answer quality + +### 8. Ask Endpoint ✅ +- Tests `/ask` POST endpoint +- Sends a real query: "Explain vector search" +- Verifies response structure +- Checks for citations + +### 9. Agents Service Health ✅ +- Tests agents service `/api/health` endpoint +- Verifies connectivity to backend + +## Expected Results + +### Success Output +``` +[PASS] Backend is healthy +[PASS] Config endpoint working +[PASS] Cache working (Redis/In-memory) +[PASS] Embedding router working +[PASS] Chat endpoint working + Answer length: 250 characters + Citations: 3 +[PASS] Ask endpoint working +``` + +### Failure Output +``` +[FAIL] Backend not running at http://localhost:50505 +[SKIP] Agents service not running +[WARN] No citations found +``` + +## Troubleshooting + +### Backend Not Running +```powershell +# Start backend +cd app\backend +uvicorn main:app --reload +``` + +### Authentication Required +If endpoints return 401, you may need to: +1. Disable authentication for testing +2. Or provide auth tokens in the test script + +### No Citations +- Ensure documents are indexed in Azure AI Search +- Check that search index has content +- Verify search service is accessible + +### Empty Answers +- Check Azure OpenAI service is accessible +- Verify API keys are set correctly +- Check service logs for errors + +## Advanced Testing + +### Test with Custom Queries +Edit `test_functionality.py` and modify: +```python +await tester.test_chat_endpoint("Your custom query here") +``` + +### Test Specific Features +Comment out tests you don't need in the `run_all_tests()` method. + +### Integration with pytest +You can also use the existing pytest tests: +```powershell +pytest tests/e2e_agents_test.py -v +pytest tests/test_app.py -v +``` + +## Next Steps + +1. **Run the test suite** to verify everything works +2. **Check the summary** to see what passed/failed +3. **Fix any issues** based on the test results +4. **Re-run tests** to verify fixes + +## Continuous Testing + +For CI/CD, you can run: +```powershell +python tests/test_functionality.py --backend-url $BACKEND_URL --agents-url $AGENTS_URL +``` + +Exit code 0 = all tests passed +Exit code 1 = some tests failed + diff --git a/tests/check_env_vars.py b/tests/check_env_vars.py new file mode 100644 index 0000000000..f98d2bdaf4 --- /dev/null +++ b/tests/check_env_vars.py @@ -0,0 +1,209 @@ +""" +Check Environment Variables + +Verifies which environment variables are set and which are missing. +""" + +import os +import sys +import json +import subprocess +from pathlib import Path + +# Add backend to path +sys.path.insert(0, str(Path(__file__).parent.parent / "app" / "backend")) + +def load_env_files(): + """Try to load environment variables from azd and local .env files.""" + loaded_files = [] + from dotenv import load_dotenv + + # Try azd environment first (as the app does) + azd_env_path = None + try: + result = subprocess.run("azd env list -o json", shell=True, capture_output=True, text=True) + if result.returncode == 0: + env_json = json.loads(result.stdout) + for entry in env_json: + if entry.get("IsDefault"): + env_file_path = entry.get("DotEnvPath") + if env_file_path and os.path.exists(env_file_path): + azd_env_path = env_file_path + load_dotenv(env_file_path, override=True) + loaded_files.append(env_file_path) + break + except Exception: + pass + + # Also check app/backend/.env (for checking purposes, even though azd takes precedence) + backend_env = Path(__file__).parent.parent / "app" / "backend" / ".env" + if backend_env.exists(): + # Load without override to see what's in .env (azd vars already loaded) + # But we need to check what's actually set, so load it + if not azd_env_path: # Only load if azd wasn't found + load_dotenv(backend_env, override=True) + else: + # Load to see what's there, but azd takes precedence + load_dotenv(backend_env, override=False) + loaded_files.append(str(backend_env)) + + return loaded_files if loaded_files else None + +def check_env_vars(): + """Check all required and optional environment variables.""" + + print("=" * 70) + print("ENVIRONMENT VARIABLES STATUS CHECK") + print("=" * 70) + + # Try to load environment files (azd first, then .env) + loaded_files = load_env_files() + if loaded_files: + print(f"[INFO] Loaded environment from:") + for f in loaded_files: + print(f" - {f}") + print("[NOTE] The app loads env vars in this order: azd env -> app/backend/.env -> shell") + else: + print("[INFO] No environment files found - checking current shell environment only") + print("[NOTE] The app loads env vars from azd or .env at runtime") + print() + + # Required for core functionality + required_core = { + "AZURE_STORAGE_ACCOUNT": "Azure Storage Account name", + "AZURE_STORAGE_CONTAINER": "Azure Storage Container name", + "AZURE_SEARCH_SERVICE": "Azure AI Search service name", + "AZURE_SEARCH_INDEX": "Azure AI Search index name", + "AZURE_OPENAI_CHATGPT_MODEL": "OpenAI ChatGPT model name", + } + + # Optional but important + optional_important = { + "AZURE_OPENAI_SERVICE": "Azure OpenAI service name", + "AZURE_OPENAI_CHATGPT_DEPLOYMENT": "Azure OpenAI ChatGPT deployment", + "AZURE_OPENAI_EMB_DEPLOYMENT": "Azure OpenAI Embedding deployment", + "AZURE_OPENAI_EMB_MODEL_NAME": "Embedding model name", + "AZURE_OPENAI_EMB_DIMENSIONS": "Embedding dimensions", + "AZURE_SEARCH_KEY": "Azure Search API key", + "AZURE_OPENAI_API_KEY": "Azure OpenAI API key", + } + + # Feature flags + feature_flags = { + "OCR_PROVIDER": "OCR provider (ollama, azure_document_intelligence, none)", + "OCR_ON_INGEST": "Run OCR during ingestion (true/false)", + "ENABLE_WEB_SEARCH": "Enable web search (true/false)", + "SERPER_API_KEY": "Serper API key for web search", + "REDIS_URL": "Redis cache URL", + "ENABLE_NOMIC_EMBEDDINGS": "Enable NOMIC embeddings (true/false)", + "NOMIC_API_KEY": "NOMIC API key", + } + + # OCR specific + ocr_vars = { + "OLLAMA_BASE_URL": "Ollama base URL", + "OLLAMA_OCR_MODEL": "Ollama OCR model name", + "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT": "Azure Document Intelligence endpoint", + "AZURE_DOCUMENT_INTELLIGENCE_KEY": "Azure Document Intelligence key", + } + + # Authentication + auth_vars = { + "AZURE_USE_AUTHENTICATION": "Enable authentication (true/false)", + "AZURE_TENANT_ID": "Azure Tenant ID", + "AZURE_SERVER_APP_ID": "Azure Server App ID", + "AZURE_SERVER_APP_SECRET": "Azure Server App Secret", + "AZURE_CLIENT_APP_ID": "Azure Client App ID", + } + + def check_section(name, vars_dict, required=False): + """Check a section of environment variables.""" + print(f"\n{name}:") + print("-" * 70) + set_count = 0 + missing = [] + + for var, description in vars_dict.items(): + value = os.getenv(var) + if value: + # Mask sensitive values + if "KEY" in var or "SECRET" in var or "PASSWORD" in var: + display_value = f"{value[:4]}...{value[-4:]}" if len(value) > 8 else "***" + else: + display_value = value + print(f" [SET] {var:40} = {display_value}") + set_count += 1 + else: + status = "[REQUIRED - MISSING]" if required else "[OPTIONAL - NOT SET]" + print(f" {status} {var:40} - {description}") + if required: + missing.append(var) + + return set_count, len(vars_dict), missing + + # Check all sections + core_set, core_total, core_missing = check_section("CORE REQUIRED VARIABLES", required_core, required=True) + opt_set, opt_total, _ = check_section("OPTIONAL IMPORTANT VARIABLES", optional_important) + feat_set, feat_total, _ = check_section("FEATURE FLAGS", feature_flags) + ocr_set, ocr_total, _ = check_section("OCR CONFIGURATION", ocr_vars) + auth_set, auth_total, _ = check_section("AUTHENTICATION", auth_vars) + + # Summary + print("\n" + "=" * 70) + print("SUMMARY") + print("=" * 70) + print(f"Core Required: {core_set}/{core_total} set") + print(f"Optional Important: {opt_set}/{opt_total} set") + print(f"Feature Flags: {feat_set}/{feat_total} set") + print(f"OCR Config: {ocr_set}/{ocr_total} set") + print(f"Authentication: {auth_set}/{auth_total} set") + + total_set = core_set + opt_set + feat_set + ocr_set + auth_set + total_vars = core_total + opt_total + feat_total + ocr_total + auth_total + + print(f"\nOverall: {total_set}/{total_vars} variables set") + + if core_missing: + print(f"\n[CRITICAL] Missing required variables: {', '.join(core_missing)}") + print("The application will NOT work without these!") + return False + + # Check feature status + print("\n" + "=" * 70) + print("FEATURE STATUS") + print("=" * 70) + + ocr_provider = os.getenv("OCR_PROVIDER", "none").lower() + web_search = os.getenv("ENABLE_WEB_SEARCH", "false").lower() == "true" + serper_key = os.getenv("SERPER_API_KEY") + redis_url = os.getenv("REDIS_URL") + nomic_enabled = os.getenv("ENABLE_NOMIC_EMBEDDINGS", "false").lower() == "true" + nomic_key = os.getenv("NOMIC_API_KEY") + + print(f"OCR: {'[ENABLED]' if ocr_provider != 'none' else '[DISABLED]'}") + if ocr_provider == "ollama": + ollama_url = os.getenv("OLLAMA_BASE_URL", "http://localhost:11434/v1") + print(f" - Ollama URL: {ollama_url}") + elif ocr_provider == "azure_document_intelligence": + di_endpoint = os.getenv("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT") + print(f" - DI Endpoint: {'[SET]' if di_endpoint else '[NOT SET]'}") + + print(f"Web Search: {'[ENABLED]' if web_search and serper_key else '[DISABLED]'}") + if web_search and not serper_key: + print(" - WARNING: ENABLE_WEB_SEARCH=true but SERPER_API_KEY not set") + + print(f"Redis Cache: {'[ENABLED]' if redis_url else '[DISABLED - using in-memory]'}") + print(f"NOMIC Embeddings: {'[ENABLED]' if nomic_enabled and nomic_key else '[DISABLED]'}") + if nomic_enabled and not nomic_key: + print(" - WARNING: ENABLE_NOMIC_EMBEDDINGS=true but NOMIC_API_KEY not set") + + print("\n" + "=" * 70) + + if core_missing: + return False + return True + +if __name__ == "__main__": + success = check_env_vars() + sys.exit(0 if success else 1) + diff --git a/tests/e2e_agents_test.py b/tests/e2e_agents_test.py new file mode 100644 index 0000000000..d84f7fab3d --- /dev/null +++ b/tests/e2e_agents_test.py @@ -0,0 +1,323 @@ +""" +End-to-End Tests for Agents Service. + +Tests the Agents service endpoints and Bot Framework integration. +""" + +import pytest +import aiohttp +import json +import os +from typing import Dict, Any, Optional + + +class TestAgentsService: + """End-to-end tests for Agents service.""" + + @pytest.fixture + def agents_url(self) -> str: + """Get Agents service URL from environment or use default.""" + return os.getenv("AGENTS_SERVICE_URL", "http://localhost:8000") + + @pytest.fixture + def backend_url(self) -> str: + """Get Backend service URL from environment or use default.""" + return os.getenv("BACKEND_URL", "http://localhost:50505") + + @pytest.mark.asyncio + async def test_health_endpoint(self, agents_url: str): + """Test Agents service health endpoint.""" + try: + async with aiohttp.ClientSession() as session: + async with session.get(f"{agents_url}/api/health", timeout=aiohttp.ClientTimeout(total=5)) as response: + assert response.status == 200 + data = await response.json() + assert "status" in data + assert "agent" in data + assert "services" in data + print(f"Health check response: {json.dumps(data, indent=2)}") + except aiohttp.ClientConnectorError: + pytest.skip(f"Agents service not running at {agents_url}") + except Exception as e: + pytest.fail(f"Unexpected error: {e}") + + @pytest.mark.asyncio + async def test_config_endpoint(self, agents_url: str): + """Test Agents service config endpoint.""" + try: + async with aiohttp.ClientSession() as session: + async with session.get(f"{agents_url}/api/config", timeout=aiohttp.ClientTimeout(total=5)) as response: + assert response.status == 200 + data = await response.json() + assert "agent_name" in data + assert "channels" in data + print(f"Config response: {json.dumps(data, indent=2)}") + except aiohttp.ClientConnectorError: + pytest.skip(f"Agents service not running at {agents_url}") + except Exception as e: + pytest.fail(f"Unexpected error: {e}") + + @pytest.mark.asyncio + async def test_messages_endpoint_requires_auth(self, agents_url: str): + """Test that /api/messages requires authentication.""" + # Test without auth header + message_body = { + "type": "message", + "text": "Hello", + "from": {"id": "test-user", "name": "Test User"}, + "channelId": "webchat" + } + + try: + async with aiohttp.ClientSession() as session: + async with session.post( + f"{agents_url}/api/messages", + json=message_body, + timeout=aiohttp.ClientTimeout(total=5) + ) as response: + # Should return 401 or 400 (depending on Bot Framework validation) + assert response.status in [400, 401, 403] + print(f"Auth required test passed: {response.status}") + except aiohttp.ClientConnectorError: + pytest.skip(f"Agents service not running at {agents_url}") + except Exception as e: + pytest.fail(f"Unexpected error: {e}") + + @pytest.mark.asyncio + async def test_backend_health_from_agents(self, agents_url: str, backend_url: str): + """Test that Agents service can reach Backend service.""" + try: + async with aiohttp.ClientSession() as session: + # Check Agents health (which checks backend) + async with session.get(f"{agents_url}/api/health", timeout=aiohttp.ClientTimeout(total=5)) as response: + assert response.status == 200 + data = await response.json() + + # Verify backend status is reported + if "services" in data and "backend" in data["services"]: + backend_status = data["services"]["backend"] + print(f"Backend status from Agents: {json.dumps(backend_status, indent=2)}") + + # If backend is reachable, check it directly + if backend_status.get("ok"): + try: + async with session.get(f"{backend_url}/health", timeout=aiohttp.ClientTimeout(total=5)) as backend_response: + assert backend_response.status == 200 + print("Backend health check passed") + except aiohttp.ClientConnectorError: + pytest.skip(f"Backend service not running at {backend_url}") + except aiohttp.ClientConnectorError: + pytest.skip(f"Agents service not running at {agents_url}") + except Exception as e: + pytest.fail(f"Unexpected error: {e}") + + @pytest.mark.asyncio + async def test_correlation_id_propagation(self, agents_url: str, backend_url: str): + """Test that correlation IDs are propagated from Agents to Backend.""" + # This test requires both services running + # For now, just verify the endpoint exists + traceparent = "00-12345678901234567890123456789012-1234567890123456-01" + + message_body = { + "type": "message", + "text": "Test correlation ID", + "from": {"id": "test-user", "name": "Test User"}, + "channelId": "emulator" # Use emulator for local testing + } + + headers = { + "Content-Type": "application/json", + "x-traceparent": traceparent + } + + try: + async with aiohttp.ClientSession() as session: + # Note: This will fail without proper auth, but we can verify the endpoint accepts it + async with session.post( + f"{agents_url}/api/messages", + json=message_body, + headers=headers, + timeout=aiohttp.ClientTimeout(total=5) + ) as response: + # Even if auth fails, correlation ID should be logged + print(f"Correlation ID test response: {response.status}") + # In production, we'd verify the correlation ID in logs + except aiohttp.ClientConnectorError: + pytest.skip(f"Agents service not running at {agents_url}") + except Exception as e: + pytest.fail(f"Unexpected error: {e}") + + +class TestBackendService: + """End-to-end tests for Backend RAG service.""" + + @pytest.fixture + def backend_url(self) -> str: + """Get Backend service URL from environment or use default.""" + return os.getenv("BACKEND_URL", "http://localhost:50505") + + @pytest.fixture + def auth_token(self) -> Optional[str]: + """Get auth token for testing (if available).""" + return os.getenv("TEST_AUTH_TOKEN") + + @pytest.mark.asyncio + async def test_health_endpoint(self, backend_url: str): + """Test Backend service health endpoint with dependency checks.""" + try: + async with aiohttp.ClientSession() as session: + async with session.get(f"{backend_url}/health", timeout=aiohttp.ClientTimeout(total=5)) as response: + assert response.status == 200 + data = await response.json() + assert "status" in data + assert "dependencies" in data + print(f"Backend health check: {json.dumps(data, indent=2)}") + except aiohttp.ClientConnectorError: + pytest.skip(f"Backend service not running at {backend_url}") + except Exception as e: + pytest.fail(f"Unexpected error: {e}") + + @pytest.mark.asyncio + async def test_config_endpoint(self, backend_url: str): + """Test Backend config endpoint.""" + try: + async with aiohttp.ClientSession() as session: + async with session.get(f"{backend_url}/config", timeout=aiohttp.ClientTimeout(total=5)) as response: + assert response.status == 200 + data = await response.json() + assert "showVectorOption" in data + print(f"Backend config: {json.dumps(data, indent=2)}") + except aiohttp.ClientConnectorError: + pytest.skip(f"Backend service not running at {backend_url}") + except Exception as e: + pytest.fail(f"Unexpected error: {e}") + + @pytest.mark.asyncio + async def test_chat_endpoint_requires_auth(self, backend_url: str): + """Test that /chat endpoint handles authentication (required or optional).""" + chat_body = { + "messages": [{"role": "user", "content": "Hello"}], + "context": {} + } + + try: + async with aiohttp.ClientSession() as session: + async with session.post( + f"{backend_url}/chat", + json=chat_body, + timeout=aiohttp.ClientTimeout(total=5) + ) as response: + # In production: should return 401 Unauthorized + # In local dev: may return 200 if auth is disabled + status = response.status + assert status in [200, 401], f"Unexpected status code: {status}" + + if status == 401: + print("Chat endpoint correctly requires authentication (401)") + elif status == 200: + data = await response.json() + print("Chat endpoint allows unauthenticated access (200) - likely local dev mode") + # Verify it's a valid response structure + # Response structure: {"message": {...}, "context": {...}, "session_state": ...} + # Or error response: {"error": ...} + assert ( + "message" in data + or "error" in data + or "context" in data + or "answer" in data + or "choices" in data + ), f"Unexpected response structure: {list(data.keys())}" + except aiohttp.ClientConnectorError: + pytest.skip(f"Backend service not running at {backend_url}") + except Exception as e: + pytest.fail(f"Unexpected error: {e}") + + +class TestIntegration: + """Integration tests between Agents and Backend.""" + + @pytest.fixture + def agents_url(self) -> str: + """Get Agents service URL.""" + return os.getenv("AGENTS_SERVICE_URL", "http://localhost:8000") + + @pytest.fixture + def backend_url(self) -> str: + """Get Backend service URL.""" + return os.getenv("BACKEND_URL", "http://localhost:50505") + + @pytest.mark.asyncio + async def test_agents_to_backend_connectivity(self, agents_url: str, backend_url: str): + """Test that Agents service can connect to Backend.""" + try: + async with aiohttp.ClientSession() as session: + # Check Agents health (which pings backend) + async with session.get(f"{agents_url}/api/health", timeout=aiohttp.ClientTimeout(total=5)) as response: + assert response.status == 200 + data = await response.json() + + if "services" in data and "backend" in data["services"]: + backend_info = data["services"]["backend"] + print(f"Backend connectivity: {json.dumps(backend_info, indent=2)}") + + # Verify backend URL is correct + if "url" in backend_info: + assert backend_url in backend_info["url"] or backend_info["url"] in backend_url + except aiohttp.ClientConnectorError: + pytest.skip(f"Agents service not running at {agents_url}") + except Exception as e: + pytest.fail(f"Unexpected error: {e}") + + +if __name__ == "__main__": + """Run tests directly.""" + import asyncio + + async def run_tests(): + """Run basic connectivity tests.""" + agents_url = os.getenv("AGENTS_SERVICE_URL", "http://localhost:8000") + backend_url = os.getenv("BACKEND_URL", "http://localhost:50505") + + print("=" * 60) + print("E2E Connectivity Tests") + print("=" * 60) + + async with aiohttp.ClientSession() as session: + # Test Agents health + print("\n1. Testing Agents service health...") + try: + async with session.get(f"{agents_url}/api/health", timeout=aiohttp.ClientTimeout(total=5)) as resp: + if resp.status == 200: + data = await resp.json() + print(f" ✅ Agents service is healthy") + print(f" Status: {data.get('status')}") + if "services" in data and "backend" in data["services"]: + backend_status = data["services"]["backend"] + print(f" Backend: {'✅ OK' if backend_status.get('ok') else '❌ Not reachable'}") + else: + print(f" ❌ Agents service returned {resp.status}") + except Exception as e: + print(f" ❌ Failed to connect to Agents service: {e}") + + # Test Backend health + print("\n2. Testing Backend service health...") + try: + async with session.get(f"{backend_url}/health", timeout=aiohttp.ClientTimeout(total=5)) as resp: + if resp.status == 200: + data = await resp.json() + print(f" ✅ Backend service is healthy") + print(f" Status: {data.get('status')}") + if "dependencies" in data: + deps = data["dependencies"] + print(f" Dependencies: {len([d for d in deps.values() if d.get('ok')])}/{len(deps)} healthy") + else: + print(f" ❌ Backend service returned {resp.status}") + except Exception as e: + print(f" ❌ Failed to connect to Backend service: {e}") + + print("\n" + "=" * 60) + print("Tests completed") + print("=" * 60) + + asyncio.run(run_tests()) + diff --git a/tests/test-patentsberta.py b/tests/test-patentsberta.py new file mode 100755 index 0000000000..dc857a59f3 --- /dev/null +++ b/tests/test-patentsberta.py @@ -0,0 +1,384 @@ +import asyncio +import aiohttp +import os +import sys + +# Add the backend directory to the path +sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'app', 'backend')) + +from prepdocslib.patentsberta_embeddings import PatentsBertaEmbeddings + +class PatentsBertaTestSuite: + def __init__(self, endpoint: str, api_key: str = None): + self.endpoint = endpoint + self.api_key = api_key + self.embedding_service = PatentsBertaEmbeddings(endpoint, api_key) + + async def test_health(self) -> bool: + print("🏥 Testing health endpoint...") + try: + async with aiohttp.ClientSession() as session: + async with session.get(f"{self.endpoint}/health") as response: + if response.status == 200: + data = await response.json() + print(f"✅ Health check passed: {data}") + return True + else: + print(f"❌ Health check failed: {response.status}") + return False + except Exception as e: + print(f"❌ Health check error: {e}") + return False + + async def test_info(self) -> bool: + print("ℹ️ Testing info endpoint...") + try: + async with aiohttp.ClientSession() as session: + async with session.get(f"{self.endpoint}/info") as response: + if response.status == 200: + data = await response.json() + print(f"✅ Info endpoint: {data}") + return True + else: + print(f"❌ Info endpoint failed: {response.status}") + return False + except Exception as e: + print(f"❌ Info endpoint error: {e}") + return False + + async def test_single_embedding(self) -> bool: + print("🔍 Testing single embedding generation...") + test_text = "structural engineering patent for seismic isolation system" + + try: + embedding = await self.embedding_service.create_embedding(test_text) + + if embedding and len(embedding) == 768: + print(f"✅ Single embedding generated successfully") + print(f" Dimensions: {len(embedding)}") + print(f" Sample values: {embedding[:5]}...") + return True + else: + print(f"❌ Single embedding failed: wrong dimensions {len(embedding) if embedding else 0}") + return False + + except Exception as e: + print(f"❌ Single embedding error: {e}") + return False + + async def test_batch_embeddings(self) -> bool: + print("📦 Testing batch embedding generation...") + test_texts = [ + "foundation system with load distribution mechanism", + "composite structural beam with carbon fiber reinforcement", + "damping apparatus for earthquake resistant buildings", + "steel frame connection with moment resistance", + "concrete column with spiral reinforcement design" + ] + + try: + embeddings = await self.embedding_service.create_embeddings(test_texts) + + if embeddings and len(embeddings) == len(test_texts): + all_correct_dims = all(len(emb) == 768 for emb in embeddings) + if all_correct_dims: + print(f"✅ Batch embeddings generated successfully") + print(f" Count: {len(embeddings)}") + print(f" Dimensions: {len(embeddings[0])}") + return True + else: + print(f"❌ Batch embeddings failed: incorrect dimensions") + return False + else: + print(f"❌ Batch embeddings failed: wrong count {len(embeddings) if embeddings else 0}") + return False + + except Exception as e: + print(f"❌ Batch embeddings error: {e}") + return False + + async def test_patent_terminology(self) -> bool: + print("🔬 Testing patent-specific terminology...") + + patent_queries = [ + "apparatus for structural vibration control", + "method of reinforcing concrete structures", + "system for seismic base isolation", + "device for load transfer in buildings", + "composition of high-strength concrete mixture" + ] + + try: + embeddings = await self.embedding_service.create_embeddings(patent_queries) + + if embeddings and len(embeddings) == len(patent_queries): + print(f"✅ Patent terminology embeddings generated") + + # Test similarity between related concepts + # This is a basic test - in practice you'd want more sophisticated similarity testing + print(" Testing conceptual similarity...") + + # Compare "apparatus" and "device" embeddings (should be similar) + apparatus_emb = embeddings[0] # "apparatus for structural vibration control" + device_emb = embeddings[3] # "device for load transfer in buildings" + + # Simple cosine similarity calculation + import numpy as np + + def cosine_similarity(a, b): + return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)) + + similarity = cosine_similarity(apparatus_emb, device_emb) + print(f" Similarity between 'apparatus' and 'device': {similarity:.3f}") + + if similarity > 0.5: # Reasonable threshold for related concepts + print("✅ Patent terminology shows good semantic understanding") + return True + else: + print("⚠️ Patent terminology similarity lower than expected") + return True # Still pass, but note the issue + else: + print(f"❌ Patent terminology test failed") + return False + + except Exception as e: + print(f"❌ Patent terminology error: {e}") + return False + + async def test_performance(self) -> bool: + print("⚡ Testing performance...") + + import time + + # Test single embedding performance + start_time = time.time() + await self.embedding_service.create_embedding("test performance query") + single_time = time.time() - start_time + + # Test batch performance + batch_texts = ["performance test query"] * 10 + start_time = time.time() + await self.embedding_service.create_embeddings(batch_texts) + batch_time = time.time() - start_time + + print(f"✅ Performance results:") + print(f" Single embedding: {single_time:.2f}s") + print(f" Batch (10 items): {batch_time:.2f}s") + print(f" Avg per item in batch: {batch_time/10:.2f}s") + + # Performance is acceptable if single < 10s and batch avg < 2s + if single_time < 10 and (batch_time/10) < 2: + print("✅ Performance is acceptable") + return True + else: + print("⚠️ Performance may be slower than expected") + return True # Still pass, but note the issue + + async def test_authentication(self) -> bool: + """Test API key authentication""" + print("🔐 Testing API key authentication...") + + try: + # Test with correct API key (should work) + if self.api_key: + headers = {'Content-Type': 'application/json', 'X-API-Key': self.api_key} + payload = {'texts': ['test authentication'], 'normalize': True} + + async with aiohttp.ClientSession() as session: + async with session.post( + f"{self.endpoint}/embeddings", + json=payload, + headers=headers, + timeout=aiohttp.ClientTimeout(total=30) + ) as response: + if response.status == 200: + print("✅ Authentication with correct API key works") + else: + print(f"❌ Authentication failed with correct key: {response.status}") + return False + + # Test without API key (should fail if key is required) + headers_no_key = {'Content-Type': 'application/json'} + async with session.post( + f"{self.endpoint}/embeddings", + json=payload, + headers=headers_no_key, + timeout=aiohttp.ClientTimeout(total=30) + ) as response: + if response.status == 401: + print("✅ Authentication properly blocks requests without API key") + return True + else: + print(f"⚠️ No API key required (status: {response.status}) - service may be in no-auth mode") + return True # Still pass if no auth is configured + else: + print("⚠️ No API key configured - skipping authentication test") + return True + + except Exception as e: + print(f"❌ Authentication test error: {e}") + return False + + async def test_input_validation(self): + """Test input validation and size limits""" + print("🛡️ Testing input validation and size limits...") + + try: + headers = {'Content-Type': 'application/json'} + if self.api_key: + headers['X-API-Key'] = self.api_key + + async with aiohttp.ClientSession() as session: + # Test empty texts array + payload = {'texts': [], 'normalize': True} + async with session.post( + f"{self.endpoint}/embeddings", + json=payload, + headers=headers, + timeout=aiohttp.ClientTimeout(total=30) + ) as response: + if response.status == 422: + print("✅ Empty texts array properly rejected") + else: + print(f"⚠️ Empty texts array not rejected (status: {response.status})") + + # Test empty string + payload = {'texts': [''], 'normalize': True} + async with session.post( + f"{self.endpoint}/embeddings", + json=payload, + headers=headers, + timeout=aiohttp.ClientTimeout(total=30) + ) as response: + if response.status == 422: + print("✅ Empty string properly rejected") + else: + print(f"⚠️ Empty string not rejected (status: {response.status})") + + # Test oversized batch (51 items, limit is 50) + large_batch = ['test text'] * 51 + payload = {'texts': large_batch, 'normalize': True} + async with session.post( + f"{self.endpoint}/embeddings", + json=payload, + headers=headers, + timeout=aiohttp.ClientTimeout(total=30) + ) as response: + if response.status == 422: + print("✅ Oversized batch properly rejected") + else: + print(f"⚠️ Oversized batch not rejected (status: {response.status})") + + # Test oversized text (8193 chars, limit is 8192) + large_text = 'x' * 8193 + payload = {'texts': [large_text], 'normalize': True} + async with session.post( + f"{self.endpoint}/embeddings", + json=payload, + headers=headers, + timeout=aiohttp.ClientTimeout(total=30) + ) as response: + if response.status == 422: + print("✅ Oversized text properly rejected") + else: + print(f"⚠️ Oversized text not rejected (status: {response.status})") + + # Test valid input within limits + payload = {'texts': ['Valid patent text for embedding'], 'normalize': True} + async with session.post( + f"{self.endpoint}/embeddings", + json=payload, + headers=headers, + timeout=aiohttp.ClientTimeout(total=30) + ) as response: + if response.status == 200: + print("✅ Valid input accepted") + return True + else: + print(f"❌ Valid input rejected (status: {response.status})") + return False + + except Exception as e: + print(f"❌ Input validation test error: {e}") + return False + +async def main(): + print("🧪 PatentsBERTa Embedding Service Test Suite") + print("=" * 50) + + # Get endpoint from environment or command line + endpoint = os.getenv('PATENTSBERTA_ENDPOINT') + api_key = os.getenv('PATENTSBERTA_API_KEY') + + # Clean up API key (remove any trailing whitespace/newlines but preserve base64 padding) + if api_key: + api_key = api_key.strip() + + if len(sys.argv) > 1: + endpoint = sys.argv[1] + + if not endpoint: + print("❌ Please provide PatentsBERTa endpoint:") + print(" python test-patentsberta.py ") + print(" or set PATENTSBERTA_ENDPOINT environment variable") + sys.exit(1) + + print(f"🎯 Testing endpoint: {endpoint}") + if api_key: + print("🔑 Using API key authentication") + print(f"🔍 API key length: {len(api_key)} chars") + print(f"🔍 API key (first 10 chars): {api_key[:10]}...") + + # Initialize test suite + tester = PatentsBertaTestSuite(endpoint, api_key) + + # Run tests + tests = [ + ("Health Check", tester.test_health), + ("Info Endpoint", tester.test_info), + ("Authentication", tester.test_authentication), + ("Input Validation", tester.test_input_validation), + ("Single Embedding", tester.test_single_embedding), + ("Batch Embeddings", tester.test_batch_embeddings), + ("Patent Terminology", tester.test_patent_terminology), + ("Performance", tester.test_performance) + ] + + results = [] + + for test_name, test_func in tests: + print(f"\n🔍 Running: {test_name}") + print("-" * 30) + + try: + result = await test_func() + results.append((test_name, result)) + except Exception as e: + print(f"❌ {test_name} failed with exception: {e}") + results.append((test_name, False)) + + # Print summary + print("\n" + "=" * 50) + print("📊 Test Results Summary") + print("=" * 50) + + passed = 0 + total = len(results) + + for test_name, result in results: + status = "✅ PASS" if result else "❌ FAIL" + print(f"{status} {test_name}") + if result: + passed += 1 + + print(f"\n🎯 Overall: {passed}/{total} tests passed") + + if passed == total: + print("🎉 All tests passed! PatentsBERTa service is working correctly.") + sys.exit(0) + else: + print("⚠️ Some tests failed. Please check the service configuration.") + sys.exit(1) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/test_corpus_accuracy.py b/tests/test_corpus_accuracy.py new file mode 100644 index 0000000000..4e9a822d43 --- /dev/null +++ b/tests/test_corpus_accuracy.py @@ -0,0 +1,371 @@ +""" +Test Corpus Document Retrieval and Citation Accuracy + +This script tests if the RAG system correctly retrieves information from +corpus documents and provides accurate citations. +""" + +import asyncio +import aiohttp +import json +import sys +import os +from pathlib import Path +from typing import Dict, List, Any + +# Add backend to path +sys.path.insert(0, str(Path(__file__).parent.parent / "app" / "backend")) + +# Load environment +from load_azd_env import load_azd_env +load_azd_env() + + +async def test_corpus_query( + query: str, + expected_documents: List[str] = None, + backend_url: str = "http://localhost:50505", + use_corpus_only: bool = True +) -> Dict[str, Any]: + """ + Test a query to verify corpus document retrieval and citations. + + Args: + query: The question to ask + expected_documents: List of document names that should be cited (optional) + backend_url: Backend API URL + use_corpus_only: If True, force RAG-only mode (no web search) + """ + print("=" * 80) + print(f"TESTING CORPUS QUERY: {query}") + print("=" * 80) + + async with aiohttp.ClientSession() as session: + # Force RAG-only mode to test corpus retrieval + payload = { + "messages": [ + {"role": "user", "content": query} + ], + "context": { + "overrides": { + "mode": "rag" if use_corpus_only else "hybrid", # Force corpus-only + "retrieval_mode": "hybrid", # Use both text and vector search + "top": 5, # Get more results + "send_text_sources": True + } + } + } + + print(f"\n[INFO] Query: {query}") + print(f"[INFO] Mode: {'RAG-only (corpus)' if use_corpus_only else 'Hybrid (corpus + web)'}") + + try: + async with session.post( + f"{backend_url}/chat", + json=payload, + timeout=aiohttp.ClientTimeout(total=60) + ) as response: + if response.status == 200: + data = await response.json() + + # Extract response components + answer = data.get("message", {}).get("content", "") + context = data.get("context", {}) + data_points = context.get("data_points", {}) + text_sources = data_points.get("text", []) + citations = data_points.get("citations", []) + thoughts = context.get("thoughts", []) + + print(f"\n[RESPONSE]") + print(f"Answer ({len(answer)} characters):") + print("-" * 80) + print(answer) + print("-" * 80) + + # Analyze sources + corpus_sources = [] + web_sources = [] + + for source in text_sources: + if isinstance(source, str): + # Check if it's a web source (URL) or corpus source (filename) + if source.startswith("http://") or source.startswith("https://"): + web_sources.append(source) + else: + corpus_sources.append(source) + + print(f"\n[SOURCES ANALYSIS]") + print(f"Total text sources: {len(text_sources)}") + print(f"Corpus sources: {len(corpus_sources)}") + print(f"Web sources: {len(web_sources)}") + + # Show corpus sources + if corpus_sources: + print(f"\n[CORPUS SOURCES]") + for i, source in enumerate(corpus_sources[:5], 1): + # Extract document name + if ":" in source: + doc_name = source.split(":")[0] + content_preview = source.split(":", 1)[1][:150] if len(source.split(":", 1)) > 1 else "" + else: + doc_name = source + content_preview = "" + + print(f" {i}. {doc_name}") + if content_preview: + print(f" Preview: {content_preview}...") + else: + print(f"\n[WARNING] No corpus sources found!") + + # Show web sources + if web_sources: + print(f"\n[WEB SOURCES]") + for i, source in enumerate(web_sources[:3], 1): + url = source.split(":")[0] if ":" in source else source + print(f" {i}. {url}") + + # Show citations + print(f"\n[CITATIONS]") + print(f"Total citations: {len(citations)}") + corpus_citations = [c for c in citations if not c.startswith("http")] + web_citations = [c for c in citations if c.startswith("http")] + + if corpus_citations: + print(f"Corpus citations ({len(corpus_citations)}):") + for i, cit in enumerate(corpus_citations[:10], 1): + print(f" {i}. {cit}") + + if web_citations: + print(f"Web citations ({len(web_citations)}):") + for i, cit in enumerate(web_citations[:5], 1): + print(f" {i}. {cit}") + + # Verify expected documents + if expected_documents: + print(f"\n[VERIFICATION]") + print(f"Expected documents: {expected_documents}") + found_docs = [] + for expected_doc in expected_documents: + # Check if expected doc is in any citation or source + for cit in citations: + if expected_doc.lower() in cit.lower(): + found_docs.append(expected_doc) + break + if expected_doc not in found_docs: + for source in corpus_sources: + if expected_doc.lower() in source.lower(): + found_docs.append(expected_doc) + break + + if len(found_docs) == len(expected_documents): + print(f"[PASS] All expected documents found: {found_docs}") + else: + missing = set(expected_documents) - set(found_docs) + print(f"[PARTIAL] Found: {found_docs}") + print(f"[MISSING] Not found: {missing}") + + # Check if answer uses corpus citations + print(f"\n[ACCURACY CHECK]") + if corpus_citations: + # Extract citations actually used in the answer text + citations_used_in_answer = [] + answer_lower = answer.lower() + + for cit in corpus_citations: + # Check if citation appears in answer (format: [doc.pdf#page=1]) + cit_in_answer = f"[{cit}]" in answer or cit in answer + if cit_in_answer: + citations_used_in_answer.append(cit) + + # Check if all citations are relevant + if expected_documents: + expected_doc_names = [doc.split("#")[0].split("/")[-1] if "#" in doc else doc.split("/")[-1] for doc in expected_documents] + irrelevant_citations = [] + + for cit in citations_used_in_answer: + cit_doc_name = cit.split("#")[0].split("/")[-1] if "#" in cit else cit.split("/")[-1] + # Check if this citation matches any expected document + is_relevant = any(exp_doc.lower() in cit_doc_name.lower() or cit_doc_name.lower() in exp_doc.lower() for exp_doc in expected_doc_names) + if not is_relevant: + irrelevant_citations.append(cit) + + if irrelevant_citations: + print(f"[FAIL] Answer includes irrelevant citations: {irrelevant_citations}") + print(f" Expected only: {expected_documents}") + print(f" Citations in answer: {citations_used_in_answer}") + else: + print(f"[PASS] Answer only cites relevant documents") + print(f" Citations used: {citations_used_in_answer}") + else: + # No expected documents, just check if citations are in answer + if citations_used_in_answer: + print(f"[PASS] Answer includes corpus citations: {citations_used_in_answer}") + else: + print(f"[WARN] Answer may not be citing corpus sources properly") + print(f" Available citations: {corpus_citations[:3]}") + else: + print(f"[WARN] No corpus citations found - answer may be from web or generic") + + # Check for "I don't know" + if "don't know" in answer.lower() or "i don't know" in answer.lower(): + if corpus_sources: + print(f"[WARN] Answer says 'I don't know' but corpus sources were found") + print(f" This may indicate the sources don't contain relevant information") + else: + print(f"[INFO] Answer says 'I don't know' and no corpus sources found") + + return { + "query": query, + "answer": answer, + "corpus_sources": corpus_sources, + "web_sources": web_sources, + "corpus_citations": corpus_citations, + "web_citations": web_citations, + "text_sources": text_sources, + "citations": citations, + "thoughts": thoughts + } + else: + error_text = await response.text() + print(f"\n[ERROR] Backend returned status {response.status}") + print(f"Error: {error_text[:500]}") + return None + except Exception as e: + print(f"\n[ERROR] Request failed: {e}") + import traceback + traceback.print_exc() + return None + + +async def list_indexed_documents(backend_url: str = "http://localhost:50505") -> List[str]: + """Try to get a list of documents in the index.""" + print("\n" + "=" * 80) + print("ATTEMPTING TO LIST INDEXED DOCUMENTS") + print("=" * 80) + + # Try asking what documents are in the index + result = await test_corpus_query( + "What documents are in the knowledge base? List all document names.", + backend_url=backend_url, + use_corpus_only=True + ) + + if result and result.get("corpus_sources"): + # Extract document names from sources + doc_names = set() + for source in result["corpus_sources"]: + if ":" in source: + doc_name = source.split(":")[0].strip() + if doc_name and not doc_name.startswith("http"): + doc_names.add(doc_name) + + if doc_names: + print(f"\n[FOUND DOCUMENTS]") + for i, doc in enumerate(sorted(doc_names), 1): + print(f" {i}. {doc}") + return list(doc_names) + + return [] + + +async def main(): + """Run corpus accuracy tests.""" + backend_url = os.getenv("BACKEND_URL", "http://localhost:50505") + + print("\n" + "=" * 80) + print("CORPUS DOCUMENT RETRIEVAL AND CITATION ACCURACY TEST") + print("=" * 80) + + # First, try to discover what documents are indexed + print("\n[STEP 1] Discovering indexed documents...") + indexed_docs = await list_indexed_documents(backend_url) + + # Test queries - customize these based on your documents + # Format: {"query": "your question", "expected_docs": ["Document1.pdf", "Document2.pdf"], "description": "what this tests"} + test_queries = [ + { + "query": "What documents are in the knowledge base?", + "expected_docs": None, # Will be set after discovery + "description": "List all documents" + }, + # ADD YOUR CUSTOM TEST QUERIES HERE: + # Example tests based on your documents: + { + "query": "What is the code review process?", + "expected_docs": ["Code_Review_Checklist.pdf"], + "description": "Test retrieval from Code Review Checklist" + }, + { + "query": "What is the release validation process?", + "expected_docs": ["Release_Validation_Process.pdf"], + "description": "Test retrieval from Release Validation Process" + }, + { + "query": "Summarize what is code review documents saying", + "expected_docs": ["Code_Review_Checklist.pdf"], + "description": "Test summarization of Code Review Checklist" + }, + # Add more queries that match content in your documents: + # { + # "query": "What are the requirements for X?", + # "expected_docs": ["YourDocument.pdf"], + # "description": "Test specific information retrieval" + # }, + ] + + # If we found documents, add a test query + if indexed_docs: + # Use the first document as an example + first_doc = indexed_docs[0] + doc_topic = first_doc.replace(".pdf", "").replace("_", " ").replace("-", " ") + test_queries.append({ + "query": f"What information is in {first_doc}?", + "expected_docs": [first_doc], + "description": f"Test retrieval from {first_doc}" + }) + + print(f"\n[STEP 2] Running {len(test_queries)} test queries...") + print() + + results = [] + for i, test in enumerate(test_queries, 1): + print(f"\n[TEST {i}/{len(test_queries)}] {test['description']}") + result = await test_corpus_query( + test["query"], + expected_documents=test.get("expected_docs"), + backend_url=backend_url, + use_corpus_only=True # Test corpus-only first + ) + results.append(result) + print() + + # Summary + print("\n" + "=" * 80) + print("TEST SUMMARY") + print("=" * 80) + + total_tests = len(results) + tests_with_corpus = sum(1 for r in results if r and r.get("corpus_sources")) + tests_with_citations = sum(1 for r in results if r and r.get("corpus_citations")) + + print(f"Total tests: {total_tests}") + print(f"Tests with corpus sources: {tests_with_corpus}/{total_tests}") + print(f"Tests with corpus citations: {tests_with_citations}/{total_tests}") + + if tests_with_corpus < total_tests: + print(f"\n[WARNING] {total_tests - tests_with_corpus} tests did not retrieve corpus sources") + print(" This may indicate:") + print(" - Documents are not indexed") + print(" - Queries don't match document content") + print(" - Search index needs to be rebuilt") + + print("\n[RECOMMENDATIONS]") + print("1. Verify your documents are indexed in Azure AI Search") + print("2. Check that queries match the content in your documents") + print("3. Review the corpus sources shown above to verify retrieval accuracy") + print("4. Check citations in answers to ensure they point to correct documents") + + +if __name__ == "__main__": + asyncio.run(main()) + diff --git a/tests/test_functionality.py b/tests/test_functionality.py new file mode 100644 index 0000000000..423bc3c8e3 --- /dev/null +++ b/tests/test_functionality.py @@ -0,0 +1,426 @@ +""" +Complete Functionality Test Suite + +Tests the actual functionality of the application including: +- Backend API endpoints +- RAG responses with citations +- OCR functionality (if enabled) +- Web search (if enabled) +- Agents service (if running) +- Response accuracy +""" + +import asyncio +import aiohttp +import json +import sys +import os +from pathlib import Path +from typing import Dict, Any, Optional, List + +# Add backend to path +sys.path.insert(0, str(Path(__file__).parent.parent / "app" / "backend")) + +# Load environment +from load_azd_env import load_azd_env +load_azd_env() + +class FunctionalityTester: + """Test application functionality end-to-end.""" + + def __init__(self, backend_url: str = "http://localhost:50505", agents_url: str = "http://localhost:8000"): + self.backend_url = backend_url.rstrip('/') + self.agents_url = agents_url.rstrip('/') + self.session: Optional[aiohttp.ClientSession] = None + self.results: List[Dict[str, Any]] = [] + + async def __aenter__(self): + self.session = aiohttp.ClientSession() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + if self.session: + await self.session.close() + + async def test_backend_health(self) -> bool: + """Test backend health endpoint.""" + print("\n=== Testing Backend Health ===") + try: + async with self.session.get(f"{self.backend_url}/health", timeout=aiohttp.ClientTimeout(total=10)) as response: + if response.status == 200: + data = await response.json() + print(f"[PASS] Backend is healthy") + print(f" Status: {data.get('status')}") + if 'dependencies' in data: + deps = data['dependencies'] + healthy = sum(1 for d in deps.values() if d.get('ok', False)) + total = len(deps) + print(f" Dependencies: {healthy}/{total} healthy") + return True + else: + print(f"[FAIL] Backend returned status {response.status}") + return False + except aiohttp.ClientConnectorError: + print(f"[SKIP] Backend not running at {self.backend_url}") + return False + except Exception as e: + print(f"[FAIL] Backend health check failed: {e}") + return False + + async def test_backend_config(self) -> bool: + """Test backend config endpoint.""" + print("\n=== Testing Backend Config ===") + try: + async with self.session.get(f"{self.backend_url}/config", timeout=aiohttp.ClientTimeout(total=10)) as response: + if response.status == 200: + data = await response.json() + print(f"[PASS] Config endpoint working") + print(f" Features: {', '.join(data.get('features', []))}") + return True + else: + print(f"[FAIL] Config returned status {response.status}") + return False + except Exception as e: + print(f"[FAIL] Config check failed: {e}") + return False + + async def test_chat_endpoint(self, query: str = "What is machine learning?") -> bool: + """Test chat endpoint with a real query.""" + print(f"\n=== Testing Chat Endpoint ===") + print(f"Query: {query}") + try: + payload = { + "messages": [ + {"role": "user", "content": query} + ], + "context": { + "overrides": { + "retrieval_mode": "hybrid", + "top": 3 + } + } + } + + async with self.session.post( + f"{self.backend_url}/chat", + json=payload, + timeout=aiohttp.ClientTimeout(total=60) + ) as response: + if response.status == 200: + data = await response.json() + answer = data.get("message", {}).get("content", "") + citations = data.get("context", {}).get("data_points", {}).get("citations", []) + + print(f"[PASS] Chat endpoint working") + print(f" Answer length: {len(answer)} characters") + print(f" Citations: {len(citations)}") + + if answer: + print(f" Answer preview: {answer[:100]}...") + else: + print(f" [WARN] Empty answer") + + if citations: + print(f" Citation examples: {citations[:2]}") + else: + print(f" [WARN] No citations found") + + # Validate response structure + has_answer = bool(answer) + has_citations = len(citations) > 0 + + return has_answer + elif response.status == 401: + print(f"[SKIP] Chat endpoint requires authentication") + return False + else: + error_text = await response.text() + print(f"[FAIL] Chat returned status {response.status}: {error_text[:200]}") + return False + except Exception as e: + print(f"[FAIL] Chat test failed: {e}") + return False + + async def test_ask_endpoint(self, question: str = "What is artificial intelligence?") -> bool: + """Test ask endpoint.""" + print(f"\n=== Testing Ask Endpoint ===") + print(f"Question: {question}") + try: + # Ask endpoint expects "messages" format, not "question" + payload = { + "messages": [ + {"role": "user", "content": question} + ], + "context": { + "overrides": { + "retrieval_mode": "hybrid", + "top": 3 + } + } + } + + async with self.session.post( + f"{self.backend_url}/ask", + json=payload, + timeout=aiohttp.ClientTimeout(total=60) + ) as response: + if response.status == 200: + data = await response.json() + # Ask endpoint returns same format as chat + answer = data.get("message", {}).get("content", "") + citations = data.get("context", {}).get("data_points", {}).get("citations", []) + + print(f"[PASS] Ask endpoint working") + print(f" Answer length: {len(answer)} characters") + print(f" Citations: {len(citations)}") + + if answer: + print(f" Answer preview: {answer[:100]}...") + else: + print(f" [WARN] Empty answer") + + return bool(answer) + elif response.status == 401: + print(f"[SKIP] Ask endpoint requires authentication") + return False + else: + error_text = await response.text() + print(f"[FAIL] Ask returned status {response.status}: {error_text[:200]}") + return False + except Exception as e: + print(f"[FAIL] Ask test failed: {e}") + import traceback + traceback.print_exc() + return False + + async def test_agents_health(self) -> bool: + """Test agents service health.""" + print("\n=== Testing Agents Service Health ===") + try: + async with self.session.get(f"{self.agents_url}/api/health", timeout=aiohttp.ClientTimeout(total=10)) as response: + if response.status == 200: + data = await response.json() + print(f"[PASS] Agents service is healthy") + print(f" Status: {data.get('status')}") + if 'services' in data: + services = data['services'] + print(f" Services: {', '.join(services.keys())}") + return True + elif response.status == 404: + print(f"[SKIP] Agents service endpoint not found (may be running on different port or path)") + print(f"[INFO] Agents service may not be running or URL is incorrect") + return True # Don't fail the test if agents isn't running + else: + print(f"[WARN] Agents returned status {response.status}") + return True # Don't fail the test + except aiohttp.ClientConnectorError: + print(f"[SKIP] Agents service not running at {self.agents_url}") + print(f"[INFO] This is optional - agents service is not required for basic functionality") + return True # Don't fail the test if agents isn't running + except Exception as e: + print(f"[WARN] Agents health check failed: {e}") + return True # Don't fail the test + + async def test_ocr_functionality(self) -> bool: + """Test OCR functionality if enabled.""" + print("\n=== Testing OCR Functionality ===") + try: + from config import OCR_PROVIDER, OCR_ON_INGEST, OLLAMA_OCR_MODEL, OLLAMA_BASE_URL + from services.ocr_service import OCRService, OCRProviderType + + if OCR_PROVIDER == 'none': + print("[SKIP] OCR is disabled (OCR_PROVIDER=none)") + print(f"[INFO] OCR Model configured: {OLLAMA_OCR_MODEL} (will be used when OCR is enabled)") + if OLLAMA_OCR_MODEL == "llava:7b": + print("[PASS] OCR model is correctly set to llava:7b") + else: + print(f"[INFO] Current OCR model: {OLLAMA_OCR_MODEL}") + print("[INFO] To enable OCR, set OCR_PROVIDER=ollama or OCR_PROVIDER=azure_document_intelligence") + return True + + print(f"OCR Provider: {OCR_PROVIDER}") + print(f"OCR on Ingest: {OCR_ON_INGEST}") + + if OCR_PROVIDER == 'ollama': + print(f"Ollama Base URL: {OLLAMA_BASE_URL}") + print(f"Ollama OCR Model: {OLLAMA_OCR_MODEL}") + if OLLAMA_OCR_MODEL != "llava:7b": + print(f"[WARN] Expected llava:7b, but found {OLLAMA_OCR_MODEL}") + else: + print("[PASS] OCR model is correctly set to llava:7b") + + service = OCRService() + if not service.is_enabled(): + print("[SKIP] OCR service not enabled (check configuration)") + return True + + # Test service initialization + print("[PASS] OCR service is configured and ready") + if OCR_PROVIDER == 'ollama': + print(f"[INFO] Using Ollama model: {OLLAMA_OCR_MODEL}") + print("[NOTE] To fully test OCR, upload an image with text") + print("[NOTE] Ensure Ollama is running: ollama serve") + + return True + except Exception as e: + print(f"[FAIL] OCR test failed: {e}") + import traceback + traceback.print_exc() + return False + + async def test_web_search_functionality(self) -> bool: + """Test web search functionality if enabled.""" + print("\n=== Testing Web Search Functionality ===") + try: + from config import ENABLE_WEB_SEARCH, SERPER_API_KEY + + if not ENABLE_WEB_SEARCH: + print("[SKIP] Web search is disabled") + return True + + if not SERPER_API_KEY: + print("[WARN] ENABLE_WEB_SEARCH=true but SERPER_API_KEY not set") + return False + + print("[PASS] Web search is configured") + print("[NOTE] Web search will be used when query requires current information") + + return True + except Exception as e: + print(f"[FAIL] Web search test failed: {e}") + return False + + async def test_cache_functionality(self) -> bool: + """Test cache functionality.""" + print("\n=== Testing Cache Functionality ===") + try: + from config import REDIS_URL + from services.cache import create_cache + + cache = await create_cache() + + # Test set/get + await cache.set("test_functionality_key", "test_value", ttl_s=60) + value = await cache.get("test_functionality_key") + + if value == "test_value": + cache_type = "Redis" if REDIS_URL else "In-memory" + print(f"[PASS] Cache working ({cache_type})") + await cache.close() + return True + else: + print(f"[FAIL] Cache returned unexpected value: {value}") + await cache.close() + return False + except Exception as e: + print(f"[FAIL] Cache test failed: {e}") + return False + + async def test_embedding_router(self) -> bool: + """Test embedding router functionality.""" + print("\n=== Testing Embedding Router ===") + try: + from services.embedding_router import EmbeddingRouter + import os + + # Get embedding deployment from environment (not from config.py) + emb_deployment = os.getenv("AZURE_OPENAI_EMB_DEPLOYMENT", "text-embedding-3-large") + + router = EmbeddingRouter(baseline_deployment=emb_deployment) + + # Test routing + technical_text = "This patent describes a novel method for processing data." + model = router.select_model(technical_text) + + print(f"[PASS] Embedding router working") + print(f" Selected model: {model.value}") + print(f" Baseline deployment: {emb_deployment}") + + return True + except Exception as e: + print(f"[FAIL] Embedding router test failed: {e}") + import traceback + traceback.print_exc() + return False + + async def run_all_tests(self): + """Run all functionality tests.""" + print("=" * 70) + print("COMPLETE FUNCTIONALITY TEST SUITE") + print("=" * 70) + print(f"Backend URL: {self.backend_url}") + print(f"Agents URL: {self.agents_url}") + print("=" * 70) + + tests = [ + ("Backend Health", self.test_backend_health), + ("Backend Config", self.test_backend_config), + ("Cache Functionality", self.test_cache_functionality), + ("Embedding Router", self.test_embedding_router), + ("OCR Functionality", self.test_ocr_functionality), + ("Web Search Functionality", self.test_web_search_functionality), + ("Chat Endpoint", lambda: self.test_chat_endpoint("What is RAG?")), + ("Ask Endpoint", lambda: self.test_ask_endpoint("Explain vector search")), + ("Agents Health", self.test_agents_health), + ] + + results = [] + for test_name, test_func in tests: + try: + result = await test_func() + results.append((test_name, result)) + except Exception as e: + print(f"[ERROR] {test_name} failed with exception: {e}") + results.append((test_name, False)) + + # Summary + print("\n" + "=" * 70) + print("TEST SUMMARY") + print("=" * 70) + + passed = sum(1 for _, result in results if result) + total = len(results) + skipped = sum(1 for _, result in results if result is None) + + for test_name, result in results: + if result is None: + status = "[SKIP]" + elif result: + status = "[PASS]" + else: + status = "[FAIL]" + print(f"{status} {test_name}") + + print(f"\nResults: {passed}/{total} passed, {skipped} skipped, {total - passed - skipped} failed") + + if passed == total - skipped: + print("\n[SUCCESS] All applicable tests passed!") + else: + print(f"\n[WARNING] {total - passed - skipped} test(s) failed") + print("\nTo fix issues:") + print(" 1. Ensure backend is running: uvicorn main:app --reload") + print(" 2. Check environment variables are set correctly") + print(" 3. Verify Azure services are accessible") + print(" 4. Check service logs for errors") + + return passed == total - skipped + + +async def main(): + """Main entry point.""" + import argparse + + parser = argparse.ArgumentParser(description="Test application functionality") + parser.add_argument("--backend-url", default="http://localhost:50505", help="Backend service URL") + parser.add_argument("--agents-url", default="http://localhost:8000", help="Agents service URL") + + args = parser.parse_args() + + async with FunctionalityTester(args.backend_url, args.agents_url) as tester: + success = await tester.run_all_tests() + sys.exit(0 if success else 1) + + +if __name__ == "__main__": + asyncio.run(main()) + diff --git a/tests/test_rag_debug.py b/tests/test_rag_debug.py new file mode 100644 index 0000000000..ca0d3cf8b7 --- /dev/null +++ b/tests/test_rag_debug.py @@ -0,0 +1,190 @@ +""" +Debug RAG Responses + +Tests and debugs why RAG is returning "I don't know" responses. +""" + +import asyncio +import aiohttp +import json +import sys +import os +from pathlib import Path + +# Add backend to path +sys.path.insert(0, str(Path(__file__).parent.parent / "app" / "backend")) + +# Load environment +from load_azd_env import load_azd_env +load_azd_env() + + +async def debug_rag_query(query: str, backend_url: str = "http://localhost:50505"): + """Debug a RAG query to see what's happening.""" + print("=" * 70) + print(f"DEBUGGING RAG QUERY: {query}") + print("=" * 70) + + async with aiohttp.ClientSession() as session: + payload = { + "messages": [ + {"role": "user", "content": query} + ], + "context": { + "overrides": { + "retrieval_mode": "hybrid", + "top": 5, # Get more results + "send_text_sources": True + } + } + } + + print(f"\n1. Sending query to backend...") + print(f" URL: {backend_url}/chat") + print(f" Query: {query}") + + try: + async with session.post( + f"{backend_url}/chat", + json=payload, + timeout=aiohttp.ClientTimeout(total=60) + ) as response: + if response.status == 200: + data = await response.json() + + # Extract response components + answer = data.get("message", {}).get("content", "") + context = data.get("context", {}) + data_points = context.get("data_points", {}) + text_sources = data_points.get("text", []) + citations = data_points.get("citations", []) + thoughts = context.get("thoughts", []) + + print(f"\n2. RESPONSE ANALYSIS:") + print(f" Answer: {answer}") + print(f" Answer length: {len(answer)} characters") + print(f" Citations found: {len(citations)}") + print(f" Text sources retrieved: {len(text_sources)}") + + # Show search query + if thoughts: + print(f"\n3. SEARCH PROCESS:") + for i, thought in enumerate(thoughts, 1): + title = thought.get("title", "") + description = thought.get("description", "") + if "search query" in title.lower() or "query" in title.lower(): + print(f" {i}. {title}: {description}") + + # Show retrieved documents + if text_sources: + print(f"\n4. RETRIEVED DOCUMENTS:") + for i, source in enumerate(text_sources[:3], 1): # Show first 3 + # Handle both dict and string formats + if isinstance(source, dict): + sourcepage = source.get("sourcepage", "unknown") + content = source.get("content", "") + else: + # String format: "filename.pdf#page=1: content here" + if ":" in str(source): + parts = str(source).split(":", 1) + sourcepage = parts[0] if parts else "unknown" + content = parts[1] if len(parts) > 1 else "" + else: + sourcepage = "unknown" + content = str(source) + + content_preview = content[:200] if content else "(empty)" + print(f" {i}. {sourcepage}") + print(f" Content preview: {content_preview}...") + print(f" Content length: {len(content)} characters") + + # Check if content is relevant + query_lower = query.lower() + content_lower = content.lower() + query_words = [w for w in query_lower.split() if len(w) > 2] + if query_lower in content_lower or any(word in content_lower for word in query_words): + print(f" [RELEVANT] Contains query terms") + else: + print(f" [NOT RELEVANT] Doesn't contain query terms") + else: + print(f"\n4. RETRIEVED DOCUMENTS: None found") + + # Analyze why "I don't know" + if "don't know" in answer.lower() or "i don't know" in answer.lower(): + print(f"\n5. WHY 'I DON'T KNOW'?") + if not text_sources: + print(" ❌ No documents retrieved from search") + elif len(text_sources) > 0: + print(" ⚠️ Documents retrieved but content may not be relevant") + print(" ⚠️ LLM is following prompt: 'say you don't know if sources don't contain answer'") + + # Check content relevance + relevant_count = 0 + for source in text_sources: + # Handle both dict and string formats + if isinstance(source, dict): + content = source.get("content", "").lower() + else: + # String format: "filename.pdf#page=1: content here" + if ":" in str(source): + content = str(source).split(":", 1)[1].lower() if len(str(source).split(":", 1)) > 1 else "" + else: + content = str(source).lower() + + query_words = [w for w in query.lower().split() if len(w) > 3] + if any(word in content for word in query_words): + relevant_count += 1 + + print(f" 📊 Relevance: {relevant_count}/{len(text_sources)} sources contain query terms") + + if relevant_count == 0: + print(" 💡 SOLUTION: Index documents that contain information about the query topic") + print(" 💡 Or try a different query related to your indexed documents") + + # Show citations + if citations: + print(f"\n6. CITATIONS:") + for i, citation in enumerate(citations[:5], 1): + print(f" {i}. {citation}") + + return { + "answer": answer, + "citations": citations, + "text_sources": text_sources, + "thoughts": thoughts + } + else: + error_text = await response.text() + print(f"\n[ERROR] Backend returned status {response.status}") + print(f"Error: {error_text[:500]}") + return None + except Exception as e: + print(f"\n[ERROR] Request failed: {e}") + import traceback + traceback.print_exc() + return None + + +async def main(): + """Run debug tests.""" + queries = [ + "What is RAG?", + "Explain vector search", + "What documents are in the index?" + ] + + for query in queries: + result = await debug_rag_query(query) + print("\n" + "=" * 70 + "\n") + + if result and result.get("text_sources"): + # Try a query that might match the actual documents + print("💡 TIP: Try asking about topics in your indexed documents:") + print(" - Check what documents you have indexed") + print(" - Ask questions related to those document topics") + break + + +if __name__ == "__main__": + asyncio.run(main()) +