final fixes, ci + workflow

Author: Zvi Fried

examples/rate_limit_demo.py

@@ -7,7 +7,6 @@
 """
 
 import asyncio
-import os
 from unittest.mock import patch
 
 import litellm
@@ -20,7 +19,7 @@ async def demo_rate_limit_handling():
     """Demonstrate rate limit handling with exponential backoff."""
     print("🚀 Rate Limit Handling Demo")
     print("=" * 50)
-    
+
     # Create a test LLM configuration
     config = LLMConfig(
         api_key="demo-key",
@@ -29,69 +28,69 @@ async def demo_rate_limit_handling():
         max_tokens=1000,
         temperature=0.1,
     )
-    
+
     client = LLMClient(config)
-    
+
     print("📝 Test 1: Successful retry after rate limit errors")
     print("-" * 50)
-    
+
     # Mock response for successful case
     from unittest.mock import MagicMock
     mock_response = MagicMock()
     mock_response.choices = [MagicMock()]
     mock_response.choices[0].message.content = "Success after retries!"
-    
+
     # Simulate rate limit errors followed by success
     with patch.object(client, "_litellm") as mock_litellm:
         mock_litellm.completion.side_effect = [
             litellm.RateLimitError("Rate limit exceeded", "openai", "gpt-4"),
             litellm.RateLimitError("Rate limit exceeded", "openai", "gpt-4"),
             mock_response,  # Success on third attempt
         ]
-        
+
         messages = [{"role": "user", "content": "Hello, world!"}]
-        
+
         try:
             result = await client.generate_text(messages)
             print(f"✅ Success: {result}")
             print(f"📊 Total attempts: {mock_litellm.completion.call_count}")
         except Exception as e:
             print(f"❌ Failed: {e}")
-    
+
     print("\n📝 Test 2: Rate limit exhaustion (all retries fail)")
     print("-" * 50)
-    
+
     # Simulate persistent rate limit errors
     with patch.object(client, "_litellm") as mock_litellm:
         mock_litellm.completion.side_effect = litellm.RateLimitError(
             "Rate limit exceeded", "openai", "gpt-4"
         )
-        
+
         messages = [{"role": "user", "content": "This will fail"}]
-        
+
         try:
             result = await client.generate_text(messages)
             print(f"✅ Unexpected success: {result}")
         except Exception as e:
             print(f"❌ Expected failure after retries: {e}")
             print(f"📊 Total attempts: {mock_litellm.completion.call_count}")
-    
+
     print("\n📝 Test 3: Non-rate-limit error (no retries)")
     print("-" * 50)
-    
+
     # Simulate a different type of error
     with patch.object(client, "_litellm") as mock_litellm:
         mock_litellm.completion.side_effect = ValueError("Invalid input")
-        
+
         messages = [{"role": "user", "content": "This will fail immediately"}]
-        
+
         try:
             result = await client.generate_text(messages)
             print(f"✅ Unexpected success: {result}")
         except Exception as e:
             print(f"❌ Expected immediate failure: {e}")
             print(f"📊 Total attempts: {mock_litellm.completion.call_count}")
-    
+
     print("\n🎯 Rate Limit Configuration")
     print("-" * 50)
     print("• Max retries: 5 attempts")
@@ -100,7 +99,7 @@ async def demo_rate_limit_handling():
     print("• Exponential base: 2.0")
     print("• Jitter: Enabled (±20%)")
     print("\nDelay pattern: ~2s, ~4s, ~8s, ~16s, ~32s")
-    
+
     print("\n✨ Demo completed!")
 
 

pyproject.toml

@@ -143,6 +143,8 @@ addopts = [
     "--strict-config",
     "--verbose",
 ]
+# No filterwarnings - we want to catch any remaining Pydantic issues
+# filterwarnings = []
 markers = [
     "slow: marks tests as slow (deselect with '-m \"not slow\"')",
     "integration: marks tests as integration tests",

scripts/test_dynamic_urls.py

@@ -12,9 +12,6 @@
     URLValidationResult,
 )
 from src.mcp_as_a_judge.models.task_metadata import TaskMetadata
-from src.mcp_as_a_judge.research_requirements_analyzer import (
-    ResearchRequirementsAnalyzer,
-)
 
 
 def test_models():
@@ -72,12 +69,9 @@ async def test_analyzer_mock():
     """Test the ResearchRequirementsAnalyzer with a mock scenario."""
     print("\n🧪 Testing ResearchRequirementsAnalyzer...")
 
-    analyzer = ResearchRequirementsAnalyzer()
-
     # Test scenario
     task_title = "Implement JWT Authentication System"
     task_description = "Build secure user authentication with JWT tokens, password hashing, and session management"
-    user_requirements = "Users should be able to register, login, logout securely with proper token validation"
 
     print(f"📋 Task: {task_title}")
     print(f"📋 Description: {task_description}")

src/mcp_as_a_judge/elicitation/__init__.py

@@ -6,16 +6,19 @@
 fallback prompts, etc.).
 """
 
-from mcp_as_a_judge.elicitation.factory import ElicitationProviderFactory, elicitation_provider
+from mcp_as_a_judge.elicitation.factory import (
+    ElicitationProviderFactory,
+    elicitation_provider,
+)
 from mcp_as_a_judge.elicitation.fallback_provider import FallbackElicitationProvider
 from mcp_as_a_judge.elicitation.interface import ElicitationProvider, ElicitationResult
 from mcp_as_a_judge.elicitation.mcp_provider import MCPElicitationProvider
 
 __all__ = [
     "ElicitationProvider",
-    "ElicitationResult", 
-    "MCPElicitationProvider",
-    "FallbackElicitationProvider",
     "ElicitationProviderFactory",
+    "ElicitationResult",
+    "FallbackElicitationProvider",
+    "MCPElicitationProvider",
     "elicitation_provider",
 ]

src/mcp_as_a_judge/elicitation/factory.py

@@ -61,10 +61,10 @@ async def elicit_user_input(
 
     def get_available_providers(self, ctx: Context) -> dict[str, dict[str, object]]:
         """Get information about all available providers.
-        
+
         Args:
             ctx: MCP context for capability checking
-            
+
         Returns:
             Dictionary with provider availability information
         """

src/mcp_as_a_judge/elicitation/fallback_provider.py

@@ -25,12 +25,12 @@ async def _elicit(
         self, message: str, schema: type[BaseModel], ctx: Context
     ) -> ElicitationResult:
         """Generate a fallback message for the AI assistant to prompt the user.
-        
+
         Args:
             message: Original message to display to the user
             schema: Pydantic model schema defining expected fields
             ctx: MCP context
-            
+
         Returns:
             ElicitationResult with success=False and a formatted message
         """

src/mcp_as_a_judge/elicitation/interface.py

@@ -22,7 +22,7 @@ def __init__(
         message: str | None = None,
     ):
         """Initialize elicitation result.
-        
+
         Args:
             success: Whether the elicitation was successful
             data: Data returned from successful elicitation
@@ -44,12 +44,12 @@ async def elicit(
         This is the public interface method that can contain common logic
         for all providers (logging, validation, etc.) and calls the internal
         implementation method.
-        
+
         Args:
             message: Message to display to the user
             schema: Pydantic model schema defining expected fields
             ctx: MCP context
-            
+
         Returns:
             ElicitationResult with success status and data/message
         """
@@ -69,12 +69,12 @@ async def _elicit(
         self, message: str, schema: type[BaseModel], ctx: Context
     ) -> ElicitationResult:
         """Internal elicitation implementation - override this in subclasses.
-        
+
         Args:
             message: Message to display to the user
             schema: Pydantic model schema defining expected fields
             ctx: MCP context
-            
+
         Returns:
             ElicitationResult with success status and data/message
         """

src/mcp_as_a_judge/elicitation/mcp_provider.py

@@ -23,12 +23,12 @@ async def _elicit(
         self, message: str, schema: type[BaseModel], ctx: Context
     ) -> ElicitationResult:
         """Elicit user input using MCP elicitation.
-        
+
         Args:
             message: Message to display to the user
             schema: Pydantic model schema defining expected fields
             ctx: MCP context
-            
+
         Returns:
             ElicitationResult with success status and data/message
         """

src/mcp_as_a_judge/llm/llm_client.py

@@ -20,9 +20,9 @@
 from mcp_as_a_judge.core.logging_config import get_logger
 from mcp_as_a_judge.llm.llm_integration import LLMConfig, LLMVendor
 
-# Set global drop_params to handle model-specific parameter restrictions
-# This is especially important for GPT-5 models which don't support temperature at all
+
 litellm.drop_params = True
+litellm.suppress_debug_info = True
 
 # Set up logger
 logger = get_logger(__name__)

src/mcp_as_a_judge/messaging/interface.py

@@ -10,7 +10,11 @@
 
 from pydantic import BaseModel, Field
 
-from mcp_as_a_judge.core.constants import DEFAULT_TEMPERATURE, DEFAULT_TIMEOUT, MAX_TOKENS
+from mcp_as_a_judge.core.constants import (
+    DEFAULT_TEMPERATURE,
+    DEFAULT_TIMEOUT,
+    MAX_TOKENS,
+)
 
 
 class Message(BaseModel):