Merge branch 'UT-10888-backend' of https://github.com/microsoft/Multi-Agent-Custom-Automation-Engine-Solution-Accelerator into UT-10888-backend

Author: Harmanpreet Kaur

.github/workflows/test.yml

@@ -37,7 +37,6 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -r src/backend/requirements.txt
-          pip install pytest-cov
 
       - name: Check if test files exist
         id: check_tests
@@ -49,7 +48,6 @@ jobs:
             echo "Test files found, running tests."
             echo "skip_tests=false" >> $GITHUB_ENV
           fi
-
       - name: Run tests with coverage
         if: env.skip_tests == 'false'
         run: |
@@ -58,4 +56,4 @@ jobs:
       - name: Skip coverage report if no tests
         if: env.skip_tests == 'true'
         run: |
-          echo "Skipping coverage report because no tests were found."
+          echo "Skipping coverage report because no tests were found."

src/backend/requirements.txt

@@ -14,3 +14,8 @@ opentelemetry-instrumentation-fastapi
 opentelemetry-instrumentation-openai
 opentelemetry-exporter-otlp-proto-http
 opentelemetry-exporter-otlp-proto-grpc
+
+# Testing tools
+pytest>=8.2,<9  # Compatible version for pytest-asyncio
+pytest-asyncio==0.24.0
+pytest-cov==5.0.0

src/backend/tests/agents/test_human.py

@@ -0,0 +1,97 @@
+"""
+Test cases for HumanAgent class in the backend agents module.
+"""
+
+# Standard library imports
+import os
+import sys
+from unittest.mock import AsyncMock, MagicMock, patch
+import pytest
+
+# Project-specific imports
+from autogen_core.base import AgentInstantiationContext, AgentRuntime
+from src.backend.agents.human import HumanAgent
+from src.backend.models.messages import HumanFeedback, Step, StepStatus, BAgentType
+
+# Set environment variables before any imports to avoid runtime errors
+os.environ["COSMOSDB_ENDPOINT"] = "https://mock-endpoint"
+os.environ["COSMOSDB_KEY"] = "mock-key"
+os.environ["COSMOSDB_DATABASE"] = "mock-database"
+os.environ["COSMOSDB_CONTAINER"] = "mock-container"
+os.environ["APPLICATIONINSIGHTS_INSTRUMENTATION_KEY"] = "mock-instrumentation-key"
+os.environ["AZURE_OPENAI_DEPLOYMENT_NAME"] = "mock-deployment-name"
+os.environ["AZURE_OPENAI_API_VERSION"] = "2023-01-01"
+os.environ["AZURE_OPENAI_ENDPOINT"] = "https://mock-openai-endpoint"
+
+# Mock Azure modules
+sys.modules["azure.monitor.events.extension"] = MagicMock()
+
+
+@pytest.fixture
+def setup_agent():
+    """
+    Fixture to set up a HumanAgent and its dependencies.
+    """
+    memory = AsyncMock()
+    user_id = "test_user"
+    group_chat_manager_id = "group_chat_manager"
+
+    # Mock runtime and agent ID
+    mock_runtime = MagicMock(spec=AgentRuntime)
+    mock_agent_id = "test_agent_id"
+
+    # Set up the context
+    with patch.object(AgentInstantiationContext, "current_runtime", return_value=mock_runtime):
+        with patch.object(AgentInstantiationContext, "current_agent_id", return_value=mock_agent_id):
+            agent = HumanAgent(memory, user_id, group_chat_manager_id)
+
+    session_id = "session123"
+    step_id = "step123"
+    plan_id = "plan123"
+
+    # Mock HumanFeedback message
+    feedback_message = HumanFeedback(
+        session_id=session_id,
+        step_id=step_id,
+        plan_id=plan_id,
+        approved=True,
+        human_feedback="Great job!",
+    )
+
+    # Mock Step with all required fields
+    step = Step(
+        plan_id=plan_id,
+        action="Test Action",
+        agent=BAgentType.human_agent,
+        status=StepStatus.planned,
+        session_id=session_id,
+        user_id=user_id,
+        human_feedback=None,
+    )
+
+    return agent, memory, feedback_message, step, session_id, step_id, plan_id
+
+
+@patch("src.backend.agents.human.logging.info")
+@patch("src.backend.agents.human.track_event")
+@pytest.mark.asyncio
+async def test_handle_step_feedback_step_not_found(mock_track_event, mock_logging, setup_agent):
+    """
+    Test scenario where the step is not found in memory.
+    """
+    agent, memory, feedback_message, _, _, step_id, _ = setup_agent
+
+    # Mock no step found
+    memory.get_step.return_value = None
+
+    # Run the method
+    await agent.handle_step_feedback(feedback_message, MagicMock())
+
+    # Check if log and return were called correctly
+    mock_logging.assert_called_with(f"No step found with id: {step_id}")
+    memory.update_step.assert_not_called()
+    mock_track_event.assert_not_called()
+
+
+if __name__ == "__main__":
+    pytest.main()

src/backend/tests/agents/test_planner.py

@@ -0,0 +1,199 @@
+import os
+import sys
+from unittest.mock import AsyncMock, MagicMock, patch
+import pytest
+from src.backend.agents.planner import PlannerAgent
+from src.backend.models.messages import InputTask, HumanClarification, Plan, PlanStatus
+from src.backend.context.cosmos_memory import CosmosBufferedChatCompletionContext
+
+
+# Mock azure.monitor.events.extension globally
+sys.modules['azure.monitor.events.extension'] = MagicMock()
+
+# Mock environment variables
+os.environ["COSMOSDB_ENDPOINT"] = "https://mock-endpoint"
+os.environ["COSMOSDB_KEY"] = "mock-key"
+os.environ["COSMOSDB_DATABASE"] = "mock-database"
+os.environ["COSMOSDB_CONTAINER"] = "mock-container"
+os.environ["AZURE_OPENAI_DEPLOYMENT_NAME"] = "mock-deployment-name"
+os.environ["AZURE_OPENAI_API_VERSION"] = "2023-01-01"
+os.environ["AZURE_OPENAI_ENDPOINT"] = "https://mock-openai-endpoint"
+
+
+@pytest.fixture
+def mock_context():
+    """Mock the CosmosBufferedChatCompletionContext."""
+    return MagicMock(spec=CosmosBufferedChatCompletionContext)
+
+
+@pytest.fixture
+def mock_model_client():
+    """Mock the Azure OpenAI model client."""
+    return MagicMock()
+
+
+@pytest.fixture
+def mock_runtime_context():
+    """Mock the runtime context for AgentInstantiationContext."""
+    with patch(
+        "autogen_core.base._agent_instantiation.AgentInstantiationContext.AGENT_INSTANTIATION_CONTEXT_VAR",
+        new=MagicMock(),
+    ) as mock_context_var:
+        yield mock_context_var
+
+
+@pytest.fixture
+def planner_agent(mock_model_client, mock_context, mock_runtime_context):
+    """Return an instance of PlannerAgent with mocked dependencies."""
+    # Mock the context variable to ensure runtime context is properly simulated
+    mock_runtime_context.get.return_value = (MagicMock(), "mock-agent-id")
+    return PlannerAgent(
+        model_client=mock_model_client,
+        session_id="test-session",
+        user_id="test-user",
+        memory=mock_context,
+        available_agents=["HumanAgent", "MarketingAgent", "TechSupportAgent"],
+        agent_tools_list=["tool1", "tool2"],
+    )
+
+
+@pytest.mark.asyncio
+async def test_handle_plan_clarification(planner_agent, mock_context):
+    """Test the handle_plan_clarification method."""
+    # Prepare mock clarification and context
+    mock_clarification = HumanClarification(
+        session_id="test-session",
+        plan_id="plan-1",
+        human_clarification="Test clarification",
+    )
+
+    mock_context.get_plan_by_session = AsyncMock(return_value=Plan(
+        id="plan-1",
+        session_id="test-session",
+        user_id="test-user",
+        initial_goal="Test Goal",
+        overall_status="in_progress",
+        source="PlannerAgent",
+        summary="Mock Summary",
+        human_clarification_request=None,
+    ))
+    mock_context.update_plan = AsyncMock()
+    mock_context.add_item = AsyncMock()
+
+    # Execute the method
+    await planner_agent.handle_plan_clarification(mock_clarification, None)
+
+    # Assertions
+    mock_context.get_plan_by_session.assert_called_with(session_id="test-session")
+    mock_context.update_plan.assert_called()
+    mock_context.add_item.assert_called()
+
+
+@pytest.mark.asyncio
+async def test_generate_instruction_with_special_characters(planner_agent):
+    """Test _generate_instruction with special characters in the objective."""
+    special_objective = "Solve this task: @$%^&*()"
+    instruction = planner_agent._generate_instruction(special_objective)
+
+    # Assertions
+    assert "Solve this task: @$%^&*()" in instruction
+    assert "HumanAgent" in instruction
+    assert "tool1" in instruction
+
+
+@pytest.mark.asyncio
+async def test_handle_plan_clarification_updates_plan_correctly(planner_agent, mock_context):
+    """Test handle_plan_clarification ensures correct plan updates."""
+    mock_clarification = HumanClarification(
+        session_id="test-session",
+        plan_id="plan-1",
+        human_clarification="Updated clarification text",
+    )
+
+    mock_plan = Plan(
+        id="plan-1",
+        session_id="test-session",
+        user_id="test-user",
+        initial_goal="Test Goal",
+        overall_status="in_progress",
+        source="PlannerAgent",
+        summary="Mock Summary",
+        human_clarification_request="Previous clarification needed",
+    )
+
+    # Mock get_plan_by_session and update_plan
+    mock_context.get_plan_by_session = AsyncMock(return_value=mock_plan)
+    mock_context.update_plan = AsyncMock()
+
+    # Execute the method
+    await planner_agent.handle_plan_clarification(mock_clarification, None)
+
+    # Assertions
+    assert mock_plan.human_clarification_response == "Updated clarification text"
+    mock_context.update_plan.assert_called_with(mock_plan)
+
+
+@pytest.mark.asyncio
+async def test_handle_input_task_with_exception(planner_agent, mock_context):
+    """Test handle_input_task gracefully handles exceptions."""
+    # Mock InputTask
+    input_task = InputTask(description="Test task causing exception", session_id="test-session")
+
+    # Mock _create_structured_plan to raise an exception
+    planner_agent._create_structured_plan = AsyncMock(side_effect=Exception("Mocked exception"))
+
+    # Execute the method
+    with pytest.raises(Exception, match="Mocked exception"):
+        await planner_agent.handle_input_task(input_task, None)
+
+    # Assertions
+    planner_agent._create_structured_plan.assert_called()
+    mock_context.add_item.assert_not_called()
+    mock_context.add_plan.assert_not_called()
+    mock_context.add_step.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_handle_plan_clarification_handles_memory_error(planner_agent, mock_context):
+    """Test handle_plan_clarification gracefully handles memory errors."""
+    mock_clarification = HumanClarification(
+        session_id="test-session",
+        plan_id="plan-1",
+        human_clarification="Test clarification",
+    )
+
+    # Mock get_plan_by_session to raise an exception
+    mock_context.get_plan_by_session = AsyncMock(side_effect=Exception("Memory error"))
+
+    # Execute the method
+    with pytest.raises(Exception, match="Memory error"):
+        await planner_agent.handle_plan_clarification(mock_clarification, None)
+
+    # Ensure no updates or messages are added after failure
+    mock_context.update_plan.assert_not_called()
+    mock_context.add_item.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_generate_instruction_with_missing_objective(planner_agent):
+    """Test _generate_instruction with a missing or empty objective."""
+    instruction = planner_agent._generate_instruction("")
+    assert "Your objective is:" in instruction
+    assert "The agents you have access to are:" in instruction
+    assert "These agents have access to the following functions:" in instruction
+
+
+@pytest.mark.asyncio
+async def test_create_structured_plan_with_error(planner_agent, mock_context):
+    """Test _create_structured_plan when an error occurs during plan creation."""
+    planner_agent._model_client.create = AsyncMock(side_effect=Exception("Mocked error"))
+
+    messages = [{"content": "Test message", "source": "PlannerAgent"}]
+    plan, steps = await planner_agent._create_structured_plan(messages)
+
+    # Assertions
+    assert plan.initial_goal == "Error generating plan"
+    assert plan.overall_status == PlanStatus.failed
+    assert len(steps) == 0
+    mock_context.add_plan.assert_not_called()
+    mock_context.add_step.assert_not_called()