Adding End-to-End Test and Streamlining Existing Tests (#247)

* Combined updates

---------

Co-authored-by: Matt Dotson <[email protected]>

Author: ianjensenisme

.devcontainer/devcontainer.json

@@ -36,7 +36,9 @@
                 "terraform-linters.tflint-vscode",
                 "microsoft-IsvExpTools.powerplatform-vscode",
                 "ms-vscode.azurecli",
-                "bierner.markdown-mermaid"
+                "bierner.markdown-mermaid",
+                "ms-dotnettools.csharp",
+                "ms-dotnettools.vscode-dotnet-runtime"
                 // Include other VSCode extensions if needed
                 // Right click on an extension inside VSCode to add directly to devcontainer.json, or copy the extension ID
             ],

.devcontainer/postCreate.sh

@@ -12,4 +12,19 @@ tflint --init
 echo "Installing PowerApps CLI..."
 dotnet tool install --global Microsoft.PowerApps.CLI.Tool --version 1.43.6
 
-echo "Post-create setup completed successfully!"
+# Restore .NET packages including Microsoft.Agents.CopilotStudio.Client
+echo "Restoring .NET packages..."
+if [ -f "Directory.Build.props" ]; then
+    dotnet restore
+    echo ".NET packages restored successfully!"
+else
+    # Fallback to individual project restore
+    if [ -f "tests/Copilot/CopilotTests.csproj" ]; then
+        dotnet restore tests/Copilot/CopilotTests.csproj
+        echo "Copilot project packages restored successfully!"
+    else
+        echo "No .NET projects found, skipping package restore"
+    fi
+fi
+
+echo "Post-create setup completed successfully!"

.gitignore

@@ -493,3 +493,7 @@ power_platform_deployment_settings
 *_env/
 *_venv/
 *.venv/
+
+# Ignore environment files
+.env
+.env.*

Directory.Build.props

@@ -0,0 +1,22 @@
+<Project>
+  
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <LangVersion>latest</LangVersion>
+    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+    <WarningsNotAsErrors>NU1503</WarningsNotAsErrors>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <!-- Common package versions -->
+    <MicrosoftAgentsCopilotStudioClientVersion>1.0.0</MicrosoftAgentsCopilotStudioClientVersion>
+    <MicrosoftNETTestSdkVersion>17.8.0</MicrosoftNETTestSdkVersion>
+    <XunitVersion>2.4.2</XunitVersion>
+    <XunitRunnerVersion>2.4.5</XunitRunnerVersion>
+    <CoverletVersion>6.0.0</CoverletVersion>
+  </PropertyGroup>
+
+</Project>
+

README.md

@@ -18,6 +18,9 @@ network security.
   - [Deploying](#deploying)
   - [Using the bot](#using-the-bot)
   - [Clean up](#clean-up)
+- [Testing](#testing)
+  - [Copilot Studio Agent Tests](#copilot-studio-agent-test)
+  - [AI Search Tests (Optional)](#ai-search-test-optional)
 - [Advanced scenarios](#advanced-scenarios)
   - [GitHub self-hosted runners](#github-self-hosted-runners)
 - [Resource Configuration Notes](#resource-configuration-notes)
@@ -215,7 +218,105 @@ To clean up all the resources created by this sample:
 
 All the Azure and Power Platform resources will be deleted.
 
-## Advanced scenarios
+## Testing
+
+This solution includes tests that validate both Copilot Studio and Azure AI Search components after deployment.
+
+### Copilot Studio Agent Test
+
+Located in `tests/Copilot/`, this test validates:
+
+- **Conversation Flow**: End-to-end conversation test with the deployed agent
+- **Integration**: Validation that Copilot Studio can successfully query Azure AI Search
+
+Currently, [the Copilot Studio Client in the Agent SDK does not support the use of Service Principals for authentication](https://github.com/microsoft/Agents/blob/main/samples/basic/copilotstudio-client/dotnet/README.md#create-an-application-registration-in-entra-id---service-principal-login), and testing requires a cloud-native app registration as well as a test account with MFA turned off. The test user account must have access to the Power Platform environment containing the agent as well as access to the agent itself.
+
+#### Running Tests After Local Deployment Execution
+
+After a successful local deployment execution, the local .env file contains most of the information needed to run the end-to-end Copilot Studio test. Alternatively, any test input can be set directly through environment variables.
+
+Run the commands below to execute the test after a deployment.
+
+```bash
+# Navigate to the test directory
+cd tests/Copilot
+
+export POWER_PLATFORM_USERNAME="[email protected]"
+export POWER_PLATFORM_PASSWORD="passhere"
+export TEST_CLIENT_ID="native-app-guid-here"
+
+# Run tests using azd environment outputs (recommended)
+dotnet test --logger "console;verbosity=detailed"
+```
+
+#### Running Tests with Manual Environment Variable Configuration
+
+If you prefer to set environment variables manually or need to override specific values, you can configure all required variables explicitly:
+
+```bash
+# Navigate to the test directory
+cd tests/Copilot
+
+# Power Platform authentication
+export POWER_PLATFORM_USERNAME="[email protected]"
+export POWER_PLATFORM_PASSWORD="your-test-password"
+export POWER_PLATFORM_TENANT_ID="your-tenant-id"
+export POWER_PLATFORM_ENVIRONMENT_ID="your-environment-id"
+
+# Native client application ID
+export TEST_CLIENT_ID="your-native-app-client-id"
+
+# Copilot Studio configuration
+export COPILOT_STUDIO_ENDPOINT="https://api.copilotstudio.microsoft.com"
+export COPILOT_STUDIO_AGENT_ID="crfXX_agentName"
+
+# Run the test
+dotnet test --logger "console;verbosity=detailed"
+```
+
+**Important Notes:**
+- The test account must have **MFA disabled** for automated authentication
+- The user must have access to the Power Platform environment and the Copilot Studio agent
+- Environment variables take precedence over values from azd .env files
+
+### AI Search Test (Optional)
+
+Located in `tests/AISearch/`, this test validates:
+
+- **Resource Existence**: Verify all search resources (index, datasource, skillset, indexer) exist
+- **Configuration Validation**: Check resource configurations match expected settings
+- **Content Verification**: Validate index contains expected documents and supports search
+- **Pipeline Integration**: End-to-end validation of the complete search pipeline
+
+Because the Copilot agent end-to-end test includes indirect validation of the AI Search functionality, this test does not need to be run unless direct validation and troubleshooting of the AI Search resources is required.
+
+#### Prerequisites for AI Search Tests
+
+Before running AI Search tests, you must complete the following configuration:
+
+1. **Make AI Search Endpoint Public**: Unless the test is run on the same virtual network as the AI Search resource, the AI Search service must be updated to be accessible to the test script. Configure network access in the Azure portal:
+   - Navigate to your AI Search service
+   - Go to **Networking** → **Firewalls and virtual networks**
+   - Select **All networks** or add the test runner's IP to **Selected IP addresses**
+
+2. **Assign RBAC Roles**: The user or service principal running the tests must have the following roles:
+  - Navigate to your AI Search service in the Azure portal
+  - Go to **Access control (IAM)** → **Add role assignment**
+  - Select **Search Index Data Contributor** role and assign to the user or service principal that will execute the tests
+  - Add another role assignment for **Search Service Contributor** role to the same user or service principal
+
+#### Running AI Search Tests Locally
+
+```bash
+# Ensure you're authenticated and have an azd environment deployed
+az login
+
+# Run the test script
+cd tests/AISearch
+./run-tests.sh
+```
+
+The tests automatically discover configuration from your azd environment outputs.
 
 ### GitHub self-hosted runners
 

decision-log/005-agent-sdk-authentication-strategy.md

@@ -0,0 +1,82 @@
+# Decision Log 005: Agent SDK Authentication Strategy for Testing
+
+**Date:** 2025-07-30  
+**Status:** Approved
+
+## Context
+
+Our end-to-end testing strategy requires automated testing of Copilot Studio agents to validate the end-to-end flow of responses from the Azure resources to the Copilot Studio agent. We evaluated multiple approaches for implementing these tests, considering authentication requirements, platform support, and operational complexity.
+
+The key testing approaches considered were:
+- **Agent SDK with Service Principal Authentication**: Programmatic access using client credentials
+- **Agent SDK with User-Based Authentication**: Username/password authentication with real user accounts
+- **Copilot Studio Built-in Evaluation Tools**: Native evaluation capabilities within the Copilot Studio platform
+
+Our solution requires reliable, automated testing that can be integrated into CI/CD pipelines while providing comprehensive validation of the AI search integration functionality.
+
+## Decision
+
+We will use **Agent SDK with User-Based Authentication** as our primary testing approach for Copilot Studio agent validation.
+
+## Rationale
+
+1. **Broadest Platform Support**: Agent SDK provides comprehensive support across Microsoft's AI technologies, including Copilot Studio, Foundry, and other conversational AI tools. This gives us maximum flexibility and ensures our testing approach aligns with Microsoft's recommended practices.
+
+2. **Current Authentication Limitations**: Agent SDK does not currently support service principal authentication for Copilot Studio scenarios. While this limits our automation options, user-based authentication is the only viable approach for programmatic testing at this time.
+
+3. **Proven Functional Capability**: We have successfully validated that user-based authentication works reliably for our testing scenarios, providing the functionality needed to execute comprehensive end-to-end tests.
+
+4. **Native SDK Benefits**: Using the official Agent SDK ensures we stay aligned with Microsoft's evolving API surface and receive support for new features as they become available.
+
+## Trade-offs and Considerations
+
+### Accepted Trade-offs
+
+- **MFA Requirement**: User-based authentication requires test accounts with Multi-Factor Authentication (MFA) disabled or configured with app passwords, which introduces security considerations that must be managed through proper test account governance.
+
+- **Credential Management**: User credentials require more careful handling compared to service principals, necessitating additional security measures in our testing infrastructure.
+
+- **Account Lifecycle**: Test user accounts require ongoing management and may be subject to organizational password policies and account lifecycle requirements.
+
+### Alternative Approaches Considered
+
+1. **Copilot Studio Built-in Evaluation Tools**
+   - **Status**: Currently in preview
+   - **Benefits**: Lower effort setup, designed specifically for low-code users, integrated with Copilot Studio platform
+   - **Limitations**: Not generally available, limited programmatic control, unclear CI/CD integration capabilities
+   - **Future Consideration**: This will be reevaluated when these tools reach general availability
+
+2. **Service Principal Authentication**
+   - **Status**: Not supported by Agent SDK for Copilot Studio
+   - **Benefits**: Better security model, easier credential management, standard for enterprise automation
+   - **Limitations**: Currently not available for our use case
+   - **Future Consideration**: Will be adopted immediately when Agent SDK adds this capability
+
+3. **Test Channels**
+   - **Status**: Deprioritized in favor of preserving a secure base agent
+   - **Limitations**: Testing against a specific channel adds a layer of abstraction that could obscure issues in the Copilot itself. The go-to test channel (DirectLine) is unsecure, and the default agent should not include an unsecure channel by default.
+   - **Future Consideration**: Tests against channel-specific functionality may be added in the future if a particular channel proves to be heavily utilized.
+
+4. **Python-Based Tests**
+   - **Status**: Pending on SDK updates
+   - **Limitations**: At test creation time, the Python SDK did not include the full testing feature set and documentation that was available in the .NET tools.
+   - **Future Consideration**: The tests could be converted to Python-based tests when the Python SDK reaches feature parity.
+
+## Implementation Requirements
+
+1. **Test Account Management**: Establish dedicated test accounts with appropriate security configurations
+2. **Credential Security**: Implement secure credential storage and handling practices in CI/CD pipelines
+3. **Error Handling**: Robust authentication error handling and retry logic for transient failures
+4. **Monitoring**: Track authentication success/failure rates and account health
+
+## Future Considerations
+
+- **Service Principal Support**: Monitor Agent SDK releases for service principal authentication support and migrate when available
+- **Copilot Studio Evaluation Tools**: Evaluate built-in tools when they reach general availability and assess integration with our testing strategy
+- **Hybrid Approach**: Consider combining multiple testing approaches as capabilities mature to provide comprehensive coverage
+
+## Related Decisions
+
+- This decision supports the overall testing strategy established in our CI/CD pipeline design
+- Security considerations align with our credential management policies outlined in infrastructure security practices
+

global.json

@@ -0,0 +1,5 @@
+{
+  "sdk": {
+    "version": "8.0.412"
+  }
+}

infra/main.search_configuration.tf

@@ -176,7 +176,7 @@ resource "azapi_resource" "configure_search_index" {
         echo "=== Step 3: Configuring search index ==="
         python index_utils.py \
           --aisearch_name ${azurerm_search_service.ai_search.name} \
-          --base_index_name "default" \
+          --base_index_name "${var.ai_search_base_index_name}" \
           --openai_api_base ${module.azure_open_ai.endpoint} \
           --subscription_id ${data.azurerm_client_config.current.subscription_id} \
           --resource_group_name ${local.resource_group_name} \

infra/modules/copilot_studio/power_platform_users.tf

@@ -14,15 +14,32 @@ data "powerplatform_security_roles" "all_roles" {
 }
 
 locals {
-  security_role_id  = { for item in data.powerplatform_security_roles.all_roles.security_roles : item.name => item.role_id }
-  security_role_ids = [for name in var.pp_environment_user_security_role : local.security_role_id[name]]
+  # Create a map of security role names to IDs
+  security_role_id = { for item in data.powerplatform_security_roles.all_roles.security_roles : item.name => item.role_id }
+
+  # Filter to only include roles that actually exist in the environment
+  available_roles = [for name in var.pp_environment_user_security_role : name if contains(keys(local.security_role_id), name)]
+
+  # Get the role IDs for the available roles
+  security_role_ids = [for name in local.available_roles : local.security_role_id[name]]
+
+  # Calculate missing roles for debugging
+  missing_roles = [for name in var.pp_environment_user_security_role : name if !contains(keys(local.security_role_id), name)]
+}
+
+# Validation check for missing security roles
+check "security_roles_exist" {
+  assert {
+    condition     = length(local.missing_roles) == 0
+    error_message = "The following security roles were requested but do not exist in the Power Platform environment: ${join(", ", local.missing_roles)}. Available roles are: ${join(", ", keys(local.security_role_id))}"
+  }
 }
 
 # Add non-dataverse user to Power Platform environment
 resource "powerplatform_user" "new_non_dataverse_user" {
   for_each       = length(var.resource_share_user) > 0 ? var.resource_share_user : []
   environment_id = local.power_platform_environment_id
-  security_roles = local.security_role_ids # Using the same roles from the all_roles data source
+  security_roles = local.security_role_ids
   aad_id         = each.value
-  disable_delete = false
+  // disable_delete = false
 }

infra/outputs.tf

@@ -3,6 +3,16 @@ output "ai_search_resource_name" {
   value       = azurerm_search_service.ai_search.name
 }
 
+output "ai_search_endpoint" {
+  description = "The endpoint URL of the AI Search service"
+  value       = "https://${azurerm_search_service.ai_search.name}.search.windows.net"
+}
+
+output "ai_search_base_index_name" {
+  description = "The base name used for AI Search resources"
+  value       = var.ai_search_base_index_name
+}
+
 output "app_insights_instrumentation_key" {
   sensitive = true
   value     = var.include_app_insights ? azurerm_application_insights.insights[0].instrumentation_key : null
@@ -57,3 +67,23 @@ output "secondary_azure_region" {
   description = "The secondary Azure region for deployment"
   value       = local.secondary_azure_region
 }
+
+output "copilot_studio_endpoint" {
+  description = "The endpoint URL for Copilot Studio API"
+  value       = "https://api.copilotstudio.microsoft.com"
+}
+
+output "copilot_studio_agent_id" {
+  description = "The ID of the deployed Copilot Studio agent (placeholder - to be extracted from deployment)"
+  value       = "crf6d_aiSearchConnectionExample" # This should match the bot name from the solution
+}
+
+output "azure_tenant_id" {
+  description = "The Azure tenant ID"
+  value       = data.azurerm_client_config.current.tenant_id
+}
+
+output "azure_subscription_id" {
+  description = "The Azure subscription ID"
+  value       = data.azurerm_client_config.current.subscription_id
+}