Private endpoint support for container apps

app/backend/prepdocs.py

@@ -4,6 +4,7 @@
 import os
 from typing import Optional, Union
 
+import aiohttp
 from azure.core.credentials import AzureKeyCredential
 from azure.core.credentials_async import AsyncTokenCredential
 from azure.identity.aio import AzureDeveloperCliCredential, get_bearer_token_provider
@@ -45,6 +46,19 @@ def clean_key_if_exists(key: Union[str, None]) -> Union[str, None]:
     return None
 
 
+async def check_search_service_connectivity(search_service: str) -> bool:
+    """Check if the search service is accessible by hitting the /ping endpoint."""
+    ping_url = f"https://{search_service}.search.windows.net/ping"
+
+    try:
+        async with aiohttp.ClientSession() as session:
+            async with session.get(ping_url, timeout=aiohttp.ClientTimeout(total=10)) as response:
+                return response.status == 200
+    except Exception as e:
+        logger.debug(f"Search service ping failed: {e}")
+        return False
+
+
 async def setup_search_info(
     search_service: str,
     index_name: str,
@@ -323,7 +337,10 @@ async def main(strategy: Strategy, setup_index: bool = True):
 
     load_azd_env()
 
-    if os.getenv("AZURE_PUBLIC_NETWORK_ACCESS") == "Disabled":
+    if (
+        os.getenv("AZURE_PUBLIC_NETWORK_ACCESS") == "Disabled"
+        and os.getenv("AZURE_USE_VPN_GATEWAY", "").lower() != "true"
+    ):
         logger.error("AZURE_PUBLIC_NETWORK_ACCESS is set to Disabled. Exiting.")
         exit(0)
 
@@ -372,6 +389,23 @@ async def main(strategy: Strategy, setup_index: bool = True):
             search_key=clean_key_if_exists(args.searchkey),
         )
     )
+
+    # Check search service connectivity
+    search_service = os.environ["AZURE_SEARCH_SERVICE"]
+    is_connected = loop.run_until_complete(check_search_service_connectivity(search_service))
+
+    if not is_connected:
+        if os.getenv("AZURE_USE_PRIVATE_ENDPOINT"):
+            logger.error(
+                "Unable to connect to Azure AI Search service, which indicates either a network issue or a misconfiguration. You have AZURE_USE_PRIVATE_ENDPOINT enabled. Perhaps you're not yet connected to the VPN? Download the VPN configuration from the Azure portal here: %s",
+                os.getenv("AZURE_VPN_CONFIG_DOWNLOAD_LINK"),
+            )
+        else:
+            logger.error(
+                "Unable to connect to Azure AI Search service, which indicates either a network issue or a misconfiguration."
+            )
+        exit(1)
+
     blob_manager = setup_blob_manager(
         azure_credential=azd_credential,
         storage_account=os.environ["AZURE_STORAGE_ACCOUNT"],

docs/deploy_private.md

@@ -46,20 +46,59 @@ Deploying with public access disabled adds additional cost to your deployment. P
 
 ## Recommended deployment strategy for private access
 
-1. Deploy the app with private endpoints enabled and public access enabled.
+1. Configure the azd environment variables to use private endpoints and a VPN gateway, with public network access disabled. This will allow you to connect to the chat app from inside the virtual network, but not from the public Internet.
 
-  ```shell
-  azd env set AZURE_USE_PRIVATE_ENDPOINT true
-  azd env set AZURE_PUBLIC_NETWORK_ACCESS Enabled
-  azd up
-  ```
+    ```shell
+    azd env set AZURE_USE_PRIVATE_ENDPOINT true
+    azd env set AZURE_USE_VPN_GATEWAY true
+    azd env set AZURE_PUBLIC_NETWORK_ACCESS Disabled
+    azd up
+    ```
 
-1. Validate that you can connect to the chat app and it's working as expected from the internet.
-1. Re-provision the app with public access disabled.
+2. Provision all the Azure resources:
 
-  ```shell
-  azd env set AZURE_PUBLIC_NETWORK_ACCESS Disabled
-  azd provision
-  ```
+    ```bash
+    azd provision
+    ```
 
-1. Log into your network using a tool like [Azure VPN Gateway](https://azure.microsoft.com/services/vpn-gateway/) and validate that you can connect to the chat app from inside the network.
+3. Once provisioning is complete, you will see an error when it tries to run the data ingestion script, because you are not yet connected to the VPN. That message should provide a URL for the VPN configuration file download. If you don't see that URL, run this command:
+
+    ```bash
+    azd env get-value AZURE_VPN_CONFIG_DOWNLOAD_LINK
+    ```
+
+    Open that link in your browser. Select "Download VPN client" to download a ZIP file containing the VPN configuration.
+
+4. Open `AzureVPN/azurevpnconfig.xml`, and replace the `<clientconfig>` empty tag with the following:
+
+    ```xml
+      <clientconfig>
+        <dnsservers>
+          <dnsserver>10.0.11.4</dnsserver>
+        </dnsservers>
+      </clientconfig>
+    ```
+
+    > **Note:** The IP address `10.0.11.4` is the first available IP in the `dns-resolver-subnet`(10.0.11.0/28), as Azure reserves the first four IP addresses in each subnet. Adding this DNS server allows your VPN client to resolve private DNS names for Azure services accessed through private endpoints. See the network configuration in [network-isolation.bicep](../infra/network-isolation.bicep) for details.
+
+5. Install the [Azure VPN Client](https://learn.microsoft.com/azure/vpn-gateway/azure-vpn-client-versions).
+
+6. Open the Azure VPN Client and select "Import" button. Select the `azurevpnconfig.xml` file you just downloaded and modified.
+
+7. Select "Connect" and the new VPN connection. You will be prompted to select your Microsoft account and login.
+
+8. Once you're successfully connected to VPN, you can run the data ingestion script:
+
+    ```bash
+    azd hooks run postprovision
+    ```
+
+9. Finally, you can deploy the app:
+
+    ```bash
+    azd deploy
+    ```
+
+## Compatibility with other features
+
+* **GitHub Actions / Azure DevOps**: The private access deployment is not compatible with the built-in CI/CD pipelines, as it requires a VPN connection to deploy the app. You could modify the pipeline to only do provisioning, and set up a different deployment strategy for the app.

infra/abbreviations.json

@@ -110,6 +110,7 @@
   "privateEndpoint": "pe-",
   "privateLink": "pl-",
   "purviewAccounts": "pview-",
+  "privateDnsResolver": "pdr-",
   "recoveryServicesVaults": "rsv-",
   "resourcesResourceGroups": "rg-",
   "searchSearchServices": "srch-",

infra/core/host/container-app-upsert.bicep

@@ -78,9 +78,6 @@ param serviceBinds array = []
 @description('The target port for the container')
 param targetPort int = 80
 
-@allowed(['Consumption', 'D4', 'D8', 'D16', 'D32', 'E4', 'E8', 'E16', 'E32', 'NC24-A100', 'NC48-A100', 'NC96-A100'])
-param workloadProfile string = 'Consumption'
-
 param allowedOrigins array = []
 
 resource existingApp 'Microsoft.App/containerApps@2023-05-02-preview' existing = if (exists) {
@@ -98,7 +95,6 @@ module app 'container-app.bicep' = {
   name: '${deployment().name}-update'
   params: {
     name: name
-    workloadProfile: workloadProfile
     location: location
     tags: tags
     identityType: identityType

infra/core/host/container-app.bicep

@@ -79,8 +79,6 @@ param serviceType string = ''
 @description('The target port for the container')
 param targetPort int = 80
 
-param workloadProfile string = 'Consumption'
-
 resource userIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = if (!empty(identityName)) {
   name: identityName
 }
@@ -125,7 +123,6 @@ resource app 'Microsoft.App/containerApps@2023-05-02-preview' = {
   }
   properties: {
     managedEnvironmentId: containerAppsEnvironment.id
-    workloadProfileName: workloadProfile
     configuration: {
       activeRevisionsMode: revisionMode
       ingress: ingressEnabled ? {

infra/core/host/container-apps-environment.bicep

@@ -0,0 +1,59 @@
+param name string
+param location string = resourceGroup().location
+param tags object = {}
+
+param daprEnabled bool = false
+param logAnalyticsWorkspaceName string = ''
+param applicationInsightsName string = ''
+
+param subnetResourceId string
+
+param usePrivateIngress bool = true
+
+resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2025-02-02-preview' = {
+  name: name
+  location: location
+  tags: tags
+  properties: {
+    // We can't use a conditional here due to an issue with the Container Apps ARM parsing
+    appLogsConfiguration: {
+      destination: 'log-analytics'
+      logAnalyticsConfiguration: {
+        customerId: logAnalyticsWorkspace.properties.customerId
+        sharedKey: logAnalyticsWorkspace.listKeys().primarySharedKey
+      }
+    }
+    daprAIInstrumentationKey: daprEnabled && !empty(applicationInsightsName) ? applicationInsights.properties.InstrumentationKey : ''
+    publicNetworkAccess: usePrivateIngress ? 'Disabled' : 'Enabled'
+    vnetConfiguration: usePrivateIngress ? {
+      infrastructureSubnetId: subnetResourceId
+      internal: true
+    } : null
+    workloadProfiles: usePrivateIngress
+    ? [
+      {
+        name: 'Consumption'
+        workloadProfileType: 'Consumption'
+      }
+      {
+        name: 'Warm'
+        workloadProfileType: 'D4'
+        minimumCount: 1
+        maximumCount: 3
+      }
+    ]
+    : []
+  }
+}
+
+resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' existing = if (!empty(logAnalyticsWorkspaceName)) {
+  name: logAnalyticsWorkspaceName
+}
+
+resource applicationInsights 'Microsoft.Insights/components@2020-02-02' existing = if (daprEnabled && !empty(applicationInsightsName)){
+  name: applicationInsightsName
+}
+
+output defaultDomain string = containerAppsEnvironment.properties.defaultDomain
+output name string = containerAppsEnvironment.name
+output resourceId string = containerAppsEnvironment.id

infra/core/host/container-apps.bicep

@@ -1,78 +1,46 @@
-metadata description = 'Creates an Azure Container Registry and an Azure Container Apps environment.'
 param name string
 param location string = resourceGroup().location
 param tags object = {}
 
 param containerAppsEnvironmentName string
 param containerRegistryName string
-param containerRegistryResourceGroupName string = ''
-param containerRegistryAdminUserEnabled bool = false
-param logAnalyticsWorkspaceResourceId string
-param applicationInsightsName string = '' // Not used here, was used for DAPR
-param virtualNetworkSubnetId string = ''
-@allowed(['Consumption', 'D4', 'D8', 'D16', 'D32', 'E4', 'E8', 'E16', 'E32', 'NC24-A100', 'NC48-A100', 'NC96-A100'])
-param workloadProfile string
+param logAnalyticsWorkspaceName string = ''
+param applicationInsightsName string = ''
 
-var workloadProfiles = workloadProfile == 'Consumption'
-  ? [
-      {
-        name: 'Consumption'
-        workloadProfileType: 'Consumption'
-      }
-    ]
-  : [
-      {
-        name: 'Consumption'
-        workloadProfileType: 'Consumption'
-      }
-      {
-        minimumCount: 0
-        maximumCount: 2
-        name: workloadProfile
-        workloadProfileType: workloadProfile
-      }
-    ]
+@description('Virtual network name for container apps environment.')
+param vnetName string = ''
+@description('Subnet name for container apps environment integration.')
+param subnetName string = ''
 
-@description('Optional user assigned identity IDs to assign to the resource')
-param userAssignedIdentityResourceIds array = []
+param subnetResourceId string = ''
 
-module containerAppsEnvironment 'br/public:avm/res/app/managed-environment:0.8.0' = {
+param usePrivateIngress bool = true
+
+module containerAppsEnvironment 'container-apps-environment.bicep' = {
   name: '${name}-container-apps-environment'
   params: {
-    // Required parameters
-    logAnalyticsWorkspaceResourceId: logAnalyticsWorkspaceResourceId
-
-    managedIdentities: empty(userAssignedIdentityResourceIds) ? {
-      systemAssigned: true
-    } : {
-      userAssignedResourceIds: userAssignedIdentityResourceIds
-    }
-
     name: containerAppsEnvironmentName
-    // Non-required parameters
-    infrastructureResourceGroupName: containerRegistryResourceGroupName
-    infrastructureSubnetId: virtualNetworkSubnetId
     location: location
     tags: tags
-    zoneRedundant: false
-    workloadProfiles: workloadProfiles
+    logAnalyticsWorkspaceName: logAnalyticsWorkspaceName
+    applicationInsightsName: applicationInsightsName
+    usePrivateIngress: usePrivateIngress
+    subnetResourceId: subnetResourceId
   }
 }
 
-module containerRegistry 'br/public:avm/res/container-registry/registry:0.5.1' = {
+module containerRegistry 'container-registry.bicep' = {
   name: '${name}-container-registry'
-  scope: resourceGroup(!empty(containerRegistryResourceGroupName) ? containerRegistryResourceGroupName : resourceGroup().name)
   params: {
     name: containerRegistryName
     location: location
-    acrAdminUserEnabled: containerRegistryAdminUserEnabled
     tags: tags
+    useVnet: !empty(vnetName)
   }
 }
 
 output defaultDomain string = containerAppsEnvironment.outputs.defaultDomain
 output environmentName string = containerAppsEnvironment.outputs.name
 output environmentId string = containerAppsEnvironment.outputs.resourceId
-
 output registryLoginServer string = containerRegistry.outputs.loginServer
 output registryName string = containerRegistry.outputs.name

infra/core/host/container-registry.bicep

@@ -0,0 +1,38 @@
+param name string
+param location string = resourceGroup().location
+param tags object = {}
+
+param adminUserEnabled bool = true
+param anonymousPullEnabled bool = false
+param dataEndpointEnabled bool = false
+param encryption object = {
+  status: 'disabled'
+}
+param networkRuleBypassOptions string = 'AzureServices'
+param publicNetworkAccess string = useVnet ? 'Disabled' : 'Enabled' // Public network access is disabled if VNet integration is enabled
+param useVnet bool = false // Determines if VNet integration is enabled
+param sku object = {
+  name: useVnet ? 'Premium' : 'Standard' // Use Premium if VNet is required, otherwise Standard
+}
+param zoneRedundancy string = 'Disabled'
+
+// 2022-02-01-preview needed for anonymousPullEnabled
+resource containerRegistry 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = {
+  name: name
+  location: location
+  tags: tags
+  sku: sku
+  properties: {
+    adminUserEnabled: adminUserEnabled
+    anonymousPullEnabled: anonymousPullEnabled
+    dataEndpointEnabled: dataEndpointEnabled
+    encryption: encryption
+    networkRuleBypassOptions: networkRuleBypassOptions
+    publicNetworkAccess: publicNetworkAccess
+    zoneRedundancy: zoneRedundancy
+  }
+}
+
+output loginServer string = containerRegistry.properties.loginServer
+output name string = containerRegistry.name
+output resourceId string = containerRegistry.id

infra/core/storage/storage-account.bicep

@@ -59,7 +59,7 @@ resource storage 'Microsoft.Storage/storageAccounts@2024-01-01' = {
     resource container 'containers' = [for container in containers: {
       name: container.name
       properties: {
-        publicAccess: contains(container, 'publicAccess') ? container.publicAccess : 'None'
+        publicAccess: container.?publicAccess ?? 'None'
       }
     }]
   }