diff --git a/.azdo/pipelines/azure-dev.yml b/.azdo/pipelines/azure-dev.yml index bb5da8069f..3498c25dc7 100644 --- a/.azdo/pipelines/azure-dev.yml +++ b/.azdo/pipelines/azure-dev.yml @@ -109,6 +109,8 @@ steps: AZURE_ADLS_GEN2_STORAGE_ACCOUNT: $(AZURE_ADLS_GEN2_STORAGE_ACCOUNT) AZURE_ADLS_GEN2_FILESYSTEM_PATH: $(AZURE_ADLS_GEN2_FILESYSTEM_PATH) AZURE_ADLS_GEN2_FILESYSTEM: $(AZURE_ADLS_GEN2_FILESYSTEM) + DEPLOYMENT_TARGET: $(DEPLOYMENT_TARGET) + AZURE_CONTAINER_APPS_WORKLOAD_PROFILE: $(AZURE_CONTAINER_APPS_WORKLOAD_PROFILE) - task: AzureCLI@2 displayName: Deploy Application diff --git a/.github/workflows/azure-dev.yml b/.github/workflows/azure-dev.yml index 819d6cff1d..d414609eb1 100644 --- a/.github/workflows/azure-dev.yml +++ b/.github/workflows/azure-dev.yml @@ -93,6 +93,8 @@ jobs: AZURE_ADLS_GEN2_STORAGE_ACCOUNT: ${{ vars.AZURE_ADLS_GEN2_STORAGE_ACCOUNT }} AZURE_ADLS_GEN2_FILESYSTEM_PATH: ${{ vars.AZURE_ADLS_GEN2_FILESYSTEM_PATH }} AZURE_ADLS_GEN2_FILESYSTEM: ${{ vars.AZURE_ADLS_GEN2_FILESYSTEM }} + DEPLOYMENT_TARGET: ${{ vars.DEPLOYMENT_TARGET }} + AZURE_CONTAINER_APPS_WORKLOAD_PROFILE: ${{ vars.AZURE_CONTAINER_APPS_WORKLOAD_PROFILE }} steps: - name: Checkout diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index efffe0b9a8..d734be650d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,6 +22,7 @@ contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additio - [Running unit tests](#running-unit-tests) - [Running E2E tests](#running-e2e-tests) - [Code Style](#code-style) +- [Adding new azd environment variables](#add-new-azd-environment-variables) ## Code of Conduct @@ -160,3 +161,10 @@ python -m black ``` If you followed the steps above to install the pre-commit hooks, then you can just wait for those hooks to run `ruff` and `black` for you. + +## Adding new azd environment variables + +When adding new azd environment variables, please remember to update: +1. App Service's [azure.yaml](./azure.yaml) +1. [ADO pipeline](.azdo/pipelines/azure-dev.yml). +1. [Github workflows](.github/workflows/azure-dev.yml) diff --git a/README.md b/README.md index 5378774959..e7e77713e6 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ Pricing varies per region and usage, so it isn't possible to predict exact costs However, you can try the [Azure pricing calculator](https://azure.com/e/a87a169b256e43c089015fda8182ca87) for the resources below. - Azure App Service: Basic Tier with 1 CPU core, 1.75 GB RAM. Pricing per hour. [Pricing](https://azure.microsoft.com/pricing/details/app-service/linux/) +- Azure Container Apps: Only provisioned if you deploy to Azure Container Apps following [the ACA deployment guide](docs/azure_container_apps.md). Consumption plan with 1 CPU core, 2.0 GB RAM. Pricing with Pay-as-You-Go. [Pricing](https://azure.microsoft.com/pricing/details/container-apps/) - Azure OpenAI: Standard tier, GPT and Ada models. Pricing per 1K tokens used, and at least 1K tokens are used per question. [Pricing](https://azure.microsoft.com/pricing/details/cognitive-services/openai-service/) - Azure AI Document Intelligence: SO (Standard) tier using pre-built layout. Pricing per document page, sample documents have 261 pages total. [Pricing](https://azure.microsoft.com/pricing/details/form-recognizer/) - Azure AI Search: Basic tier, 1 replica, free level of semantic search. Pricing per hour. [Pricing](https://azure.microsoft.com/pricing/details/search/) @@ -126,7 +127,7 @@ A related option is VS Code Dev Containers, which will open the project in your ## Deploying -Follow these steps to provision Azure resources and deploy the application code: +The steps below will provision Azure resources and deploy the application code to Azure App Service. To deploy to Azure Container Apps instead, follow [the container apps deployment guide](docs/azure_container_apps.md). 1. Login to your Azure account: @@ -134,9 +135,9 @@ Follow these steps to provision Azure resources and deploy the application code: azd auth login ``` - For GitHub Codespaces users, if the previous command fails, try: + For GitHub Codespaces users, if the previous command fails, try: ```shell - azd auth login --use-device-code + azd auth login --use-device-code ``` 1. Create a new azd environment: diff --git a/app/backend/.dockerignore b/app/backend/.dockerignore new file mode 100644 index 0000000000..9008115fc8 --- /dev/null +++ b/app/backend/.dockerignore @@ -0,0 +1,7 @@ +.git +__pycache__ +*.pyc +*.pyo +*.pyd +.Python +env diff --git a/app/backend/Dockerfile b/app/backend/Dockerfile new file mode 100644 index 0000000000..a84bd6e0b7 --- /dev/null +++ b/app/backend/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.11-bullseye + +WORKDIR /app + +COPY ./ /app + +RUN python -m pip install -r requirements.txt + +RUN python -m pip install gunicorn + +CMD ["python3", "-m", "gunicorn", "-b", "0.0.0.0:8000", "main:app"] diff --git a/app/backend/app.py b/app/backend/app.py index aea6a52765..226ef98506 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -440,13 +440,25 @@ async def setup_clients(): USE_SPEECH_OUTPUT_BROWSER = os.getenv("USE_SPEECH_OUTPUT_BROWSER", "").lower() == "true" USE_SPEECH_OUTPUT_AZURE = os.getenv("USE_SPEECH_OUTPUT_AZURE", "").lower() == "true" + # WEBSITE_HOSTNAME is always set by App Service, RUNNING_IN_PRODUCTION is set in main.bicep + RUNNING_ON_AZURE = os.getenv("WEBSITE_HOSTNAME") is not None or os.getenv("RUNNING_IN_PRODUCTION") is not None + # Use the current user identity for keyless authentication to Azure services. # This assumes you use 'azd auth login' locally, and managed identity when deployed on Azure. # The managed identity is setup in the infra/ folder. azure_credential: Union[AzureDeveloperCliCredential, ManagedIdentityCredential] - if os.getenv("WEBSITE_HOSTNAME"): # Environment variable set on Azure Web Apps + if RUNNING_ON_AZURE: current_app.logger.info("Setting up Azure credential using ManagedIdentityCredential") - azure_credential = ManagedIdentityCredential() + if AZURE_CLIENT_ID := os.getenv("AZURE_CLIENT_ID"): + # ManagedIdentityCredential should use AZURE_CLIENT_ID if set in env, but its not working for some reason, + # so we explicitly pass it in as the client ID here. This is necessary for user-assigned managed identities. + current_app.logger.info( + "Setting up Azure credential using ManagedIdentityCredential with client_id %s", AZURE_CLIENT_ID + ) + azure_credential = ManagedIdentityCredential(client_id=AZURE_CLIENT_ID) + else: + current_app.logger.info("Setting up Azure credential using ManagedIdentityCredential") + azure_credential = ManagedIdentityCredential() elif AZURE_TENANT_ID: current_app.logger.info( "Setting up Azure credential using AzureDeveloperCliCredential with tenant_id %s", AZURE_TENANT_ID diff --git a/azure.yaml b/azure.yaml index 4deeba5c78..6a999fd140 100644 --- a/azure.yaml +++ b/azure.yaml @@ -7,9 +7,11 @@ services: backend: project: ./app/backend language: py + # Please check docs/azure_container_apps.md for more information on how to deploy to Azure Container Apps + # host: containerapp host: appservice hooks: - prepackage: + prebuild: windows: shell: pwsh run: cd ../frontend;npm install;npm run build @@ -86,6 +88,8 @@ pipeline: - AZURE_ADLS_GEN2_STORAGE_ACCOUNT - AZURE_ADLS_GEN2_FILESYSTEM_PATH - AZURE_ADLS_GEN2_FILESYSTEM + - DEPLOYMENT_TARGET + - AZURE_CONTAINER_APPS_WORKLOAD_PROFILE secrets: - AZURE_SERVER_APP_SECRET - AZURE_CLIENT_APP_SECRET diff --git a/docs/appservice.md b/docs/appservice.md index 60fffe7cb5..0fbba03279 100644 --- a/docs/appservice.md +++ b/docs/appservice.md @@ -631,15 +631,17 @@ To see any exceptions and server errors, navigate to the _Investigate -> Failure ## Configuring log levels -By default, the deployed app only logs messages with a level of `WARNING` or higher. +By default, the deployed app only logs messages from packages with a level of `WARNING` or higher, +but logs all messages from the app with a level of `INFO` or higher. These lines of code in `app/backend/app.py` configure the logging level: ```python +# Set root level to WARNING to avoid seeing overly verbose logs from SDKS +logging.basicConfig(level=logging.WARNING) +# Set the app logger level to INFO by default default_level = "INFO" -if os.getenv("WEBSITE_HOSTNAME"): # In production, don't log as heavily - default_level = "WARNING" -logging.basicConfig(level=os.getenv("APP_LOG_LEVEL", default_level)) +app.logger.setLevel(os.getenv("APP_LOG_LEVEL", default_level)) ``` To change the default level, either change `default_level` or set the `APP_LOG_LEVEL` environment variable diff --git a/docs/azure_container_apps.md b/docs/azure_container_apps.md new file mode 100644 index 0000000000..9fb1854007 --- /dev/null +++ b/docs/azure_container_apps.md @@ -0,0 +1,55 @@ +# Deploying on Azure Container Apps + +Due to [a limitation](https://github.com/Azure/azure-dev/issues/2736) of the Azure Developer CLI (`azd`), there can be only one host option in the [azure.yaml](../azure.yaml) file. +By default, `host: appservice` is used and `host: containerapp` is commented out. + +To deploy to Azure Container Apps, please follow the following steps: + +1. Comment out `host: appservice` and uncomment `host: containerapp` in the [azure.yaml](../azure.yaml) file. + +2. Login to your Azure account: + + ```bash + azd auth login + ``` + +3. Create a new `azd` environment to store the deployment parameters: + + ```bash + azd env new + ``` + + Enter a name that will be used for the resource group. + This will create a new folder in the `.azure` folder, and set it as the active environment for any calls to `azd` going forward. + +4. Set the deployment target to `containerapps`: + + ```bash + azd env set DEPLOYMENT_TARGET containerapps + ``` + +5. (Optional) This is the point where you can customize the deployment by setting other `azd1 environment variables, in order to [use existing resources](docs/deploy_existing.md), [enable optional features (such as auth or vision)](docs/deploy_features.md), or [deploy to free tiers](docs/deploy_lowcost.md). +6. Provision the resources and deploy the code: + + ```bash + azd up + ``` + + This will provision Azure resources and deploy this sample to those resources, including building the search index based on the files found in the `./data` folder. + + **Important**: Beware that the resources created by this command will incur immediate costs, primarily from the AI Search resource. These resources may accrue costs even if you interrupt the command before it is fully executed. You can run `azd down` or delete the resources manually to avoid unnecessary spending. + +## Customizing Workload Profile + +The default workload profile is Consumption. If you want to use a dedicated workload profile like D4, please run: + +```bash +azd env AZURE_CONTAINER_APPS_WORKLOAD_PROFILE D4 +``` + +For a full list of workload profiles, please check [here](https://learn.microsoft.com/azure/container-apps/workload-profiles-overview#profile-types). +Please note dedicated workload profiles have a different billing model than Consumption plan. Please check [here](https://learn.microsoft.com/azure/container-apps/billing) for details. + +## Private endpoints + +Private endpoints is still in private preview for Azure Conainer Apps and not supported for now. diff --git a/infra/abbreviations.json b/infra/abbreviations.json index 75959585e8..5084711603 100644 --- a/infra/abbreviations.json +++ b/infra/abbreviations.json @@ -135,6 +135,7 @@ "virtualNetworks": "vnet-", "webServerFarms": "plan-", "webSitesAppService": "app-", + "webSitesContainerApps": "capps-", "webSitesAppServiceEnvironment": "ase-", "webSitesFunctions": "func-", "webStaticSites": "stapp-" diff --git a/infra/core/host/container-app-upsert.bicep b/infra/core/host/container-app-upsert.bicep new file mode 100644 index 0000000000..ac2c77db1d --- /dev/null +++ b/infra/core/host/container-app-upsert.bicep @@ -0,0 +1,130 @@ +metadata description = 'Creates or updates an existing Azure Container App.' +param name string +param location string = resourceGroup().location +param tags object = {} + + +@description('The number of CPU cores allocated to a single container instance, e.g., 0.5') +param containerCpuCoreCount string = '0.5' + +@description('The maximum number of replicas to run. Must be at least 1.') +@minValue(1) +param containerMaxReplicas int = 10 + +@description('The amount of memory allocated to a single container instance, e.g., 1Gi') +param containerMemory string = '1.0Gi' + +@description('The minimum number of replicas to run. Must be at least 1.') +@minValue(1) +param containerMinReplicas int = 1 + +@description('The name of the container') +param containerName string = 'main' + +@description('The environment name for the container apps') +param containerAppsEnvironmentName string = '${containerName}env' + +@description('The name of the container registry') +param containerRegistryName string + +@description('Hostname suffix for container registry. Set when deploying to sovereign clouds') +param containerRegistryHostSuffix string = 'azurecr.io' + +@allowed(['http', 'grpc']) +@description('The protocol used by Dapr to connect to the app, e.g., HTTP or gRPC') +param daprAppProtocol string = 'http' + +@description('Enable or disable Dapr for the container app') +param daprEnabled bool = false + +@description('The Dapr app ID') +param daprAppId string = containerName + +@description('Specifies if the resource already exists') +param exists bool = false + +@description('Specifies if Ingress is enabled for the container app') +param ingressEnabled bool = true + +@description('The type of identity for the resource') +@allowed(['None', 'SystemAssigned', 'UserAssigned']) +param identityType string = 'None' + +@description('The name of the user-assigned identity') +param identityName string = '' + +@description('The name of the container image') +param imageName string = '' + +@description('The secrets required for the container') +@secure() +param secrets object = {} + +@description('The keyvault identities required for the container') +@secure() +param keyvaultIdentities object = {} + +@description('The environment variables for the container in key value pairs') +param env object = {} + +@description('Specifies if the resource ingress is exposed externally') +param external bool = true + +@description('The service binds associated with the container') +param serviceBinds array = [] + +@description('The target port for the container') +param targetPort int = 80 + +@allowed(['Consumption', 'D4', 'D8', 'D16', 'D32', 'E4', 'E8', 'E16', 'E32', 'NC24-A100', 'NC48-A100', 'NC96-A100']) +param workloadProfile string = 'Consumption' + +param allowedOrigins array = [] + +resource existingApp 'Microsoft.App/containerApps@2023-05-02-preview' existing = if (exists) { + name: name +} + +module app 'container-app.bicep' = { + name: '${deployment().name}-update' + params: { + name: name + workloadProfile: workloadProfile + location: location + tags: tags + identityType: identityType + identityName: identityName + ingressEnabled: ingressEnabled + containerName: containerName + containerAppsEnvironmentName: containerAppsEnvironmentName + containerRegistryName: containerRegistryName + containerRegistryHostSuffix: containerRegistryHostSuffix + containerCpuCoreCount: containerCpuCoreCount + containerMemory: containerMemory + containerMinReplicas: containerMinReplicas + containerMaxReplicas: containerMaxReplicas + daprEnabled: daprEnabled + daprAppId: daprAppId + daprAppProtocol: daprAppProtocol + secrets: secrets + keyvaultIdentities: keyvaultIdentities + allowedOrigins: allowedOrigins + external: external + env: [ + for key in objectKeys(env): { + name: key + value: '${env[key]}' + } + ] + imageName: !empty(imageName) ? imageName : exists ? existingApp.properties.template.containers[0].image : '' + targetPort: targetPort + serviceBinds: serviceBinds + } +} + +output defaultDomain string = app.outputs.defaultDomain +output imageName string = app.outputs.imageName +output name string = app.outputs.name +output uri string = app.outputs.uri +output id string = app.outputs.id +output identityPrincipalId string = app.outputs.identityPrincipalId diff --git a/infra/core/host/container-app.bicep b/infra/core/host/container-app.bicep new file mode 100644 index 0000000000..af41cb91e2 --- /dev/null +++ b/infra/core/host/container-app.bicep @@ -0,0 +1,185 @@ +metadata description = 'Creates a container app in an Azure Container App environment.' +param name string +param location string = resourceGroup().location +param tags object = {} + +@description('Allowed origins') +param allowedOrigins array = [] + +@description('Name of the environment for container apps') +param containerAppsEnvironmentName string + +@description('CPU cores allocated to a single container instance, e.g., 0.5') +param containerCpuCoreCount string = '0.5' + +@description('The maximum number of replicas to run. Must be at least 1.') +@minValue(1) +param containerMaxReplicas int = 10 + +@description('Memory allocated to a single container instance, e.g., 1Gi') +param containerMemory string = '1.0Gi' + +@description('The minimum number of replicas to run. Must be at least 1.') +param containerMinReplicas int = 1 + +@description('The name of the container') +param containerName string = 'main' + +@description('The name of the container registry') +param containerRegistryName string = '' + +@description('Hostname suffix for container registry. Set when deploying to sovereign clouds') +param containerRegistryHostSuffix string = 'azurecr.io' + +@description('The protocol used by Dapr to connect to the app, e.g., http or grpc') +@allowed([ 'http', 'grpc' ]) +param daprAppProtocol string = 'http' + +@description('The Dapr app ID') +param daprAppId string = containerName + +@description('Enable Dapr') +param daprEnabled bool = false + +@description('The environment variables for the container') +param env array = [] + +@description('Specifies if the resource ingress is exposed externally') +param external bool = true + +@description('The name of the user-assigned identity') +param identityName string = '' + +@description('The type of identity for the resource') +@allowed([ 'None', 'SystemAssigned', 'UserAssigned' ]) +param identityType string = 'None' + +@description('The name of the container image') +param imageName string = '' + +@description('Specifies if Ingress is enabled for the container app') +param ingressEnabled bool = true + +param revisionMode string = 'Single' + +@description('The secrets required for the container') +@secure() +param secrets object = {} + +@description('The keyvault identities required for the container') +@secure() +param keyvaultIdentities object = {} + +@description('The service binds associated with the container') +param serviceBinds array = [] + +@description('The name of the container apps add-on to use. e.g. redis') +param serviceType string = '' + +@description('The target port for the container') +param targetPort int = 80 + +param workloadProfile string = 'Consumption' + +resource userIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = if (!empty(identityName)) { + name: identityName +} + +// Private registry support requires both an ACR name and a User Assigned managed identity +var usePrivateRegistry = !empty(identityName) && !empty(containerRegistryName) + +// Automatically set to `UserAssigned` when an `identityName` has been set +var normalizedIdentityType = !empty(identityName) ? 'UserAssigned' : identityType + +var keyvalueSecrets = [for secret in items(secrets): { + name: secret.key + value: secret.value +}] + +var keyvaultIdentitySecrets = [for secret in items(keyvaultIdentities): { + name: secret.key + keyVaultUrl: secret.value.keyVaultUrl + identity: secret.value.identity +}] + +module containerRegistryAccess '../security/registry-access.bicep' = if (usePrivateRegistry) { + name: '${deployment().name}-registry-access' + params: { + containerRegistryName: containerRegistryName + principalId: usePrivateRegistry ? userIdentity.properties.principalId : '' + } +} + +resource app 'Microsoft.App/containerApps@2023-05-02-preview' = { + name: name + location: location + tags: tags + // It is critical that the identity is granted ACR pull access before the app is created + // otherwise the container app will throw a provision error + // This also forces us to use an user assigned managed identity since there would no way to + // provide the system assigned identity with the ACR pull access before the app is created + dependsOn: usePrivateRegistry ? [ containerRegistryAccess ] : [] + identity: { + type: normalizedIdentityType + userAssignedIdentities: !empty(identityName) && normalizedIdentityType == 'UserAssigned' ? { '${userIdentity.id}': {} } : null + } + properties: { + managedEnvironmentId: containerAppsEnvironment.id + workloadProfileName: workloadProfile + configuration: { + activeRevisionsMode: revisionMode + ingress: ingressEnabled ? { + external: external + targetPort: targetPort + transport: 'auto' + corsPolicy: { + allowedOrigins: union([ 'https://portal.azure.com', 'https://ms.portal.azure.com' ], allowedOrigins) + } + } : null + dapr: daprEnabled ? { + enabled: true + appId: daprAppId + appProtocol: daprAppProtocol + appPort: ingressEnabled ? targetPort : 0 + } : { enabled: false } + secrets: concat(keyvalueSecrets, keyvaultIdentitySecrets) + service: !empty(serviceType) ? { type: serviceType } : null + registries: usePrivateRegistry ? [ + { + server: '${containerRegistryName}.${containerRegistryHostSuffix}' + identity: userIdentity.id + } + ] : [] + } + template: { + serviceBinds: !empty(serviceBinds) ? serviceBinds : null + containers: [ + { + image: !empty(imageName) ? imageName : 'mcr.microsoft.com/azuredocs/containerapps-helloworld:latest' + name: containerName + env: env + resources: { + cpu: json(containerCpuCoreCount) + memory: containerMemory + } + } + ] + scale: { + minReplicas: containerMinReplicas + maxReplicas: containerMaxReplicas + } + } + } +} + +resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' existing = { + name: containerAppsEnvironmentName +} + +output defaultDomain string = containerAppsEnvironment.properties.defaultDomain +output identityPrincipalId string = normalizedIdentityType == 'None' ? '' : (empty(identityName) ? app.identity.principalId : userIdentity.properties.principalId) +output imageName string = imageName +output name string = app.name +output serviceBind object = !empty(serviceType) ? { serviceId: app.id, name: name } : {} +output uri string = ingressEnabled ? 'https://${app.properties.configuration.ingress.fqdn}' : '' +output id string = app.id diff --git a/infra/core/host/container-apps.bicep b/infra/core/host/container-apps.bicep new file mode 100644 index 0000000000..81646daba0 --- /dev/null +++ b/infra/core/host/container-apps.bicep @@ -0,0 +1,80 @@ +metadata description = 'Creates an Azure Container Registry and an Azure Container Apps environment.' +param name string +param location string = resourceGroup().location +param tags object = {} + +param containerAppsEnvironmentName string +param containerRegistryName string +param containerRegistryResourceGroupName string = '' +param containerRegistryAdminUserEnabled bool = false +param logAnalyticsWorkspaceResourceId string +param applicationInsightsName string = '' // Not used here, was used for DAPR +param virtualNetworkSubnetId string = '' +@allowed(['Consumption', 'D4', 'D8', 'D16', 'D32', 'E4', 'E8', 'E16', 'E32', 'NC24-A100', 'NC48-A100', 'NC96-A100']) +param workloadProfile string + +var workloadProfiles = workloadProfile == 'Consumption' + ? [ + { + name: 'Consumption' + workloadProfileType: 'Consumption' + } + ] + : [ + { + name: 'Consumption' + workloadProfileType: 'Consumption' + } + { + minimumCount: 0 + maximumCount: 2 + name: workloadProfile + workloadProfileType: workloadProfile + } + ] + +@description('Optional user assigned identity IDs to assign to the resource') +param userAssignedIdentityResourceIds array = [] + +module containerAppsEnvironment 'br/public:avm/res/app/managed-environment:0.5.2' = { + name: '${name}-container-apps-environment' + params: { + // Required parameters + logAnalyticsWorkspaceResourceId: logAnalyticsWorkspaceResourceId + + managedIdentities: empty(userAssignedIdentityResourceIds) ? { + systemAssigned: true + } : { + userAssignedResourceIds: userAssignedIdentityResourceIds + } + + name: containerAppsEnvironmentName + // Non-required parameters + infrastructureResourceGroupName: containerRegistryResourceGroupName + infrastructureSubnetId: virtualNetworkSubnetId + location: location + tags: tags + zoneRedundant: false + workloadProfiles: workloadProfiles + } +} + +module containerRegistry 'br/public:avm/res/container-registry/registry:0.3.1' = { + name: '${name}-container-registry' + scope: !empty(containerRegistryResourceGroupName) + ? resourceGroup(containerRegistryResourceGroupName) + : resourceGroup() + params: { + name: containerRegistryName + location: location + acrAdminUserEnabled: containerRegistryAdminUserEnabled + tags: tags + } +} + +output defaultDomain string = containerAppsEnvironment.outputs.defaultDomain +output environmentName string = containerAppsEnvironment.outputs.name +output environmentId string = containerAppsEnvironment.outputs.resourceId + +output registryLoginServer string = containerRegistry.outputs.loginServer +output registryName string = containerRegistry.outputs.name diff --git a/infra/core/security/aca-identity.bicep b/infra/core/security/aca-identity.bicep new file mode 100644 index 0000000000..a7fcd5a459 --- /dev/null +++ b/infra/core/security/aca-identity.bicep @@ -0,0 +1,10 @@ +param identityName string +param location string + +resource webIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: identityName + location: location +} + +output principalId string = webIdentity.properties.principalId +output clientId string = webIdentity.properties.clientId diff --git a/infra/core/security/registry-access.bicep b/infra/core/security/registry-access.bicep new file mode 100644 index 0000000000..fc66837a12 --- /dev/null +++ b/infra/core/security/registry-access.bicep @@ -0,0 +1,19 @@ +metadata description = 'Assigns ACR Pull permissions to access an Azure Container Registry.' +param containerRegistryName string +param principalId string + +var acrPullRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d') + +resource aksAcrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: containerRegistry // Use when specifying a scope that is different than the deployment scope + name: guid(subscription().id, resourceGroup().id, principalId, acrPullRole) + properties: { + roleDefinitionId: acrPullRole + principalType: 'ServicePrincipal' + principalId: principalId + } +} + +resource containerRegistry 'Microsoft.ContainerRegistry/registries@2023-01-01-preview' existing = { + name: containerRegistryName +} diff --git a/infra/main.bicep b/infra/main.bicep index c132e39113..04c4832c10 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -206,6 +206,20 @@ param runningOnGh string = '' @description('Whether the deployment is running on Azure DevOps Pipeline') param runningOnAdo string = '' +@description('Used by azd for containerapps deployment') +param webAppExists bool + +@allowed(['Consumption', 'D4', 'D8', 'D16', 'D32', 'E4', 'E8', 'E16', 'E32', 'NC24-A100', 'NC48-A100', 'NC96-A100']) +param azureContainerAppsWorkloadProfile string + +@allowed(['appservice', 'containerapps']) +param deploymentTarget string = 'appservice' +param acaIdentityName string = deploymentTarget == 'containerapps' ? '${environmentName}-aca-identity' : '' +param acaManagedEnvironmentName string = deploymentTarget == 'containerapps' ? '${environmentName}-aca-env' : '' +param containerRegistryName string = deploymentTarget == 'containerapps' + ? '${replace(environmentName, '-', '')}acr' + : '' + // Organize resources in a resource group resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { name: !empty(resourceGroupName) ? resourceGroupName : '${abbrs.resourcesResourceGroups}${environmentName}' @@ -267,7 +281,7 @@ module applicationInsightsDashboard 'backend-dashboard.bicep' = if (useApplicati } // Create an App Service Plan to group applications under the same payment plan and SKU -module appServicePlan 'core/host/appserviceplan.bicep' = { +module appServicePlan 'core/host/appserviceplan.bicep' = if (deploymentTarget == 'appservice') { name: 'appserviceplan' scope: resourceGroup params: { @@ -282,15 +296,76 @@ module appServicePlan 'core/host/appserviceplan.bicep' = { } } -// The application frontend -module backend 'core/host/appservice.bicep' = { +var appEnvVariables = { + AZURE_STORAGE_ACCOUNT: storage.outputs.name + AZURE_STORAGE_CONTAINER: storageContainerName + AZURE_SEARCH_INDEX: searchIndexName + AZURE_SEARCH_SERVICE: searchService.outputs.name + AZURE_SEARCH_SEMANTIC_RANKER: actualSearchServiceSemanticRankerLevel + AZURE_VISION_ENDPOINT: useGPT4V ? computerVision.outputs.endpoint : '' + AZURE_SEARCH_QUERY_LANGUAGE: searchQueryLanguage + AZURE_SEARCH_QUERY_SPELLER: searchQuerySpeller + APPLICATIONINSIGHTS_CONNECTION_STRING: useApplicationInsights + ? monitoring.outputs.applicationInsightsConnectionString + : '' + AZURE_SPEECH_SERVICE_ID: useSpeechOutputAzure ? speech.outputs.resourceId : '' + AZURE_SPEECH_SERVICE_LOCATION: useSpeechOutputAzure ? speech.outputs.location : '' + ENABLE_LANGUAGE_PICKER: enableLanguagePicker + USE_SPEECH_INPUT_BROWSER: useSpeechInputBrowser + USE_SPEECH_OUTPUT_BROWSER: useSpeechOutputBrowser + USE_SPEECH_OUTPUT_AZURE: useSpeechOutputAzure + // Shared by all OpenAI deployments + OPENAI_HOST: openAiHost + AZURE_OPENAI_EMB_MODEL_NAME: embedding.modelName + AZURE_OPENAI_EMB_DIMENSIONS: embedding.dimensions + AZURE_OPENAI_CHATGPT_MODEL: chatGpt.modelName + AZURE_OPENAI_GPT4V_MODEL: gpt4vModelName + // Specific to Azure OpenAI + AZURE_OPENAI_SERVICE: isAzureOpenAiHost && deployAzureOpenAi ? openAi.outputs.name : '' + AZURE_OPENAI_CHATGPT_DEPLOYMENT: chatGpt.deploymentName + AZURE_OPENAI_EMB_DEPLOYMENT: embedding.deploymentName + AZURE_OPENAI_GPT4V_DEPLOYMENT: useGPT4V ? gpt4vDeploymentName : '' + AZURE_OPENAI_API_VERSION: azureOpenAiApiVersion + AZURE_OPENAI_API_KEY_OVERRIDE: azureOpenAiApiKey + AZURE_OPENAI_CUSTOM_URL: azureOpenAiCustomUrl + // Used only with non-Azure OpenAI deployments + OPENAI_API_KEY: openAiApiKey + OPENAI_ORGANIZATION: openAiApiOrganization + // Optional login and document level access control system + AZURE_USE_AUTHENTICATION: useAuthentication + AZURE_ENFORCE_ACCESS_CONTROL: enforceAccessControl + AZURE_ENABLE_GLOBAL_DOCUMENT_ACCESS: enableGlobalDocuments + AZURE_ENABLE_UNAUTHENTICATED_ACCESS: enableUnauthenticatedAccess + AZURE_SERVER_APP_ID: serverAppId + AZURE_SERVER_APP_SECRET: serverAppSecret + AZURE_CLIENT_APP_ID: clientAppId + AZURE_CLIENT_APP_SECRET: clientAppSecret + AZURE_TENANT_ID: tenantId + AZURE_AUTH_TENANT_ID: tenantIdForAuth + AZURE_AUTHENTICATION_ISSUER_URI: authenticationIssuerUri + // CORS support, for frontends on other hosts + ALLOWED_ORIGIN: allowedOrigin + USE_VECTORS: useVectors + USE_GPT4V: useGPT4V + USE_USER_UPLOAD: useUserUpload + AZURE_USERSTORAGE_ACCOUNT: useUserUpload ? userStorage.outputs.name : '' + AZURE_USERSTORAGE_CONTAINER: useUserUpload ? userStorageContainerName : '' + AZURE_DOCUMENTINTELLIGENCE_SERVICE: documentIntelligence.outputs.name + USE_LOCAL_PDF_PARSER: useLocalPdfParser + USE_LOCAL_HTML_PARSER: useLocalHtmlParser + RUNNING_IN_PRODUCTION: 'true' +} + +// App Service for the web application (Python Quart app with JS frontend) +module backend 'core/host/appservice.bicep' = if (deploymentTarget == 'appservice') { name: 'web' scope: resourceGroup params: { name: !empty(backendServiceName) ? backendServiceName : '${abbrs.webSitesAppService}backend-${resourceToken}' location: location tags: union(tags, { 'azd-service-name': 'backend' }) - appServicePlanId: appServicePlan.outputs.id + // Need to check deploymentTarget again due to https://github.com/Azure/bicep/issues/3990 + appServicePlanId: deploymentTarget == 'appservice' ? appServicePlan.outputs.id : '' runtimeName: 'python' runtimeVersion: '3.11' appCommandLine: 'python3 -m gunicorn main:app' @@ -307,64 +382,62 @@ module backend 'core/host/appservice.bicep' = { authenticationIssuerUri: authenticationIssuerUri use32BitWorkerProcess: appServiceSkuName == 'F1' alwaysOn: appServiceSkuName != 'F1' - appSettings: { - AZURE_STORAGE_ACCOUNT: storage.outputs.name - AZURE_STORAGE_CONTAINER: storageContainerName - AZURE_SEARCH_INDEX: searchIndexName - AZURE_SEARCH_SERVICE: searchService.outputs.name - AZURE_SEARCH_SEMANTIC_RANKER: actualSearchServiceSemanticRankerLevel - AZURE_VISION_ENDPOINT: useGPT4V ? computerVision.outputs.endpoint : '' - AZURE_SEARCH_QUERY_LANGUAGE: searchQueryLanguage - AZURE_SEARCH_QUERY_SPELLER: searchQuerySpeller - APPLICATIONINSIGHTS_CONNECTION_STRING: useApplicationInsights - ? monitoring.outputs.applicationInsightsConnectionString - : '' - AZURE_SPEECH_SERVICE_ID: useSpeechOutputAzure ? speech.outputs.resourceId : '' - AZURE_SPEECH_SERVICE_LOCATION: useSpeechOutputAzure ? speech.outputs.location : '' - ENABLE_LANGUAGE_PICKER: enableLanguagePicker - USE_SPEECH_INPUT_BROWSER: useSpeechInputBrowser - USE_SPEECH_OUTPUT_BROWSER: useSpeechOutputBrowser - USE_SPEECH_OUTPUT_AZURE: useSpeechOutputAzure - // Shared by all OpenAI deployments - OPENAI_HOST: openAiHost - AZURE_OPENAI_EMB_MODEL_NAME: embedding.modelName - AZURE_OPENAI_EMB_DIMENSIONS: embedding.dimensions - AZURE_OPENAI_CHATGPT_MODEL: chatGpt.modelName - AZURE_OPENAI_GPT4V_MODEL: gpt4vModelName - // Specific to Azure OpenAI - AZURE_OPENAI_SERVICE: isAzureOpenAiHost && deployAzureOpenAi ? openAi.outputs.name : '' - AZURE_OPENAI_CHATGPT_DEPLOYMENT: chatGpt.deploymentName - AZURE_OPENAI_EMB_DEPLOYMENT: embedding.deploymentName - AZURE_OPENAI_GPT4V_DEPLOYMENT: useGPT4V ? gpt4vDeploymentName : '' - AZURE_OPENAI_API_VERSION: azureOpenAiApiVersion - AZURE_OPENAI_API_KEY_OVERRIDE: azureOpenAiApiKey - AZURE_OPENAI_CUSTOM_URL: azureOpenAiCustomUrl - // Used only with non-Azure OpenAI deployments - OPENAI_API_KEY: openAiApiKey - OPENAI_ORGANIZATION: openAiApiOrganization - // Optional login and document level access control system - AZURE_USE_AUTHENTICATION: useAuthentication - AZURE_ENFORCE_ACCESS_CONTROL: enforceAccessControl - AZURE_ENABLE_GLOBAL_DOCUMENT_ACCESS: enableGlobalDocuments - AZURE_ENABLE_UNAUTHENTICATED_ACCESS: enableUnauthenticatedAccess - AZURE_SERVER_APP_ID: serverAppId - AZURE_SERVER_APP_SECRET: serverAppSecret - AZURE_CLIENT_APP_ID: clientAppId - AZURE_CLIENT_APP_SECRET: clientAppSecret - AZURE_TENANT_ID: tenantId - AZURE_AUTH_TENANT_ID: tenantIdForAuth - AZURE_AUTHENTICATION_ISSUER_URI: authenticationIssuerUri - // CORS support, for frontends on other hosts - ALLOWED_ORIGIN: allowedOrigin - USE_VECTORS: useVectors - USE_GPT4V: useGPT4V - USE_USER_UPLOAD: useUserUpload - AZURE_USERSTORAGE_ACCOUNT: useUserUpload ? userStorage.outputs.name : '' - AZURE_USERSTORAGE_CONTAINER: useUserUpload ? userStorageContainerName : '' - AZURE_DOCUMENTINTELLIGENCE_SERVICE: documentIntelligence.outputs.name - USE_LOCAL_PDF_PARSER: useLocalPdfParser - USE_LOCAL_HTML_PARSER: useLocalHtmlParser - } + appSettings: appEnvVariables + } +} + +// Azure container apps resources (Only deployed if deploymentTarget is 'containerapps') + +// User-assigned identity for pulling images from ACR +module acaIdentity 'core/security/aca-identity.bicep' = if (deploymentTarget == 'containerapps') { + name: 'aca-identity' + scope: resourceGroup + params: { + identityName: acaIdentityName + location: location + } +} + +module containerApps 'core/host/container-apps.bicep' = if (deploymentTarget == 'containerapps') { + name: 'container-apps' + scope: resourceGroup + params: { + name: 'app' + tags: tags + location: location + workloadProfile: azureContainerAppsWorkloadProfile + containerAppsEnvironmentName: acaManagedEnvironmentName + containerRegistryName: '${containerRegistryName}${resourceToken}' + logAnalyticsWorkspaceResourceId: monitoring.outputs.logAnalyticsWorkspaceId + } +} + +// Container Apps for the web application (Python Quart app with JS frontend) +module acaBackend 'core/host/container-app-upsert.bicep' = if (deploymentTarget == 'containerapps') { + name: 'aca-web' + scope: resourceGroup + dependsOn: [ + containerApps + acaIdentity + ] + params: { + name: !empty(backendServiceName) ? backendServiceName : '${abbrs.webSitesContainerApps}backend-${resourceToken}' + location: location + identityName: (deploymentTarget == 'containerapps') ? acaIdentityName : '' + exists: webAppExists + workloadProfile: azureContainerAppsWorkloadProfile + containerRegistryName: (deploymentTarget == 'containerapps') ? containerApps.outputs.registryName : '' + containerAppsEnvironmentName: (deploymentTarget == 'containerapps') ? containerApps.outputs.environmentName : '' + identityType: 'UserAssigned' + tags: union(tags, { 'azd-service-name': 'backend' }) + targetPort: 8000 + containerCpuCoreCount: '1.0' + containerMemory: '2Gi' + allowedOrigins: [allowedOrigin] + env: union(appEnvVariables, { + // For using managed identity to access Azure resources. See https://github.com/microsoft/azure-container-apps/issues/442 + AZURE_CLIENT_ID: (deploymentTarget == 'containerapps') ? acaIdentity.outputs.clientId : '' + }) } } @@ -678,7 +751,9 @@ module openAiRoleBackend 'core/security/role.bicep' = if (isAzureOpenAiHost && d scope: openAiResourceGroup name: 'openai-role-backend' params: { - principalId: backend.outputs.identityPrincipalId + principalId: (deploymentTarget == 'appservice') + ? backend.outputs.identityPrincipalId + : acaBackend.outputs.identityPrincipalId roleDefinitionId: '5e0bd9bd-7b93-4f28-af87-19fc36ad61bd' principalType: 'ServicePrincipal' } @@ -688,7 +763,9 @@ module openAiRoleSearchService 'core/security/role.bicep' = if (isAzureOpenAiHos scope: openAiResourceGroup name: 'openai-role-searchservice' params: { - principalId: searchService.outputs.principalId + principalId: (deploymentTarget == 'appservice') + ? backend.outputs.identityPrincipalId + : acaBackend.outputs.identityPrincipalId roleDefinitionId: '5e0bd9bd-7b93-4f28-af87-19fc36ad61bd' principalType: 'ServicePrincipal' } @@ -698,7 +775,9 @@ module storageRoleBackend 'core/security/role.bicep' = { scope: storageResourceGroup name: 'storage-role-backend' params: { - principalId: backend.outputs.identityPrincipalId + principalId: (deploymentTarget == 'appservice') + ? backend.outputs.identityPrincipalId + : acaBackend.outputs.identityPrincipalId roleDefinitionId: '2a2b9908-6ea1-4ae2-8e65-a410df84e7d1' principalType: 'ServicePrincipal' } @@ -708,7 +787,9 @@ module storageOwnerRoleBackend 'core/security/role.bicep' = if (useUserUpload) { scope: storageResourceGroup name: 'storage-owner-role-backend' params: { - principalId: backend.outputs.identityPrincipalId + principalId: (deploymentTarget == 'appservice') + ? backend.outputs.identityPrincipalId + : acaBackend.outputs.identityPrincipalId roleDefinitionId: 'b7e6dc6d-f1e8-4753-8033-0f276bb0955b' principalType: 'ServicePrincipal' } @@ -718,7 +799,9 @@ module storageRoleSearchService 'core/security/role.bicep' = if (useIntegratedVe scope: storageResourceGroup name: 'storage-role-searchservice' params: { - principalId: searchService.outputs.principalId + principalId: (deploymentTarget == 'appservice') + ? backend.outputs.identityPrincipalId + : acaBackend.outputs.identityPrincipalId roleDefinitionId: '2a2b9908-6ea1-4ae2-8e65-a410df84e7d1' principalType: 'ServicePrincipal' } @@ -730,7 +813,9 @@ module searchRoleBackend 'core/security/role.bicep' = { scope: searchServiceResourceGroup name: 'search-role-backend' params: { - principalId: backend.outputs.identityPrincipalId + principalId: (deploymentTarget == 'appservice') + ? backend.outputs.identityPrincipalId + : acaBackend.outputs.identityPrincipalId roleDefinitionId: '1407120a-92aa-4202-b7e9-c0e197c71c8f' principalType: 'ServicePrincipal' } @@ -740,7 +825,9 @@ module speechRoleBackend 'core/security/role.bicep' = { scope: speechResourceGroup name: 'speech-role-backend' params: { - principalId: backend.outputs.identityPrincipalId + principalId: (deploymentTarget == 'appservice') + ? backend.outputs.identityPrincipalId + : acaBackend.outputs.identityPrincipalId roleDefinitionId: 'f2dc8367-1007-4938-bd23-fe263f013447' principalType: 'ServicePrincipal' } @@ -750,17 +837,19 @@ module isolation 'network-isolation.bicep' = { name: 'networks' scope: resourceGroup params: { + deploymentTarget: deploymentTarget location: location tags: tags vnetName: '${abbrs.virtualNetworks}${resourceToken}' - appServicePlanName: appServicePlan.outputs.name + // Need to check deploymentTarget due to https://github.com/Azure/bicep/issues/3990 + appServicePlanName: deploymentTarget == 'appservice' ? appServicePlan.outputs.name : '' usePrivateEndpoint: usePrivateEndpoint } } var environmentData = environment() -var openAiPrivateEndpointConnection = (isAzureOpenAiHost && deployAzureOpenAi) +var openAiPrivateEndpointConnection = (isAzureOpenAiHost && deployAzureOpenAi && deploymentTarget == 'appservice') ? [ { groupId: 'account' @@ -773,7 +862,7 @@ var openAiPrivateEndpointConnection = (isAzureOpenAiHost && deployAzureOpenAi) } ] : [] -var otherPrivateEndpointConnections = usePrivateEndpoint +var otherPrivateEndpointConnections = (usePrivateEndpoint && deploymentTarget == 'appservice') ? [ { groupId: 'blob' @@ -795,7 +884,7 @@ var otherPrivateEndpointConnections = usePrivateEndpoint var privateEndpointConnections = concat(otherPrivateEndpointConnections, openAiPrivateEndpointConnection) -module privateEndpoints 'private-endpoints.bicep' = if (usePrivateEndpoint) { +module privateEndpoints 'private-endpoints.bicep' = if (usePrivateEndpoint && deploymentTarget == 'appservice') { name: 'privateEndpoints' scope: resourceGroup params: { @@ -816,7 +905,9 @@ module searchReaderRoleBackend 'core/security/role.bicep' = if (useAuthenticatio scope: searchServiceResourceGroup name: 'search-reader-role-backend' params: { - principalId: backend.outputs.identityPrincipalId + principalId: (deploymentTarget == 'appservice') + ? backend.outputs.identityPrincipalId + : acaBackend.outputs.identityPrincipalId roleDefinitionId: 'acdd72a7-3385-48ef-bd42-f606fba81ae7' principalType: 'ServicePrincipal' } @@ -827,7 +918,9 @@ module searchContribRoleBackend 'core/security/role.bicep' = if (useUserUpload) scope: searchServiceResourceGroup name: 'search-contrib-role-backend' params: { - principalId: backend.outputs.identityPrincipalId + principalId: (deploymentTarget == 'appservice') + ? backend.outputs.identityPrincipalId + : acaBackend.outputs.identityPrincipalId roleDefinitionId: '8ebe5a00-799e-43f5-93ac-243d3dce84a7' principalType: 'ServicePrincipal' } @@ -838,7 +931,9 @@ module computerVisionRoleBackend 'core/security/role.bicep' = if (useGPT4V) { scope: computerVisionResourceGroup name: 'computervision-role-backend' params: { - principalId: backend.outputs.identityPrincipalId + principalId: (deploymentTarget == 'appservice') + ? backend.outputs.identityPrincipalId + : acaBackend.outputs.identityPrincipalId roleDefinitionId: 'a97b65f3-24c7-4388-baec-2e87135dc908' principalType: 'ServicePrincipal' } @@ -849,7 +944,9 @@ module documentIntelligenceRoleBackend 'core/security/role.bicep' = if (useUserU scope: documentIntelligenceResourceGroup name: 'documentintelligence-role-backend' params: { - principalId: backend.outputs.identityPrincipalId + principalId: (deploymentTarget == 'appservice') + ? backend.outputs.identityPrincipalId + : acaBackend.outputs.identityPrincipalId roleDefinitionId: 'a97b65f3-24c7-4388-baec-2e87135dc908' principalType: 'ServicePrincipal' } @@ -898,4 +995,7 @@ output AZURE_USERSTORAGE_RESOURCE_GROUP string = storageResourceGroup.name output AZURE_USE_AUTHENTICATION bool = useAuthentication -output BACKEND_URI string = backend.outputs.uri +output BACKEND_URI string = deploymentTarget == 'appservice' ? backend.outputs.uri : acaBackend.outputs.uri +output AZURE_CONTAINER_REGISTRY_ENDPOINT string = deploymentTarget == 'containerapps' + ? containerApps.outputs.registryLoginServer + : '' diff --git a/infra/main.parameters.json b/infra/main.parameters.json index 2a4615d911..3575cd8d5b 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -235,6 +235,15 @@ }, "runningOnAdo": { "value": "${TF_BUILD}" + }, + "deploymentTarget": { + "value": "${DEPLOYMENT_TARGET=appservice}" + }, + "webAppExists": { + "value": "${SERVICE_WEB_RESOURCE_EXISTS=false}" + }, + "azureContainerAppsWorkloadProfile": { + "value": "${AZURE_CONTAINER_APPS_WORKLOAD_PROFILE=Consumption}" } } } diff --git a/infra/network-isolation.bicep b/infra/network-isolation.bicep index fcc69ba5ef..4dd1e49f86 100644 --- a/infra/network-isolation.bicep +++ b/infra/network-isolation.bicep @@ -14,7 +14,10 @@ param appServicePlanName string param usePrivateEndpoint bool = false -resource appServicePlan 'Microsoft.Web/serverfarms@2022-03-01' existing = { +@allowed(['appservice', 'containerapps']) +param deploymentTarget string + +resource appServicePlan 'Microsoft.Web/serverfarms@2022-03-01' existing = if (deploymentTarget == 'appservice') { name: appServicePlanName }