diff --git a/.azuredevops/pipelines/delete-review-app.yml b/.azuredevops/pipelines/delete-review-app.yml new file mode 100644 index 000000000..fd8cd977e --- /dev/null +++ b/.azuredevops/pipelines/delete-review-app.yml @@ -0,0 +1,46 @@ +trigger: none +pr: none + +parameters: + - name: commitSHA + displayName: Commit SHA + type: string + - name: prNumber + displayName: Pull request number + type: string + +stages: + - stage: review + displayName: Delete review app + pool: + name: private-pool-dev-uks + isSkippable: false + + jobs: + - deployment: DeleteReviewApp + displayName: Delete review app + environment: review + strategy: + runOnce: + deploy: + steps: + - checkout: self + + - task: TerraformInstaller@1 + displayName: Install terraform + inputs: + terraformVersion: 1.7.0 + + - task: AzureCLI@2 + displayName: Run terraform + inputs: + azureSubscription: manbrs-review + scriptType: bash + scriptLocation: inlineScript + addSpnToEnvironment: true + inlineScript: | + export ARM_TENANT_ID="$tenantId" + export ARM_CLIENT_ID="$servicePrincipalId" + export ARM_OIDC_TOKEN="$idToken" + export ARM_USE_OIDC=true + make ci review terraform-destroy DOCKER_IMAGE_TAG=git-sha-${{ parameters.commitSHA }} PR_NUMBER=${{ parameters.prNumber }} diff --git a/.azuredevops/pipelines/deploy.yml b/.azuredevops/pipelines/deploy.yml index c86281436..dc09953af 100644 --- a/.azuredevops/pipelines/deploy.yml +++ b/.azuredevops/pipelines/deploy.yml @@ -5,54 +5,56 @@ parameters: - name: commitSHA displayName: Commit SHA type: string - - name: environments - type: object - default: - - dev + - name: environment + displayName: Environment + type: string + - name: prNumber + displayName: Pull request number + type: string + default: '' stages: - - ${{ each env in parameters.environments }}: - - stage: ${{ env }} - displayName: Deploy to ${{ env }} environment - pool: - name: private-pool-dev-uks - lockBehavior: sequential - isSkippable: false + - stage: ${{ parameters.environment }} + displayName: Deploy to ${{ parameters.environment }} environment + pool: + name: private-pool-dev-uks + lockBehavior: sequential + isSkippable: false - jobs: - - deployment: DeployApp - displayName: Deploy application - environment: ${{ env }} - strategy: - runOnce: - deploy: - steps: - - checkout: self + jobs: + - deployment: DeployApp + displayName: Deploy application + environment: ${{ parameters.environment }} + strategy: + runOnce: + deploy: + steps: + - checkout: self - - task: TerraformInstaller@1 - displayName: Install terraform - inputs: - terraformVersion: 1.7.0 + - task: TerraformInstaller@1 + displayName: Install terraform + inputs: + terraformVersion: 1.7.0 - - task: AzureCLI@2 - displayName: Run terraform - inputs: - azureSubscription: manbrs-${{ env }} - scriptType: bash - scriptLocation: inlineScript - addSpnToEnvironment: true - inlineScript: | - export ARM_TENANT_ID="$tenantId" - export ARM_CLIENT_ID="$servicePrincipalId" - export ARM_OIDC_TOKEN="$idToken" - export ARM_USE_OIDC=true - make ci ${{ env }} terraform-apply DOCKER_IMAGE_TAG=git-sha-${{ parameters.commitSHA }} + - task: AzureCLI@2 + displayName: Run terraform + inputs: + azureSubscription: manbrs-${{ parameters.environment }} + scriptType: bash + scriptLocation: inlineScript + addSpnToEnvironment: true + inlineScript: | + export ARM_TENANT_ID="$tenantId" + export ARM_CLIENT_ID="$servicePrincipalId" + export ARM_OIDC_TOKEN="$idToken" + export ARM_USE_OIDC=true + make ci ${{ parameters.environment }} terraform-apply DOCKER_IMAGE_TAG=git-sha-${{ parameters.commitSHA }} PR_NUMBER=${{ parameters.prNumber }} - - task: AzureCLI@2 - displayName: Run database migration - inputs: - azureSubscription: manbrs-${{ env }} - scriptType: bash - scriptLocation: inlineScript - addSpnToEnvironment: true - inlineScript: ./scripts/bash/db_migrate.sh ${{ env }} + - task: AzureCLI@2 + displayName: Run database migration + inputs: + azureSubscription: manbrs-${{ parameters.environment }} + scriptType: bash + scriptLocation: inlineScript + addSpnToEnvironment: true + inlineScript: ./scripts/bash/db_migrate.sh ${{ parameters.environment }} ${{ parameters.prNumber }} diff --git a/.github/workflows/cicd-1-pull-request-closed.yaml b/.github/workflows/cicd-1-pull-request-closed.yaml new file mode 100644 index 000000000..846f4e8f1 --- /dev/null +++ b/.github/workflows/cicd-1-pull-request-closed.yaml @@ -0,0 +1,46 @@ +name: Delete review app + +on: + pull_request: + types: [closed] + +jobs: + destroy: + if: contains(github.event.pull_request.labels.*.name, 'deploy') + name: Delete review app pr-${{ github.event.pull_request.number }} + permissions: + id-token: write + pull-requests: write + runs-on: ubuntu-latest + environment: review + concurrency: deploy-review-${{ github.ref }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Call delete review app pipeline + run: | + echo "Starting Azure devops pipeline \"Delete review app\"..." + RUN_ID=$(az pipelines run \ + --commit-id ${{ github.event.pull_request.head.sha }}\ + --name "Delete review app"\ + --org https://dev.azure.com/nhse-dtos \ + --project dtos-manage-breast-screening \ + --parameters commitSHA=${{ github.event.pull_request.head.sha }} prNumber=${{ github.event.pull_request.number }} \ + --output tsv --query id) + + scripts/bash/wait_ado_pipeline.sh "$RUN_ID" https://dev.azure.com/nhse-dtos dtos-manage-breast-screening + + - name: Post URL to PR comments + uses: marocchino/sticky-pull-request-comment@8ac02941f254c53fbda0cf44288785e1367e13bf + with: + message: | + The review app at this URL has been deleted: + https://pr-${{ github.event.pull_request.number }}.manage-breast-screening.non-live.screening.nhs.uk diff --git a/.github/workflows/cicd-1-pull-request.yaml b/.github/workflows/cicd-1-pull-request.yaml index 925505447..c14ed3a10 100644 --- a/.github/workflows/cicd-1-pull-request.yaml +++ b/.github/workflows/cicd-1-pull-request.yaml @@ -2,7 +2,7 @@ name: 'CI/CD pull request' on: pull_request: - types: [opened, reopened, synchronize] + types: [opened, reopened, synchronize, labeled] jobs: metadata: @@ -72,3 +72,30 @@ jobs: terraform_version: '${{ needs.metadata.outputs.terraform_version }}' version: '${{ needs.metadata.outputs.version }}' secrets: inherit + deploy-stage: + if: contains(github.event.pull_request.labels.*.name, 'deploy') + name: Deploy review app pr-${{ github.event.pull_request.number }} + needs: [build-stage] + permissions: + id-token: write + uses: ./.github/workflows/stage-4-deploy.yaml + with: + environments: "[\"review\"]" + commit_sha: ${{ github.event.pull_request.head.sha }} + pr_number: ${{ github.event.pull_request.number }} + secrets: inherit + post-url: + if: contains(github.event.pull_request.labels.*.name, 'deploy') + name: Post URL pr-${{ github.event.pull_request.number }} to PR comments + runs-on: ubuntu-latest + needs: [deploy-stage] + permissions: + pull-requests: write + steps: + - name: Post URL to PR comments + uses: marocchino/sticky-pull-request-comment@8ac02941f254c53fbda0cf44288785e1367e13bf + with: + message: | + The review app is available at this URL: + https://pr-${{ github.event.pull_request.number }}.manage-breast-screening.non-live.screening.nhs.uk + You must authenticate with Entra ID diff --git a/.github/workflows/cicd-2-main-branch.yaml b/.github/workflows/cicd-2-main-branch.yaml index cc8fdde7d..87375c99f 100644 --- a/.github/workflows/cicd-2-main-branch.yaml +++ b/.github/workflows/cicd-2-main-branch.yaml @@ -85,5 +85,6 @@ jobs: id-token: write uses: ./.github/workflows/stage-4-deploy.yaml with: + environments: "[\"review\",\"dev\"]" commit_sha: ${{ github.sha }} secrets: inherit diff --git a/.github/workflows/stage-4-deploy.yaml b/.github/workflows/stage-4-deploy.yaml index e36eb0e44..f4cdd8c15 100644 --- a/.github/workflows/stage-4-deploy.yaml +++ b/.github/workflows/stage-4-deploy.yaml @@ -3,16 +3,29 @@ name: Deployment stage on: workflow_call: inputs: + environments: + description: List of environments to deploy to (String array) + required: true + type: string commit_sha: description: Commit SHA used to fetch ADO pipeline and docker image required: true type: string + pr_number: + description: Pull request number when used in a pull request + required: false + type: string jobs: deploy: name: Deploy runs-on: ubuntu-latest - environment: azure + strategy: + matrix: + environment: ${{ fromJson(inputs.environments) }} + max-parallel: 1 + environment: ${{ matrix.environment }} + concurrency: deploy-${{ matrix.environment }}-${{ github.ref }} steps: - name: Checkout code @@ -26,5 +39,13 @@ jobs: - name: Call deployment pipeline run: | - az pipelines run --commit-id ${{inputs.commit_sha}} --name "Deploy to Azure" --org https://dev.azure.com/nhse-dtos --project dtos-manage-breast-screening \ - --parameters commitSHA=${{inputs.commit_sha}} + echo "Starting Azure devops pipeline \"Deploy to Azure - ${{ matrix.environment }}\"..." + RUN_ID=$(az pipelines run \ + --commit-id ${{inputs.commit_sha}} \ + --name "Deploy to Azure - ${{ matrix.environment }}" \ + --org https://dev.azure.com/nhse-dtos \ + --project dtos-manage-breast-screening \ + --parameters commitSHA=${{inputs.commit_sha}} prNumber=${{inputs.pr_number}} environment=${{ matrix.environment }} \ + --output tsv --query id) + + scripts/bash/wait_ado_pipeline.sh "$RUN_ID" https://dev.azure.com/nhse-dtos dtos-manage-breast-screening diff --git a/.gitleaksignore b/.gitleaksignore index 12a4c6d28..5fb74c4dc 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -10,5 +10,10 @@ manage_breast_screening/templates/components/pagination/template.njk:ipv4:26 infrastructure/terraform/resource_group_init/core.bicep:generic-api-key:10 infrastructure/terraform/resource_group_init/core.bicep:generic-api-key:11 infrastructure/terraform/resource_group_init/core.bicep:generic-api-key:12 -infrastructure/terraform/resource_group_init/main.bicep:generic-api-key:80 +infrastructure/terraform/resource_group_init/main.bicep:generic-api-key:29 +infrastructure/terraform/resource_group_init/main.bicep:generic-api-key:30 +infrastructure/terraform/resource_group_init/main.bicep:generic-api-key:31 +infrastructure/terraform/resource_group_init/main.bicep:generic-api-key:32 +infrastructure/terraform/resource_group_init/main.bicep:generic-api-key:33 infrastructure/terraform/resource_group_init/storage.bicep:generic-api-key:59 +infrastructure/terraform/resource_group_init/keyVault.bicep:generic-api-key:10 diff --git a/README.md b/README.md index 872d855c8..7a75a922b 100644 --- a/README.md +++ b/README.md @@ -110,58 +110,9 @@ To generate a new app, run: poetry run ./manage.py startapp manage_breast_screening/` ``` -## Manual Deployment +## Deployment -The build pipeline builds and pushes a docker image to [Github container registry](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry). The app is deployed to an [Azure container app](https://azure.microsoft.com/en-us/products/container-apps) using terraform. - -For each environment, e.g. 'dev': - -1. Connect to [Azure virtual desktop](https://azure.microsoft.com/en-us/products/virtual-desktop). Ask the platform team for access with Administrator role. -1. If not present, install the following software: terraform (version 1.7.0), git, make, jq. - - Run a Command prompt as administrator - - choco install terraform --version 1.7.0 - - choco install terraform git make jq -1. Open git bash -1. Clone the repository: `git clone https://github.com/NHSDigital/dtos-manage-breast-screening.git` -1. Enter the directory and select the branch, tag, commit... -1. Login: `az login` -1. Create the resource group: `make dev resource-group-init`. This is only required when creating the environment from scratch. -1. Deploy: - ```shell - make dev terraform-plan DOCKER_IMAGE_TAG=git-sha-af32637e7e6a07e36158dcb8d7ed90be49be1xyz - ``` -1. The web app URL will be displayed as output. Copy it into a browser on the AVD to access the app. - -## Manual deployment of the review environments - -Review environments differ slightly from other environments. They are lightweight versions of the application and are designed to share much of the core Azure infrastructure. As a result, there is a one-to-many relationship between the container apps and the container app environment. - -### Step 1 -If you run the following command *without* the `PR_NUMBER` parameter, it will apply only the infrastructure module: - -```shell -make review terraform-apply -``` - -### Step 2 - -If you include the `PR_NUMBER` parameter, it will apply the container_app module instead of the infrastructure module: - -```shell -make review terraform-apply DOCKER_IMAGE_TAG=git-sha-01ecb79d561f55be60072a093dd167fe8eb5b42e PR_NUMBER=123 -``` - -## Continuous deployment - -When a PR is merged, Github actions securely triggers the deployment pipeline on the Azure devops pool running on the internal network. It currently deploys the dev environment automatically. - -Access [Azure devops](https://dev.azure.com/nhse-dtos/dtos-manage-breast-screening/_build?definitionId=86) to see the pipeline. - -## Application secrets - -The app requires secrets provided as environment variables. Terraform creates an Azure key vault and all its secrets are mapped directly to the app as environment variables. Devs can access the key vault to create and update the secrets manually. - -Note [the process requires multiple steps](https://github.com/NHSDigital/dtos-devops-templates/tree/main/infrastructure/modules/container-app#key-vault-secrets) to set up an environment initially. +See [Deployment](docs/infrastructure/deployment.md). ## Contributing diff --git a/docs/infrastructure/create-environment.md b/docs/infrastructure/create-environment.md new file mode 100644 index 000000000..eed2e7fd3 --- /dev/null +++ b/docs/infrastructure/create-environment.md @@ -0,0 +1,83 @@ +# Create an environment + +This is the initial manual process to create a new environment like review, dev, production... + +## Code +- Create the configuration files in `infrastructure/environments/[environment]` +- Add the `[environment]:` target in `scripts/terraform/terraform.mk` +- Add [environment] to the list of environments in `deploy-stage` step of `cicd-2-main-branch.yaml`. For the review enviornment, there is a single item in `cicd-1-pull-request.yaml`. +- Set the `fetch_secrets_from_app_key_vault` terraform variable to `false`. This is to let terraform create the key vault and prevent reading before it is ready. + +## Entra ID +- Create postgres Entra ID group in `DTOS Administrative Unit (AU)`: `postgres_manbrs_[environment]_uks_admin` +- Ask CCOE to assign role: + - [Form for PIM](https://nhsdigitallive.service-now.com/nhs_digital?id=sc_cat_item&sys_id=28f3ab4f1bf3ca1078ac4337b04bcb78&sysparm_category=114fced51bdae1502eee65b9bd4bcbdc) + - Approver: Add someone from the infrastructure team + - Role Name: `Group.Read.All` + - Application Name: `mi-manbrs-[environment]-adotoaz-uks` + - Application ID: [client.id] + - Managed identity: `mi-manbrs-[environment]-adotoaz-uks` + - Description: + - Managed identity: `mi-manbrs-[environment]-adotoaz-uks` + - Role: permanent on Directory + +## Bicep +- Run bicep from AVD: `make [environment] resource-group-init` + +## Infra secrets +- Add the infrastructure secrets to the *inf* key vault `kv-manbrs-[review]-inf` + +## Azure devops +- Create ADO group + - Name: `Run pipeline - [environment]` + - Members: `mi-manbrs-[environment]-ghtoado-uks`. There may be more than 1 in the list. Check client id printed below the name. + - Permissions: + - View project-level information +- Create new pipeline: + - Name: `Deploy to Azure - [environment]` + - Pipeline yaml: `.azuredevops/pipelines/deploy.yml` +- Manage pipeline security: + - Add group: `Run pipeline - [environment]` + - Permissions: + - Edit queue build configuration + - Queue builds + - View build pipeline + - View builds +- Create service connection (ADO) + - Connection type: `Azure Resource Manager` + - Identity type: `Managed identity` + - Subscription for managed identity: `Digital Screening DToS - Devops` + - Resource group for managed identity: `rg-mi-[environment]-uks` + - Managed identity: `mi-manbrs-[environment]-adotoaz-uks` + - Scope level: `Subscription` + - Subscription: `Digital Screening DToS - Core Services Dev` + - Resource group for Service connection: leave blank + - Service Connection Name: `manbrs-[environment]` + - Do NOT tick: Grant access permission to all pipelines + - Security: allow `Deploy to Azure - [environment]` pipeline +- Create ADO environment: [environment] + - Set: exclusive lock (except for review) + - Add pipeline permission for `Deploy to Azure - [environment]` pipeline + +## Github +- Create Github environment [environment] +- Add the protection rule (except in review): + - Deselect `Allow administrators to bypass configured protection rules` + - In `Deployment branches and tags` choose `Selected branches and tags` from the drop-down menu + - Click `Add deployment branch or tag rule` and enter "main" +- Add environment secrets, from `mi-manbrs-[environment]-ghtoado-uks` in github + - *AZURE_CLIENT_ID* + - *AZURE_SUBSCRIPTION_ID* + +## First run +- Test running terraform manually from the AVD (Optional) +- Raise a pull request, review and merge to trigger the pipeline +- Check ADO pipeline. You may be prompted to authorise: + - Pipeline: service connection + - Environment: service connection and agent pool + +## App secrets +- Add the application secrets to the *app* key vault `kv-manbrs-[review]-app` +- Set `fetch_secrets_from_app_key_vault` terraform variable to `true` +- Test running terraform manually from the AVD (Optional) +- Raise a pull request, review and merge to trigger the pipeline diff --git a/docs/infrastructure/deployment.md b/docs/infrastructure/deployment.md new file mode 100644 index 000000000..3183d80fc --- /dev/null +++ b/docs/infrastructure/deployment.md @@ -0,0 +1,96 @@ +# Deployment + +## Infrastructure +The code is packaged into a docker image which is deployed to [Azure container apps](https://learn.microsoft.com/en-us/azure/container-apps/). The main app is a web application, with an HTTP ingress. And the second one is an [Azure container app job](https://learn.microsoft.com/en-us/azure/container-apps/jobs?tabs=azure-cli), triggered on demand to run the database migration. + +The web application does not have a public endpoint. It is only accessible via [Azure front door](https://learn.microsoft.com/en-us/azure/frontdoor/) which is a CDN providing TLS certificates, firewall, scaling and caching. The internal endpoint is accessible via [Azure Virtual Desktop](https://learn.microsoft.com/en-us/azure/virtual-desktop/). + +The data is hosted on [Azure postgres flexible server](https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/overview). + +## Docker build +The build pipeline builds and pushes a docker image to [Github container registry](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry). The image is tagged with: +- branch name: for docker build caching +- commit SHA: to uniquely identify the image during deployment, prefixed by "git-sha-". +- image digest sha: immutable tag + +## Automated deployment +The deployment is split between: +- [Github actions](https://github.com/features/actions) for Continuous Integration (CI) +- [Azure devops](https://azure.microsoft.com/en-us/products/devops) pipelines for Continuous Deployment (CD) + +### Github actions +Runs on Github hosted runners on the internet. They run all our tests (unit, functional, security, linting...). They don't have access to our internal network nor any sensitive data. + +To deploy an environment, they authenticate to Azure and delegate the work to [Azure devops piplines](#azure-devops-pipelines). + +See [all Github actions](https://github.com/NHSDigital/dtos-manage-breast-screening/actions). + +### Azure devops pipelines +We use a public repository as required by the [NHS Service standard](https://service-manual.nhs.uk/standards-and-technology/service-standard-points/12-make-new-source-code-open). For security reasons, deployments cannot run from Github actions and run instead on Azure devops private runners inside our internal network. They have access to the network and any Azure resource deployed onto it. + +See [all Azure devops pipelines](https://dev.azure.com/nhse-dtos/dtos-manage-breast-screening/_build). + +### Review apps +When a pull request is raised, add a "deploy" label to deploy a *review app* (concept borrowed from [Heroku](https://devcenter.heroku.com/articles/github-integration-review-apps)). It triggers the [CI/CD pull request](https://github.com/NHSDigital/dtos-manage-breast-screening/actions/workflows/cicd-1-pull-request.yaml) Github action workflow, which runs tests then authenticates to Azure and triggers the [Deploy review app](https://dev.azure.com/nhse-dtos/dtos-manage-breast-screening/_build?definitionId=102) Azure devops pipeline. It runs terraform to deploy the application, database and front door configuration. + +To make this process faster and less costly, most of the infrastructure is reused for all review apps: networking, key vaults, container app environments... The base infrastructure is only updated by the pipeline on the main branch. + +When the pull request is closed or merged, and if it has the "deploy" label, the [Delete review app](https://github.com/NHSDigital/dtos-manage-breast-screening/actions/workflows/cicd-1-pull-request-closed.yaml) workflow is triggered, followed by the [Delete review app](https://dev.azure.com/nhse-dtos/dtos-manage-breast-screening/_build?definitionId=103) Azure devops pipeline. It runs *terraform destroy* to delete the resources. + +Note: terraform currently deploys a postgres server with a locked database. It must be deleted manually from the Azure portal before the pipeline runs. + +### Main branch +When a pull request is merged to the main branch, the [CI/CD main branch](https://github.com/NHSDigital/dtos-manage-breast-screening/actions/workflows/cicd-2-main-branch.yaml) is triggered. It runs tests then authenticates to Azure and triggers the [Deploy to Azure](https://dev.azure.com/nhse-dtos/dtos-manage-breast-screening/_build?definitionId=93) Azure devops pipeline. It runs terraform to deploy the entire environment, including both infrastructure and applications. Any manual change is overwritten by terraform. + +## Application secrets +The application requires secrets provided as environment variables. Terraform creates an *app* Azure key vault and all its secrets are mapped directly to the app as environment variables. Developers can access the key vault to create and update the secrets manually. + +Notes: +- [the process requires multiple steps](https://github.com/NHSDigital/dtos-devops-templates/tree/main/infrastructure/modules/container-app#key-vault-secrets) to set up an environment initially. The process is documented in [create-environment](create-environment.md). +- The secrets names in key vault are uppercase with hyphen separators. They are mapped to environment variables as uppercase with underscore separator. e.g. `SECRET-KEY` is mapped to `SECRET_KEY`. + +## Manual deployment +For each environment, e.g. 'dev': + +1. Connect to [Azure virtual desktop](https://azure.microsoft.com/en-us/products/virtual-desktop). Ask the platform team for access with Administrator role. +1. If not present, install the following software: terraform (version 1.7.0), git, make, jq. + - Run a Command prompt as administrator + - choco install terraform --version 1.7.0 + - choco install terraform git make jq +1. Open git bash +1. Clone the repository: `git clone https://github.com/NHSDigital/dtos-manage-breast-screening.git` +1. Enter the directory and select the branch, tag, commit... +1. Login: `az login` +1. Create the resource group: `make dev resource-group-init`. This is only required when creating the environment from scratch. +1. Deploy: + ```shell + make dev terraform-plan DOCKER_IMAGE_TAG=git-sha-af32637e7e6a07e36158dcb8d7ed90be49be1xyz + ``` +1. The web app URL will be displayed as output. Copy it into a browser on the AVD to access the app. + +## Manual deployment of the review app environments + +[Review app environments](#review-apps) differ slightly from other environments. They are lightweight versions of the application and are designed to share much of the core Azure infrastructure. As a result, there is a one-to-many relationship between the container apps and the container app environment. + +### Step 1 +If you run the following command *without* the `PR_NUMBER` parameter, it will apply only the infrastructure module: + +```shell +make review terraform-apply +``` + +This is usually deployed by the pipeline on the main branch. + +### Step 2 + +If you include the `PR_NUMBER` parameter (it can be any number), it will apply the container-apps module instead of the infrastructure module: + +```shell +make review terraform-apply DOCKER_IMAGE_TAG=git-sha-01ecb79d561f55be60072a093dd167fe8eb5b42e PR_NUMBER=123 +``` + +### Delete review app +Run terraform-destroy: +```shell +make review terraform-destroy DOCKER_IMAGE_TAG=git-sha-01ecb79d561f55be60072a093dd167fe8eb5b42e PR_NUMBER=123 +``` diff --git a/docs/infrastructure/infra-faq.md b/docs/infrastructure/infra-faq.md index 86406a2d5..d44431deb 100644 --- a/docs/infrastructure/infra-faq.md +++ b/docs/infrastructure/infra-faq.md @@ -1,4 +1,12 @@ -## Import into terraform state file +# Infra FAQ + +- [Terraform](#terraform) +- [Github action triggering Azure devops pipeline](#github-action-triggering-azure-devops-pipeline) +- [Bicep errors](#bicep-errors) + + +## Terraform +### Import into terraform state file To import Azure resources into the Terraform state file, you can use the following command. If you're working on an AVD machine, you may need to set the environment variables: - `ARM_USE_AZUREAD` to use Azure AD instead of a shared key @@ -13,7 +21,7 @@ export MSYS_NO_PATHCONV=true terraform -chdir=infrastructure/terraform import -var-file ../environments/${ENV_CONFIG}/variables.tfvars module.infra[0].module.postgres_subnet.azurerm_subnet.subnet /subscriptions/xxx/resourceGroups/rg-manbrs-review-uks/providers/Microsoft.Network/virtualNetworks/vnet-review-uks-manbrs/subnets/snet-postgres ``` -## Error: Failed to load state +### Error: Failed to load state This happens when running terraform commands accessing the state file like [import](#import-into-terraform-state-file), `state list` or `force-unlock`. ``` Failed to load state: blobs.Client#Get: Failure responding to request: StatusCode=403 -- Original Error: autorest/azure: Service returned an error. Status=403 Code="KeyBasedAuthenticationNotPermitted" Message="Key based authentication is not permitted on this storage account. @@ -24,3 +32,105 @@ By default terraform tries using a shared key, which is not allowed. To force us ```shell ARM_USE_AZUREAD=true terraform force-unlock xxx-yyy ``` + +## Github action triggering Azure devops pipeline +### Application with identifier '***' was not found in the directory +Example: +``` +Running Azure CLI Login. +... +Attempting Azure CLI login by using OIDC... +Error: AADSTS700016: Application with identifier '***' was not found in the directory 'NHS Strategic Tenant'. This can happen if the application has not been installed by the administrator of the tenant or consented to by any user in the tenant. You may have sent your authentication request to the wrong tenant. Trace ID: xxx Correlation ID: xxx Timestamp: xxx + +Error: Interactive authentication is needed. Please run: +az login +``` +The managed identity does not exist or Github secrets are not set correctly + +### The client '***' has no configured federated identity credentials +Example: +``` +Running Azure CLI Login. +... +Attempting Azure CLI login by using OIDC... +Error: AADSTS70025: The client '***'(mi-manbrs-ado-review-temp) has no configured federated identity credentials. Trace ID: xxx Correlation ID: xxx Timestamp: xxx + +Error: Interactive authentication is needed. Please run: +az login +``` +Federated credentials are not configured. + +### No subscriptions found for *** +Example: +``` +Running Azure CLI Login. +... +Attempting Azure CLI login by using OIDC... +Error: No subscriptions found for ***. +``` +Give the managed identity Reader role on a subscription (normally Devops) + +### Pipeline permissions +Examples: +``` +ERROR: TF401444: Please sign-in at least once as ***\***\xxx in a web browser to enable access to the service. +Error: Process completed with exit code 1. +``` +Or +``` +ERROR: TF400813: The user 'xxx' is not authorized to access this resource. +Error: Process completed with exit code 1. +``` +Or +``` +ERROR: VS800075: The project with id 'vstfs:///Classification/TeamProject/' does not exist, or you do not have permission to access it. +Error: Process completed with exit code 1. +``` +The Github secret must reflect the right managed identity, the managed identity must have the following permissions on the pipeline, via its ADO group: +- Edit queue build configuration +- Queue builds +- View build pipeline + +The ADO group must have the "View project-level information" permission. + +### The service connection does not exist +Example: +``` +The pipeline is not valid. Job DeployApp: Step input azureSubscription references service connection manbrs-review which could not be found. The service connection does not exist, has been disabled or has not been authorized for use. For authorization details, refer to https://aka.ms/yamlauthz. Job DeployApp: Step input azureSubscription references service connection manbrs-review which could not be found. The service connection does not exist, has been disabled or has not been authorized for use. For authorization details, refer to https://aka.ms/yamlauthz. +``` +The Azure service connection manbrs-[environment] is missing + +## Bicep errors +### RoleAssignmentUpdateNotPermitted +Example: +``` +ERROR: {"status":"Failed","error":{"code":"DeploymentFailed","target":"/subscriptions/xxx/providers/Microsoft.Resources/deployments/main","message":"At least one reson failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.","details":[{"code":"RoleAssignmentUpdateNotPermitted","message":"Tenprincipal ID, and scope are not allowed to be updated."},{"code":"RoleAssignmentUpdateNotPermitted","message":"Tenant ID, application ID, principal ID, and scope are not allowed to be updated."},{"cteNotPermitted","message":"Tenant ID, application ID, principal ID, and scope are not allowed to be updated."}]}} +``` +When deleting a MI, its role assignment is not deleted. When recreating the MI, bicep tries to update the role assignment and is not allowed to. Solution: +- Find the role assignment id. Here: abcd-123 +- Navigate to subscriptions and resource group IAM and search for the role assignment id +- Delete the role assignment via the portal + +If you can't find the right scope, follow this process: +- Find the role assignment id. Here: abcd-123 +``` + ~ Microsoft.Authorization/roleAssignments/abcd-123 [2022-04-01] +    ~ properties.principalId: "xxx" => "[reference('/subscriptions/xxx/resourceGroups/rg-mi-review-uks/providers/Microsoft.ManagedIdentity/userAssignedIdentities/mi-manbrs-ado-review-uks', '2024-11-30').principalId]" +``` +- Get the subscription id +- List role assignments: `az role assignment list --scope "/subscriptions/[subscription id]"` +- Look for the role assignment id abcd-123 to retrieve the other details. It may named: Unknown. +- Delete the role assignment via the portal + +### PrincipalNotFound +Example: +``` +ERROR: {"status":"Failed","error":{"code":"DeploymentFailed","target":"/subscriptions/exxx/providers/Microsoft.Resources/deployments/main","message":"At least one reson failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.","details":[{"code":"PrincipalNotFound","message":"Principal xxx does not exist in the directory xxx. Check that you have the correct principal ID. If you are creating this principal and then immediately assigning a role, this era replication delay. In this case, set the role assignment principalType property to a value, such as ServicePrincipal, User, or Group.  See https://aka.ms/docs-principaltype"}... +``` +Race condition: the managed identity is not created in time for the resources that depend on it. Solution: rerun the command. + +### The client does not have permission +``` +{"code": "InvalidTemplateDeployment", "message": "Deployment failed with multiple errors: 'Authorization failed for template resource 'xxx' of type 'Microsoft.Authorization/roleAssignments'. The client 'xxx' with object id 'xxx' does not have permission to perform action 'Microsoft.Authorization/roleAssignments/write' at scope '/subscriptions/xxx/providers/Microsoft.Authorization/roleAssignments/xxx'... +``` +Request Owner role on subscriptions via PIM. diff --git a/infrastructure/terraform/resource_group_init/core.bicep b/infrastructure/terraform/resource_group_init/core.bicep index b20fca821..3a6fd694d 100644 --- a/infrastructure/terraform/resource_group_init/core.bicep +++ b/infrastructure/terraform/resource_group_init/core.bicep @@ -8,7 +8,7 @@ param miName string // See: https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles var roleID = { contributor: 'b24988ac-6180-42a0-ab88-20f7382dd24c' - kvSecretUser: '4633458b-17de-408a-b874-0445c86b69e6' + kvSecretsUser: '4633458b-17de-408a-b874-0445c86b69e6' rbacAdmin: 'f58310d9-a9f6-439a-9e8d-f62e7b41a168' } @@ -23,12 +23,12 @@ resource contributorAssignment 'Microsoft.Authorization/roleAssignments@2022-04- } // Let the managed identity read key vault secrets during terraform plan -resource kvSecretUserAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(subscription().subscriptionId, miPrincipalId, 'kvSecretUser') +resource kvSecretsUserAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().subscriptionId, miPrincipalId, 'kvSecretsUser') properties: { - roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleID.kvSecretUser) + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleID.kvSecretsUser) principalId: miPrincipalId - description: '${miName} kvSecretUser access to subscription' + description: '${miName} kvSecretsUser access to subscription' } } @@ -38,8 +38,8 @@ resource rbacAdminAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01 properties: { roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleID.rbacAdmin) principalId: miPrincipalId - condition: '((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/write\'})) OR (@Request[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.kvSecretUser}})) AND ((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/delete\'})) OR (@Resource[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.kvSecretUser}}))' + condition: '((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/write\'})) OR (@Request[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.kvSecretsUser}})) AND ((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/delete\'})) OR (@Resource[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.kvSecretsUser}}))' conditionVersion: '2.0' - description: '${miName} Role Based Access Control Administrator access to subscription. Only allows assigning the Key Vault Secrets User role to Service Principals.' + description: '${miName} Role Based Access Control Administrator access to subscription. Only allows assigning the Key Vault Secrets User role.' } } diff --git a/infrastructure/terraform/resource_group_init/keyVault.bicep b/infrastructure/terraform/resource_group_init/keyVault.bicep index 9705fb079..3207169bc 100644 --- a/infrastructure/terraform/resource_group_init/keyVault.bicep +++ b/infrastructure/terraform/resource_group_init/keyVault.bicep @@ -1,8 +1,15 @@ param enableSoftDelete bool param keyVaultName string +param miPrincipalId string +param miName string param region string +// See: https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles +var roleID = { + kvSecretsUser: '4633458b-17de-408a-b874-0445c86b69e6' +} + resource keyVault 'Microsoft.KeyVault/vaults@2024-11-01' = { name: keyVaultName location: region @@ -21,5 +28,15 @@ resource keyVault 'Microsoft.KeyVault/vaults@2024-11-01' = { } } +// Let the managed identity read key vault secrets during terraform plan +resource kvSecretsUserAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().subscriptionId, miPrincipalId, 'kvSecretsUser') + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleID.kvSecretsUser) + principalId: miPrincipalId + description: '${miName} kvSecretsUser access to resource group' + } +} + // Output the key vault ID so it can be used to create the private endpoint output keyVaultID string = keyVault.id diff --git a/infrastructure/terraform/resource_group_init/main.bicep b/infrastructure/terraform/resource_group_init/main.bicep index cada0d0de..96b9577ee 100644 --- a/infrastructure/terraform/resource_group_init/main.bicep +++ b/infrastructure/terraform/resource_group_init/main.bicep @@ -21,6 +21,18 @@ var managedIdentityRGName = 'rg-mi-${envConfig}-uks' var infraResourceGroupName = 'rg-manbrs-${envConfig}-infra' var keyVaultName = 'kv-manbrs-${envConfig}-inf' +var miADOtoAZname = 'mi-${appShortName}-${envConfig}-adotoaz-uks' +var miGHtoADOname = 'mi-${appShortName}-${envConfig}-ghtoado-uks' + +// See: https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles +var roleID = { + CDNContributor: 'ec156ff8-a8d1-4d15-830c-5b80698ca432' + kvSecretsUser: '4633458b-17de-408a-b874-0445c86b69e6' + networkContributor: '4d97b98b-1d4f-4787-a291-c67834d212e7' + rbacAdmin: 'f58310d9-a9f6-439a-9e8d-f62e7b41a168' + reader: 'acdd72a7-3385-48ef-bd42-f606fba81ae7' +} + // Retrieve existing terraform state resource group resource storageAccountRG 'Microsoft.Resources/resourceGroups@2024-11-01' existing = { name: storageAccountRGName @@ -38,13 +50,36 @@ resource managedIdentityRG 'Microsoft.Resources/resourceGroups@2024-11-01' exist name: managedIdentityRGName } -// Create the managed identity for CD -module managedIdentiy 'managedIdentity.bicep' = { +// Create the managed identity assumed by Azure devops to connect to Azure +module managedIdentiyADOtoAZ 'managedIdentity.bicep' = { scope: managedIdentityRG params: { + name: miADOtoAZname region: region - appShortName: appShortName - envConfig: envConfig + } +} + +// Create the managed identity assumed by Github actions to trigger Azure devops pipelines +module managedIdentiyGHtoADO 'managedIdentity.bicep' = { + scope: managedIdentityRG + params: { + name: miGHtoADOname + fedCredProperties: { + audiences: [ 'api://AzureADTokenExchange' ] + issuer: 'https://token.actions.githubusercontent.com' + subject: 'repo:NHSDigital/dtos-manage-breast-screening:environment:${envConfig}' + } + region: region + } +} + +// Let the GHtoADO managed identity access a subscription +resource readerAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().subscriptionId, envConfig, 'reader') + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleID.reader) + principalId: managedIdentiyGHtoADO.outputs.miPrincipalID + description: '${miGHtoADOname} Reader access to subscription' } } @@ -55,8 +90,8 @@ module terraformStateStorageAccount 'storage.bicep' = { storageLocation: region storageName: storageAccountName enableSoftDelete: enableSoftDelete - miPrincipalID: managedIdentiy.outputs.miPrincipalID - miName: managedIdentiy.outputs.miName + miPrincipalID: managedIdentiyADOtoAZ.outputs.miPrincipalID + miName: miADOtoAZname } } @@ -89,19 +124,13 @@ module storageAccountPrivateEndpoint 'privateEndpoint.bicep' = { } } -// See: https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles -var roleID = { - CDNContributor: 'ec156ff8-a8d1-4d15-830c-5b80698ca432' - networkContributor: '4d97b98b-1d4f-4787-a291-c67834d212e7' -} - // Let the managed identity configure vnet peering and DNS records resource networkContributorAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { name: guid(subscription().subscriptionId, envConfig, 'networkContributor') properties: { roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleID.networkContributor) - principalId: managedIdentiy.outputs.miPrincipalID - description: '${managedIdentiy.outputs.miName} Network Contributor access to subscription' + principalId: managedIdentiyADOtoAZ.outputs.miPrincipalID + description: '${miADOtoAZname} Network Contributor access to subscription' } } @@ -129,9 +158,11 @@ module keyVaultModule 'keyVault.bicep' = { name: 'keyVaultDeployment' scope: resourceGroup(infraResourceGroupName) params: { + enableSoftDelete : enableSoftDelete keyVaultName: keyVaultName + miName: miADOtoAZname + miPrincipalId: managedIdentiyADOtoAZ.outputs.miPrincipalID region: region - enableSoftDelete : enableSoftDelete } } @@ -140,12 +171,24 @@ resource CDNContributorAssignment 'Microsoft.Authorization/roleAssignments@2022- name: guid(subscription().subscriptionId, envConfig, 'CDNContributor') properties: { roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleID.CDNContributor) - principalId: managedIdentiy.outputs.miPrincipalID - description: '${managedIdentiy.outputs.miName} CDN Contributor access to subscription' + principalId: managedIdentiyADOtoAZ.outputs.miPrincipalID + description: '${miADOtoAZname} CDN Contributor access to subscription' + } +} + +// Let the managed identity assign the Key Vault Secrets User role to the container app managed identity +resource rbacAdminAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().subscriptionId, envConfig, 'rbacAdmin') + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleID.rbacAdmin) + principalId: managedIdentiyADOtoAZ.outputs.miPrincipalID + condition: '((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/write\'})) OR (@Request[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.kvSecretsUser}})) AND ((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/delete\'})) OR (@Resource[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.kvSecretsUser}}))' + conditionVersion: '2.0' + description: '${miADOtoAZname} Role Based Access Control Administrator access to subscription. Only allows assigning the Key Vault Secrets User role.' } } -output miPrincipalID string = managedIdentiy.outputs.miPrincipalID -output miName string = managedIdentiy.outputs.miName +output miPrincipalID string = managedIdentiyADOtoAZ.outputs.miPrincipalID +output miName string = miADOtoAZname output keyVaultPrivateDNSZone string = keyVaultPrivateDNSZone.outputs.privateDNSZoneID output storagePrivateDNSZone string = storagePrivateDNSZone.outputs.privateDNSZoneID diff --git a/infrastructure/terraform/resource_group_init/managedIdentity.bicep b/infrastructure/terraform/resource_group_init/managedIdentity.bicep index c294699c5..d0deea68c 100644 --- a/infrastructure/terraform/resource_group_init/managedIdentity.bicep +++ b/infrastructure/terraform/resource_group_init/managedIdentity.bicep @@ -1,13 +1,16 @@ +param name string param region string -param appShortName string -param envConfig string - -var miName = 'mi-${appShortName}-${envConfig}-uks' +param fedCredProperties object = {} resource mi 'Microsoft.ManagedIdentity/userAssignedIdentities@2024-11-30' = { location: region - name: miName + name: name +} + +resource managedIdentiyGHtoADOFedCred 'Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials@2024-11-30' = if (!empty(fedCredProperties)) { + parent: mi + name: 'github-actions' + properties: fedCredProperties } output miPrincipalID string = mi.properties.principalId -output miName string = miName diff --git a/infrastructure/terraform/resource_group_init/storage.bicep b/infrastructure/terraform/resource_group_init/storage.bicep index 3978a73d9..6ebc97e3d 100644 --- a/infrastructure/terraform/resource_group_init/storage.bicep +++ b/infrastructure/terraform/resource_group_init/storage.bicep @@ -32,12 +32,10 @@ resource blobService 'Microsoft.Storage/storageAccounts/blobServices@2024-01-01' containerDeleteRetentionPolicy: { days: enableSoftDelete ? 15 : null enabled: enableSoftDelete - allowPermanentDelete: enableSoftDelete } deleteRetentionPolicy: { days: enableSoftDelete ? 15 : null enabled: enableSoftDelete - allowPermanentDelete: enableSoftDelete } isVersioningEnabled: true } diff --git a/scripts/bash/db_migrate.sh b/scripts/bash/db_migrate.sh index b3aacdac2..48c352e2c 100755 --- a/scripts/bash/db_migrate.sh +++ b/scripts/bash/db_migrate.sh @@ -2,7 +2,17 @@ set -euo pipefail -ENV=$1 +ENV_CONFIG=$1 +PR_NUMBER=${2:-} + +if [ -z "$PR_NUMBER" ]; then + # On permanent environments, the environment name is the environment config name, i.e. "production" + ENV=${ENV_CONFIG} +else + # On a review app, the environment name uses the PR number, i.e. "pr-1234" + ENV=pr-${PR_NUMBER} +fi + JOB_NAME=manbrs-dbm-${ENV} RG_NAME=rg-manbrs-${ENV}-container-app-uks TIMEOUT=300 diff --git a/scripts/bash/resource_group_init.sh b/scripts/bash/resource_group_init.sh index 17810fb66..9ee1819bb 100755 --- a/scripts/bash/resource_group_init.sh +++ b/scripts/bash/resource_group_init.sh @@ -10,17 +10,27 @@ STORAGE_ACCOUNT_NAME="$6" APP_SHORT_NAME="$7" ARM_SUBSCRIPTION_ID="$8" -echo Deploy to the hub subscription... +echo "Deploy to hub subscription $HUB_SUBSCRIPTION_ID..." +az deployment sub create --location "$REGION" --template-file infrastructure/terraform/resource_group_init/main.bicep \ + --subscription "$HUB_SUBSCRIPTION_ID" \ + --parameters enableSoftDelete="$ENABLE_SOFT_DELETE" envConfig="$ENV_CONFIG" region="$REGION" \ + storageAccountRGName="$STORAGE_ACCOUNT_RG" storageAccountName="$STORAGE_ACCOUNT_NAME" appShortName="$APP_SHORT_NAME" --what-if + +read -r -p "Are you sure you want to execute the deployment? (y/n): " confirm +[[ "$confirm" != "y" ]] && exit 0 + output=$(az deployment sub create --location "$REGION" --template-file infrastructure/terraform/resource_group_init/main.bicep \ --subscription "$HUB_SUBSCRIPTION_ID" \ --parameters enableSoftDelete="$ENABLE_SOFT_DELETE" envConfig="$ENV_CONFIG" region="$REGION" \ storageAccountRGName="$STORAGE_ACCOUNT_RG" storageAccountName="$STORAGE_ACCOUNT_NAME" appShortName="$APP_SHORT_NAME") +echo "$output" + echo Capture the outputs... miName=$(echo "$output" | jq -r '.properties.outputs.miName.value') miPrincipalID=$(echo "$output" | jq -r '.properties.outputs.miPrincipalID.value') -echo Deploy to the core subscription... +echo "Deploy to core subscription $ARM_SUBSCRIPTION_ID..." az deployment sub create --location "$REGION" --template-file infrastructure/terraform/resource_group_init/core.bicep \ --subscription "$ARM_SUBSCRIPTION_ID" \ --parameters miName="$miName" miPrincipalId="$miPrincipalID" --confirm-with-what-if diff --git a/scripts/bash/wait_ado_pipeline.sh b/scripts/bash/wait_ado_pipeline.sh new file mode 100755 index 000000000..34a18fd39 --- /dev/null +++ b/scripts/bash/wait_ado_pipeline.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -euo pipefail + +RUN_ID="$1" +ORG_URL="$2" +PROJECT="$3" + +SLEEP_TIME=15 +TIMEOUT_SECONDS=300 + +echo "Waiting for Azure DevOps pipeline run $RUN_ID to complete..." + +START_TIME=$(date +%s) + +while true; do + PIPELINE_JSON=$(az pipelines runs show \ + --id "$RUN_ID" \ + --org "$ORG_URL" \ + --project "$PROJECT" \ + --output json) + STATUS=$(echo "$PIPELINE_JSON" | jq -r '.status') + RESULT=$(echo "$PIPELINE_JSON" | jq -r '.result') + + if [[ "$STATUS" == "completed" ]]; then + if [[ "$RESULT" == "succeeded" ]]; then + echo "Status: $STATUS. Pipeline run $RUN_ID succeeded." + exit 0 + else + echo "Status: $STATUS. Pipeline run $RUN_ID failed with result: $RESULT" + exit 1 + fi + fi + + CURRENT_TIME=$(date +%s) + ELAPSED=$((CURRENT_TIME - START_TIME)) + if (( ELAPSED > TIMEOUT_SECONDS )); then + echo "ERROR: Timeout of ${TIMEOUT_SECONDS}s reached while waiting for pipeline run $RUN_ID." + exit 2 + fi + + echo "Status: $STATUS (Elapsed: ${ELAPSED}s)" + sleep "$SLEEP_TIME" +done